Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2015 Cisco and/or its affiliates. |
| 3 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | * you may not use this file except in compliance with the License. |
| 5 | * You may obtain a copy of the License at: |
| 6 | * |
| 7 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | * |
| 9 | * Unless required by applicable law or agreed to in writing, software |
| 10 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | * See the License for the specific language governing permissions and |
| 13 | * limitations under the License. |
| 14 | */ |
| 15 | #ifndef included_vlib_threads_h |
| 16 | #define included_vlib_threads_h |
| 17 | |
| 18 | #include <vlib/main.h> |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 19 | #include <vppinfra/callback.h> |
Pavel Kotucek | 1e76583 | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 20 | #include <linux/sched.h> |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 21 | |
Damjan Marion | 6a7acc2 | 2016-12-19 16:28:36 +0100 | [diff] [blame] | 22 | extern vlib_main_t **vlib_mains; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 23 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 24 | void vlib_set_thread_name (char *name); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 25 | |
| 26 | /* arg is actually a vlib__thread_t * */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 27 | typedef void (vlib_thread_function_t) (void *arg); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 28 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 29 | typedef struct vlib_thread_registration_ |
| 30 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 31 | /* constructor generated list of thread registrations */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 32 | struct vlib_thread_registration_ *next; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 33 | |
| 34 | /* config parameters */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 35 | char *name; |
| 36 | char *short_name; |
| 37 | vlib_thread_function_t *function; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 38 | uword mheap_size; |
| 39 | int fixed_count; |
| 40 | u32 count; |
| 41 | int no_data_structure_clone; |
Bud Grise | 68adab9 | 2016-02-12 10:36:11 -0500 | [diff] [blame] | 42 | u32 frame_queue_nelts; |
| 43 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 44 | /* All threads of this type run on pthreads */ |
| 45 | int use_pthreads; |
| 46 | u32 first_index; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 47 | uword *coremask; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 48 | } vlib_thread_registration_t; |
| 49 | |
Damjan Marion | 3f46baf | 2016-02-06 19:16:21 +0100 | [diff] [blame] | 50 | /* |
| 51 | * Frames have their cpu / vlib_main_t index in the low-order N bits |
| 52 | * Make VLIB_MAX_CPUS a power-of-two, please... |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 53 | */ |
Damjan Marion | 3f46baf | 2016-02-06 19:16:21 +0100 | [diff] [blame] | 54 | |
Christophe Fontaine | fef15b4 | 2016-04-09 12:38:49 +0900 | [diff] [blame] | 55 | #ifndef VLIB_MAX_CPUS |
Damjan Marion | 3f46baf | 2016-02-06 19:16:21 +0100 | [diff] [blame] | 56 | #define VLIB_MAX_CPUS 256 |
Christophe Fontaine | fef15b4 | 2016-04-09 12:38:49 +0900 | [diff] [blame] | 57 | #endif |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 58 | |
Damjan Marion | ce8debf | 2016-02-06 19:16:21 +0100 | [diff] [blame] | 59 | #if VLIB_MAX_CPUS > CLIB_MAX_MHEAPS |
| 60 | #error Please increase number of per-cpu mheaps |
| 61 | #endif |
| 62 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 63 | #define VLIB_CPU_MASK (VLIB_MAX_CPUS - 1) /* 0x3f, max */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 64 | #define VLIB_OFFSET_MASK (~VLIB_CPU_MASK) |
| 65 | |
Florin Coras | 93992a9 | 2017-05-24 18:03:56 -0700 | [diff] [blame] | 66 | #define VLIB_LOG2_THREAD_STACK_SIZE (21) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 67 | #define VLIB_THREAD_STACK_SIZE (1<<VLIB_LOG2_THREAD_STACK_SIZE) |
| 68 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 69 | typedef enum |
| 70 | { |
| 71 | VLIB_FRAME_QUEUE_ELT_DISPATCH_FRAME, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 72 | } vlib_frame_queue_msg_type_t; |
| 73 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 74 | typedef struct |
| 75 | { |
Damjan Marion | b0d680b | 2016-11-04 14:41:44 +0100 | [diff] [blame] | 76 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 77 | volatile u32 valid; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 78 | u32 msg_type; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 79 | u32 n_vectors; |
| 80 | u32 last_n_vectors; |
| 81 | |
| 82 | /* 256 * 4 = 1024 bytes, even mult of cache line size */ |
| 83 | u32 buffer_index[VLIB_FRAME_SIZE]; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 84 | } |
| 85 | vlib_frame_queue_elt_t; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 86 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 87 | typedef struct |
| 88 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 89 | /* First cache line */ |
Damjan Marion | b0d680b | 2016-11-04 14:41:44 +0100 | [diff] [blame] | 90 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 91 | volatile u32 *wait_at_barrier; |
| 92 | volatile u32 *workers_at_barrier; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 93 | |
| 94 | /* Second Cache Line */ |
Damjan Marion | b0d680b | 2016-11-04 14:41:44 +0100 | [diff] [blame] | 95 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 96 | void *thread_mheap; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 97 | u8 *thread_stack; |
| 98 | void (*thread_function) (void *); |
| 99 | void *thread_function_arg; |
| 100 | i64 recursion_level; |
| 101 | elog_track_t elog_track; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 102 | u32 instance_id; |
| 103 | vlib_thread_registration_t *registration; |
| 104 | u8 *name; |
Bud Grise | 42f2006 | 2016-03-16 13:09:46 -0400 | [diff] [blame] | 105 | u64 barrier_sync_count; |
Dave Barach | 88c6e00 | 2018-09-30 15:54:06 -0400 | [diff] [blame] | 106 | u8 barrier_elog_enabled; |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 107 | const char *barrier_caller; |
| 108 | const char *barrier_context; |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 109 | volatile u32 *node_reforks_required; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 110 | |
| 111 | long lwp; |
Mohsin Kazmi | 5d64c78 | 2018-09-11 20:27:09 +0200 | [diff] [blame] | 112 | int cpu_id; |
| 113 | int core_id; |
Dave Barach | a690fdb | 2020-01-21 12:34:55 -0500 | [diff] [blame] | 114 | int numa_id; |
Pavel Kotucek | 9876520 | 2016-10-07 08:37:28 +0200 | [diff] [blame] | 115 | pthread_t thread_id; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 116 | } vlib_worker_thread_t; |
| 117 | |
Damjan Marion | 6a7acc2 | 2016-12-19 16:28:36 +0100 | [diff] [blame] | 118 | extern vlib_worker_thread_t *vlib_worker_threads; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 119 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 120 | typedef struct |
| 121 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 122 | /* enqueue side */ |
Damjan Marion | b0d680b | 2016-11-04 14:41:44 +0100 | [diff] [blame] | 123 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 124 | volatile u64 tail; |
| 125 | u64 enqueues; |
| 126 | u64 enqueue_ticks; |
| 127 | u64 enqueue_vectors; |
| 128 | u32 enqueue_full_events; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 129 | |
| 130 | /* dequeue side */ |
Damjan Marion | b0d680b | 2016-11-04 14:41:44 +0100 | [diff] [blame] | 131 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 132 | volatile u64 head; |
| 133 | u64 dequeues; |
| 134 | u64 dequeue_ticks; |
| 135 | u64 dequeue_vectors; |
| 136 | u64 trace; |
| 137 | u64 vector_threshold; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 138 | |
| 139 | /* dequeue hint to enqueue side */ |
Damjan Marion | b0d680b | 2016-11-04 14:41:44 +0100 | [diff] [blame] | 140 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline2); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 141 | volatile u64 head_hint; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 142 | |
| 143 | /* read-only, constant, shared */ |
Damjan Marion | b0d680b | 2016-11-04 14:41:44 +0100 | [diff] [blame] | 144 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline3); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 145 | vlib_frame_queue_elt_t *elts; |
| 146 | u32 nelts; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 147 | } |
| 148 | vlib_frame_queue_t; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 149 | |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 150 | typedef struct |
| 151 | { |
Damjan Marion | 78fd7e8 | 2018-07-20 18:47:05 +0200 | [diff] [blame] | 152 | vlib_frame_queue_elt_t **handoff_queue_elt_by_thread_index; |
| 153 | vlib_frame_queue_t **congested_handoff_queue_by_thread_index; |
| 154 | } vlib_frame_queue_per_thread_data_t; |
| 155 | |
| 156 | typedef struct |
| 157 | { |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 158 | u32 node_index; |
Damjan Marion | 78fd7e8 | 2018-07-20 18:47:05 +0200 | [diff] [blame] | 159 | u32 frame_queue_nelts; |
| 160 | u32 queue_hi_thresh; |
| 161 | |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 162 | vlib_frame_queue_t **vlib_frame_queues; |
Damjan Marion | 78fd7e8 | 2018-07-20 18:47:05 +0200 | [diff] [blame] | 163 | vlib_frame_queue_per_thread_data_t *per_thread_data; |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 164 | |
| 165 | /* for frame queue tracing */ |
| 166 | frame_queue_trace_t *frame_queue_traces; |
| 167 | frame_queue_nelt_counter_t *frame_queue_histogram; |
| 168 | } vlib_frame_queue_main_t; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 169 | |
Dave Barach | 69128d0 | 2017-09-26 10:54:34 -0400 | [diff] [blame] | 170 | typedef struct |
| 171 | { |
| 172 | uword node_index; |
| 173 | uword type_opaque; |
| 174 | uword data; |
| 175 | } vlib_process_signal_event_mt_args_t; |
| 176 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 177 | /* Called early, in thread 0's context */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 178 | clib_error_t *vlib_thread_init (vlib_main_t * vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 179 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 180 | int vlib_frame_queue_enqueue (vlib_main_t * vm, u32 node_runtime_index, |
| 181 | u32 frame_queue_index, vlib_frame_t * frame, |
| 182 | vlib_frame_queue_msg_type_t type); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 183 | |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 184 | int |
| 185 | vlib_frame_queue_dequeue (vlib_main_t * vm, vlib_frame_queue_main_t * fqm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 186 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 187 | void vlib_worker_thread_node_runtime_update (void); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 188 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 189 | void vlib_create_worker_threads (vlib_main_t * vm, int n, |
| 190 | void (*thread_function) (void *)); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 191 | |
| 192 | void vlib_worker_thread_init (vlib_worker_thread_t * w); |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 193 | u32 vlib_frame_queue_main_init (u32 node_index, u32 frame_queue_nelts); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 194 | |
| 195 | /* Check for a barrier sync request every 30ms */ |
| 196 | #define BARRIER_SYNC_DELAY (0.030000) |
| 197 | |
| 198 | #if CLIB_DEBUG > 0 |
| 199 | /* long barrier timeout, for gdb... */ |
| 200 | #define BARRIER_SYNC_TIMEOUT (600.1) |
| 201 | #else |
| 202 | #define BARRIER_SYNC_TIMEOUT (1.0) |
| 203 | #endif |
| 204 | |
Damjan Marion | 8343ee5 | 2019-02-26 17:15:48 +0100 | [diff] [blame] | 205 | #define vlib_worker_thread_barrier_sync(X) {vlib_worker_thread_barrier_sync_int(X, __FUNCTION__);} |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 206 | |
Damjan Marion | 8343ee5 | 2019-02-26 17:15:48 +0100 | [diff] [blame] | 207 | void vlib_worker_thread_barrier_sync_int (vlib_main_t * vm, |
| 208 | const char *func_name); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 209 | void vlib_worker_thread_barrier_release (vlib_main_t * vm); |
Neale Ranns | 42845dd | 2020-05-26 13:12:17 +0000 | [diff] [blame] | 210 | u8 vlib_worker_thread_barrier_held (void); |
Dave Barach | c602b38 | 2019-06-03 19:48:22 -0400 | [diff] [blame] | 211 | void vlib_worker_thread_initial_barrier_sync_and_release (vlib_main_t * vm); |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 212 | void vlib_worker_thread_node_refork (void); |
Neale Ranns | 42845dd | 2020-05-26 13:12:17 +0000 | [diff] [blame] | 213 | /** |
| 214 | * Wait until each of the workers has been once around the track |
| 215 | */ |
| 216 | void vlib_worker_wait_one_loop (void); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 217 | |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 218 | static_always_inline uword |
| 219 | vlib_get_thread_index (void) |
| 220 | { |
Damjan Marion | f55f9b8 | 2017-05-10 21:06:28 +0200 | [diff] [blame] | 221 | return __os_thread_index; |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 222 | } |
| 223 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 224 | always_inline void |
| 225 | vlib_smp_unsafe_warning (void) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 226 | { |
| 227 | if (CLIB_DEBUG > 0) |
| 228 | { |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 229 | if (vlib_get_thread_index ()) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 230 | fformat (stderr, "%s: SMP unsafe warning...\n", __FUNCTION__); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 231 | } |
| 232 | } |
| 233 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 234 | typedef enum |
| 235 | { |
| 236 | VLIB_WORKER_THREAD_FORK_FIXUP_ILLEGAL = 0, |
| 237 | VLIB_WORKER_THREAD_FORK_FIXUP_NEW_SW_IF_INDEX, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 238 | } vlib_fork_fixup_t; |
| 239 | |
| 240 | void vlib_worker_thread_fork_fixup (vlib_fork_fixup_t which); |
| 241 | |
Dave Barach | 80f54e2 | 2017-03-08 19:08:56 -0500 | [diff] [blame] | 242 | #define foreach_vlib_main(body) \ |
| 243 | do { \ |
| 244 | vlib_main_t ** __vlib_mains = 0, *this_vlib_main; \ |
| 245 | int ii; \ |
| 246 | \ |
| 247 | for (ii = 0; ii < vec_len (vlib_mains); ii++) \ |
| 248 | { \ |
| 249 | this_vlib_main = vlib_mains[ii]; \ |
Damjan Marion | ce359db | 2017-03-16 16:15:38 +0100 | [diff] [blame] | 250 | ASSERT (ii == 0 || \ |
| 251 | this_vlib_main->parked_at_barrier == 1); \ |
Dave Barach | 80f54e2 | 2017-03-08 19:08:56 -0500 | [diff] [blame] | 252 | if (this_vlib_main) \ |
| 253 | vec_add1 (__vlib_mains, this_vlib_main); \ |
| 254 | } \ |
| 255 | \ |
| 256 | for (ii = 0; ii < vec_len (__vlib_mains); ii++) \ |
| 257 | { \ |
| 258 | this_vlib_main = __vlib_mains[ii]; \ |
| 259 | /* body uses this_vlib_main... */ \ |
| 260 | (body); \ |
| 261 | } \ |
| 262 | vec_free (__vlib_mains); \ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 263 | } while (0); |
| 264 | |
Pavel Kotucek | 1e76583 | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 265 | #define foreach_sched_policy \ |
| 266 | _(SCHED_OTHER, OTHER, "other") \ |
| 267 | _(SCHED_BATCH, BATCH, "batch") \ |
| 268 | _(SCHED_IDLE, IDLE, "idle") \ |
| 269 | _(SCHED_FIFO, FIFO, "fifo") \ |
| 270 | _(SCHED_RR, RR, "rr") |
| 271 | |
| 272 | typedef enum |
| 273 | { |
| 274 | #define _(v,f,s) SCHED_POLICY_##f = v, |
| 275 | foreach_sched_policy |
| 276 | #undef _ |
| 277 | SCHED_POLICY_N, |
| 278 | } sched_policy_t; |
| 279 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 280 | typedef struct |
| 281 | { |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 282 | clib_error_t *(*vlib_launch_thread_cb) (void *fp, vlib_worker_thread_t * w, |
Mohsin Kazmi | 5d64c78 | 2018-09-11 20:27:09 +0200 | [diff] [blame] | 283 | unsigned cpu_id); |
| 284 | clib_error_t *(*vlib_thread_set_lcore_cb) (u32 thread, u16 cpu); |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 285 | } vlib_thread_callbacks_t; |
| 286 | |
| 287 | typedef struct |
| 288 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 289 | /* Link list of registrations, built by constructors */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 290 | vlib_thread_registration_t *next; |
| 291 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 292 | /* Vector of registrations, w/ non-data-structure clones at the top */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 293 | vlib_thread_registration_t **registrations; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 294 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 295 | uword *thread_registrations_by_name; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 296 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 297 | vlib_worker_thread_t *worker_threads; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 298 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 299 | /* |
| 300 | * Launch all threads as pthreads, |
| 301 | * not eal_rte_launch (strict affinity) threads |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 302 | */ |
| 303 | int use_pthreads; |
| 304 | |
| 305 | /* Number of vlib_main / vnet_main clones */ |
| 306 | u32 n_vlib_mains; |
| 307 | |
| 308 | /* Number of thread stacks to create */ |
| 309 | u32 n_thread_stacks; |
| 310 | |
| 311 | /* Number of pthreads */ |
| 312 | u32 n_pthreads; |
| 313 | |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 314 | /* Number of threads */ |
| 315 | u32 n_threads; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 316 | |
| 317 | /* Number of cores to skip, must match the core mask */ |
| 318 | u32 skip_cores; |
| 319 | |
| 320 | /* Thread prefix name */ |
| 321 | u8 *thread_prefix; |
| 322 | |
| 323 | /* main thread lcore */ |
Damjan Marion | 858151f | 2018-07-11 10:51:00 +0200 | [diff] [blame] | 324 | u32 main_lcore; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 325 | |
| 326 | /* Bitmap of available CPU cores */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 327 | uword *cpu_core_bitmap; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 328 | |
| 329 | /* Bitmap of available CPU sockets (NUMA nodes) */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 330 | uword *cpu_socket_bitmap; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 331 | |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 332 | /* Worker handoff queues */ |
| 333 | vlib_frame_queue_main_t *frame_queue_mains; |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 334 | |
| 335 | /* worker thread initialization barrier */ |
| 336 | volatile u32 worker_thread_release; |
| 337 | |
Pavel Kotucek | 1e76583 | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 338 | /* scheduling policy */ |
| 339 | u32 sched_policy; |
| 340 | |
| 341 | /* scheduling policy priority */ |
| 342 | u32 sched_priority; |
| 343 | |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 344 | /* callbacks */ |
| 345 | vlib_thread_callbacks_t cb; |
| 346 | int extern_thread_mgmt; |
Dave Barach | a690fdb | 2020-01-21 12:34:55 -0500 | [diff] [blame] | 347 | |
| 348 | /* NUMA-bound heap size */ |
| 349 | uword numa_heap_size; |
| 350 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 351 | } vlib_thread_main_t; |
| 352 | |
Damjan Marion | 6a7acc2 | 2016-12-19 16:28:36 +0100 | [diff] [blame] | 353 | extern vlib_thread_main_t vlib_thread_main; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 354 | |
Damjan Marion | ce359db | 2017-03-16 16:15:38 +0100 | [diff] [blame] | 355 | #include <vlib/global_funcs.h> |
| 356 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 357 | #define VLIB_REGISTER_THREAD(x,...) \ |
| 358 | __VA_ARGS__ vlib_thread_registration_t x; \ |
| 359 | static void __vlib_add_thread_registration_##x (void) \ |
| 360 | __attribute__((__constructor__)) ; \ |
| 361 | static void __vlib_add_thread_registration_##x (void) \ |
| 362 | { \ |
| 363 | vlib_thread_main_t * tm = &vlib_thread_main; \ |
| 364 | x.next = tm->next; \ |
| 365 | tm->next = &x; \ |
| 366 | } \ |
Damjan Marion | 72d2c4f | 2018-04-05 21:32:29 +0200 | [diff] [blame] | 367 | static void __vlib_rm_thread_registration_##x (void) \ |
| 368 | __attribute__((__destructor__)) ; \ |
| 369 | static void __vlib_rm_thread_registration_##x (void) \ |
| 370 | { \ |
| 371 | vlib_thread_main_t * tm = &vlib_thread_main; \ |
| 372 | VLIB_REMOVE_FROM_LINKED_LIST (tm->next, &x, next); \ |
| 373 | } \ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 374 | __VA_ARGS__ vlib_thread_registration_t x |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 375 | |
Damjan Marion | 6403436 | 2016-11-07 22:19:55 +0100 | [diff] [blame] | 376 | always_inline u32 |
| 377 | vlib_num_workers () |
| 378 | { |
| 379 | return vlib_thread_main.n_vlib_mains - 1; |
| 380 | } |
| 381 | |
| 382 | always_inline u32 |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 383 | vlib_get_worker_thread_index (u32 worker_index) |
Damjan Marion | 6403436 | 2016-11-07 22:19:55 +0100 | [diff] [blame] | 384 | { |
| 385 | return worker_index + 1; |
| 386 | } |
| 387 | |
Damjan Marion | 3a4ed39 | 2016-11-08 13:20:42 +0100 | [diff] [blame] | 388 | always_inline u32 |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 389 | vlib_get_worker_index (u32 thread_index) |
Damjan Marion | 3a4ed39 | 2016-11-08 13:20:42 +0100 | [diff] [blame] | 390 | { |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 391 | return thread_index - 1; |
Damjan Marion | 3a4ed39 | 2016-11-08 13:20:42 +0100 | [diff] [blame] | 392 | } |
| 393 | |
| 394 | always_inline u32 |
| 395 | vlib_get_current_worker_index () |
| 396 | { |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 397 | return vlib_get_thread_index () - 1; |
Damjan Marion | 3a4ed39 | 2016-11-08 13:20:42 +0100 | [diff] [blame] | 398 | } |
| 399 | |
Damjan Marion | ce359db | 2017-03-16 16:15:38 +0100 | [diff] [blame] | 400 | static inline void |
| 401 | vlib_worker_thread_barrier_check (void) |
| 402 | { |
| 403 | if (PREDICT_FALSE (*vlib_worker_threads->wait_at_barrier)) |
| 404 | { |
Dave Barach | 88c6e00 | 2018-09-30 15:54:06 -0400 | [diff] [blame] | 405 | vlib_main_t *vm = vlib_get_main (); |
| 406 | u32 thread_index = vm->thread_index; |
| 407 | f64 t = vlib_time_now (vm); |
| 408 | |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 409 | if (PREDICT_FALSE (vec_len (vm->barrier_perf_callbacks) != 0)) |
| 410 | clib_call_callbacks (vm->barrier_perf_callbacks, vm, |
| 411 | vm->clib_time.last_cpu_time, 0 /* enter */ ); |
| 412 | |
Dave Barach | 88c6e00 | 2018-09-30 15:54:06 -0400 | [diff] [blame] | 413 | if (PREDICT_FALSE (vlib_worker_threads->barrier_elog_enabled)) |
| 414 | { |
| 415 | vlib_worker_thread_t *w = vlib_worker_threads + thread_index; |
| 416 | /* *INDENT-OFF* */ |
| 417 | ELOG_TYPE_DECLARE (e) = { |
| 418 | .format = "barrier-wait-thread-%d", |
| 419 | .format_args = "i4", |
| 420 | }; |
| 421 | /* *INDENT-ON* */ |
| 422 | |
| 423 | struct |
| 424 | { |
| 425 | u32 thread_index; |
| 426 | } __clib_packed *ed; |
| 427 | |
| 428 | ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e, |
| 429 | w->elog_track); |
| 430 | ed->thread_index = thread_index; |
| 431 | } |
| 432 | |
Damjan Marion | ce359db | 2017-03-16 16:15:38 +0100 | [diff] [blame] | 433 | if (CLIB_DEBUG > 0) |
| 434 | { |
| 435 | vm = vlib_get_main (); |
| 436 | vm->parked_at_barrier = 1; |
| 437 | } |
Alexander Kabaev | feda545 | 2019-11-01 18:26:20 -0400 | [diff] [blame] | 438 | clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, 1); |
Damjan Marion | ce359db | 2017-03-16 16:15:38 +0100 | [diff] [blame] | 439 | while (*vlib_worker_threads->wait_at_barrier) |
| 440 | ; |
Dave Barach | a4324a9 | 2019-02-19 17:05:30 -0500 | [diff] [blame] | 441 | |
| 442 | /* |
| 443 | * Recompute the offset from thread-0 time. |
| 444 | * Note that vlib_time_now adds vm->time_offset, so |
| 445 | * clear it first. Save the resulting idea of "now", to |
| 446 | * see how well we're doing. See show_clock_command_fn(...) |
| 447 | */ |
| 448 | { |
| 449 | f64 now; |
| 450 | vm->time_offset = 0.0; |
| 451 | now = vlib_time_now (vm); |
| 452 | vm->time_offset = vlib_global_main.time_last_barrier_release - now; |
| 453 | vm->time_last_barrier_release = vlib_time_now (vm); |
| 454 | } |
| 455 | |
Damjan Marion | ce359db | 2017-03-16 16:15:38 +0100 | [diff] [blame] | 456 | if (CLIB_DEBUG > 0) |
| 457 | vm->parked_at_barrier = 0; |
Sirshak Das | 2f6d7bb | 2018-10-03 22:53:51 +0000 | [diff] [blame] | 458 | clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, -1); |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 459 | |
| 460 | if (PREDICT_FALSE (*vlib_worker_threads->node_reforks_required)) |
| 461 | { |
Dave Barach | 88c6e00 | 2018-09-30 15:54:06 -0400 | [diff] [blame] | 462 | if (PREDICT_FALSE (vlib_worker_threads->barrier_elog_enabled)) |
| 463 | { |
| 464 | t = vlib_time_now (vm) - t; |
| 465 | vlib_worker_thread_t *w = vlib_worker_threads + thread_index; |
| 466 | /* *INDENT-OFF* */ |
| 467 | ELOG_TYPE_DECLARE (e) = { |
| 468 | .format = "barrier-refork-thread-%d", |
| 469 | .format_args = "i4", |
| 470 | }; |
| 471 | /* *INDENT-ON* */ |
| 472 | |
| 473 | struct |
| 474 | { |
| 475 | u32 thread_index; |
| 476 | } __clib_packed *ed; |
| 477 | |
| 478 | ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e, |
| 479 | w->elog_track); |
| 480 | ed->thread_index = thread_index; |
| 481 | } |
| 482 | |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 483 | vlib_worker_thread_node_refork (); |
Sirshak Das | 2f6d7bb | 2018-10-03 22:53:51 +0000 | [diff] [blame] | 484 | clib_atomic_fetch_add (vlib_worker_threads->node_reforks_required, |
| 485 | -1); |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 486 | while (*vlib_worker_threads->node_reforks_required) |
| 487 | ; |
| 488 | } |
Dave Barach | 88c6e00 | 2018-09-30 15:54:06 -0400 | [diff] [blame] | 489 | if (PREDICT_FALSE (vlib_worker_threads->barrier_elog_enabled)) |
| 490 | { |
| 491 | t = vlib_time_now (vm) - t; |
| 492 | vlib_worker_thread_t *w = vlib_worker_threads + thread_index; |
| 493 | /* *INDENT-OFF* */ |
| 494 | ELOG_TYPE_DECLARE (e) = { |
| 495 | .format = "barrier-released-thread-%d: %dus", |
| 496 | .format_args = "i4i4", |
| 497 | }; |
| 498 | /* *INDENT-ON* */ |
| 499 | |
| 500 | struct |
| 501 | { |
| 502 | u32 thread_index; |
| 503 | u32 duration; |
| 504 | } __clib_packed *ed; |
| 505 | |
| 506 | ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e, |
| 507 | w->elog_track); |
| 508 | ed->thread_index = thread_index; |
| 509 | ed->duration = (int) (1000000.0 * t); |
| 510 | } |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 511 | |
| 512 | if (PREDICT_FALSE (vec_len (vm->barrier_perf_callbacks) != 0)) |
| 513 | clib_call_callbacks (vm->barrier_perf_callbacks, vm, |
| 514 | vm->clib_time.last_cpu_time, 1 /* leave */ ); |
Damjan Marion | ce359db | 2017-03-16 16:15:38 +0100 | [diff] [blame] | 515 | } |
| 516 | } |
| 517 | |
Damjan Marion | 6403436 | 2016-11-07 22:19:55 +0100 | [diff] [blame] | 518 | always_inline vlib_main_t * |
| 519 | vlib_get_worker_vlib_main (u32 worker_index) |
| 520 | { |
| 521 | vlib_main_t *vm; |
| 522 | vlib_thread_main_t *tm = &vlib_thread_main; |
| 523 | ASSERT (worker_index < tm->n_vlib_mains - 1); |
| 524 | vm = vlib_mains[worker_index + 1]; |
| 525 | ASSERT (vm); |
| 526 | return vm; |
| 527 | } |
| 528 | |
Florin Coras | 568ebc7 | 2018-09-18 16:12:50 -0700 | [diff] [blame] | 529 | static inline u8 |
| 530 | vlib_thread_is_main_w_barrier (void) |
| 531 | { |
| 532 | return (!vlib_num_workers () |
| 533 | || ((vlib_get_thread_index () == 0 |
| 534 | && vlib_worker_threads->wait_at_barrier[0]))); |
| 535 | } |
| 536 | |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 537 | static inline void |
| 538 | vlib_put_frame_queue_elt (vlib_frame_queue_elt_t * hf) |
| 539 | { |
| 540 | CLIB_MEMORY_BARRIER (); |
| 541 | hf->valid = 1; |
| 542 | } |
| 543 | |
| 544 | static inline vlib_frame_queue_elt_t * |
| 545 | vlib_get_frame_queue_elt (u32 frame_queue_index, u32 index) |
| 546 | { |
| 547 | vlib_frame_queue_t *fq; |
| 548 | vlib_frame_queue_elt_t *elt; |
| 549 | vlib_thread_main_t *tm = &vlib_thread_main; |
| 550 | vlib_frame_queue_main_t *fqm = |
| 551 | vec_elt_at_index (tm->frame_queue_mains, frame_queue_index); |
| 552 | u64 new_tail; |
| 553 | |
| 554 | fq = fqm->vlib_frame_queues[index]; |
| 555 | ASSERT (fq); |
| 556 | |
Sirshak Das | 2f6d7bb | 2018-10-03 22:53:51 +0000 | [diff] [blame] | 557 | new_tail = clib_atomic_add_fetch (&fq->tail, 1); |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 558 | |
| 559 | /* Wait until a ring slot is available */ |
| 560 | while (new_tail >= fq->head_hint + fq->nelts) |
| 561 | vlib_worker_thread_barrier_check (); |
| 562 | |
| 563 | elt = fq->elts + (new_tail & (fq->nelts - 1)); |
| 564 | |
| 565 | /* this would be very bad... */ |
| 566 | while (elt->valid) |
| 567 | ; |
| 568 | |
| 569 | elt->msg_type = VLIB_FRAME_QUEUE_ELT_DISPATCH_FRAME; |
| 570 | elt->last_n_vectors = elt->n_vectors = 0; |
| 571 | |
| 572 | return elt; |
| 573 | } |
| 574 | |
| 575 | static inline vlib_frame_queue_t * |
| 576 | is_vlib_frame_queue_congested (u32 frame_queue_index, |
| 577 | u32 index, |
| 578 | u32 queue_hi_thresh, |
| 579 | vlib_frame_queue_t ** |
| 580 | handoff_queue_by_worker_index) |
| 581 | { |
| 582 | vlib_frame_queue_t *fq; |
| 583 | vlib_thread_main_t *tm = &vlib_thread_main; |
| 584 | vlib_frame_queue_main_t *fqm = |
| 585 | vec_elt_at_index (tm->frame_queue_mains, frame_queue_index); |
| 586 | |
| 587 | fq = handoff_queue_by_worker_index[index]; |
| 588 | if (fq != (vlib_frame_queue_t *) (~0)) |
| 589 | return fq; |
| 590 | |
| 591 | fq = fqm->vlib_frame_queues[index]; |
| 592 | ASSERT (fq); |
| 593 | |
| 594 | if (PREDICT_FALSE (fq->tail >= (fq->head_hint + queue_hi_thresh))) |
| 595 | { |
| 596 | /* a valid entry in the array will indicate the queue has reached |
| 597 | * the specified threshold and is congested |
| 598 | */ |
| 599 | handoff_queue_by_worker_index[index] = fq; |
| 600 | fq->enqueue_full_events++; |
| 601 | return fq; |
| 602 | } |
| 603 | |
| 604 | return NULL; |
| 605 | } |
| 606 | |
| 607 | static inline vlib_frame_queue_elt_t * |
| 608 | vlib_get_worker_handoff_queue_elt (u32 frame_queue_index, |
| 609 | u32 vlib_worker_index, |
| 610 | vlib_frame_queue_elt_t ** |
| 611 | handoff_queue_elt_by_worker_index) |
| 612 | { |
| 613 | vlib_frame_queue_elt_t *elt; |
| 614 | |
| 615 | if (handoff_queue_elt_by_worker_index[vlib_worker_index]) |
| 616 | return handoff_queue_elt_by_worker_index[vlib_worker_index]; |
| 617 | |
| 618 | elt = vlib_get_frame_queue_elt (frame_queue_index, vlib_worker_index); |
| 619 | |
| 620 | handoff_queue_elt_by_worker_index[vlib_worker_index] = elt; |
| 621 | |
| 622 | return elt; |
| 623 | } |
| 624 | |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 625 | u8 *vlib_thread_stack_init (uword thread_index); |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 626 | int vlib_thread_cb_register (struct vlib_main_t *vm, |
| 627 | vlib_thread_callbacks_t * cb); |
Dave Barach | 69128d0 | 2017-09-26 10:54:34 -0400 | [diff] [blame] | 628 | extern void *rpc_call_main_thread_cb_fn; |
| 629 | |
| 630 | void |
| 631 | vlib_process_signal_event_mt_helper (vlib_process_signal_event_mt_args_t * |
| 632 | args); |
| 633 | void vlib_rpc_call_main_thread (void *function, u8 * args, u32 size); |
Dave Barach | a690fdb | 2020-01-21 12:34:55 -0500 | [diff] [blame] | 634 | void vlib_get_thread_core_numa (vlib_worker_thread_t * w, unsigned cpu_id); |
| 635 | |
| 636 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 637 | #endif /* included_vlib_threads_h */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 638 | |
| 639 | /* |
| 640 | * fd.io coding-style-patch-verification: ON |
| 641 | * |
| 642 | * Local Variables: |
| 643 | * eval: (c-set-style "gnu") |
| 644 | * End: |
| 645 | */ |