Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2015 Cisco and/or its affiliates. |
| 3 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | * you may not use this file except in compliance with the License. |
| 5 | * You may obtain a copy of the License at: |
| 6 | * |
| 7 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | * |
| 9 | * Unless required by applicable law or agreed to in writing, software |
| 10 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | * See the License for the specific language governing permissions and |
| 13 | * limitations under the License. |
| 14 | */ |
| 15 | #ifndef included_vlib_threads_h |
| 16 | #define included_vlib_threads_h |
| 17 | |
| 18 | #include <vlib/main.h> |
Pavel Kotucek | 1e76583 | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 19 | #include <linux/sched.h> |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 20 | |
Damjan Marion | 6a7acc2 | 2016-12-19 16:28:36 +0100 | [diff] [blame] | 21 | extern vlib_main_t **vlib_mains; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 22 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 23 | void vlib_set_thread_name (char *name); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 24 | |
| 25 | /* arg is actually a vlib__thread_t * */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 26 | typedef void (vlib_thread_function_t) (void *arg); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 27 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 28 | typedef struct vlib_thread_registration_ |
| 29 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 30 | /* constructor generated list of thread registrations */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 31 | struct vlib_thread_registration_ *next; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 32 | |
| 33 | /* config parameters */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 34 | char *name; |
| 35 | char *short_name; |
| 36 | vlib_thread_function_t *function; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 37 | uword mheap_size; |
| 38 | int fixed_count; |
| 39 | u32 count; |
| 40 | int no_data_structure_clone; |
Bud Grise | 68adab9 | 2016-02-12 10:36:11 -0500 | [diff] [blame] | 41 | u32 frame_queue_nelts; |
| 42 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 43 | /* All threads of this type run on pthreads */ |
| 44 | int use_pthreads; |
| 45 | u32 first_index; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 46 | uword *coremask; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 47 | } vlib_thread_registration_t; |
| 48 | |
Damjan Marion | 3f46baf | 2016-02-06 19:16:21 +0100 | [diff] [blame] | 49 | /* |
| 50 | * Frames have their cpu / vlib_main_t index in the low-order N bits |
| 51 | * Make VLIB_MAX_CPUS a power-of-two, please... |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 52 | */ |
Damjan Marion | 3f46baf | 2016-02-06 19:16:21 +0100 | [diff] [blame] | 53 | |
Christophe Fontaine | fef15b4 | 2016-04-09 12:38:49 +0900 | [diff] [blame] | 54 | #ifndef VLIB_MAX_CPUS |
Damjan Marion | 3f46baf | 2016-02-06 19:16:21 +0100 | [diff] [blame] | 55 | #define VLIB_MAX_CPUS 256 |
Christophe Fontaine | fef15b4 | 2016-04-09 12:38:49 +0900 | [diff] [blame] | 56 | #endif |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 57 | |
Damjan Marion | ce8debf | 2016-02-06 19:16:21 +0100 | [diff] [blame] | 58 | #if VLIB_MAX_CPUS > CLIB_MAX_MHEAPS |
| 59 | #error Please increase number of per-cpu mheaps |
| 60 | #endif |
| 61 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 62 | #define VLIB_CPU_MASK (VLIB_MAX_CPUS - 1) /* 0x3f, max */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 63 | #define VLIB_OFFSET_MASK (~VLIB_CPU_MASK) |
| 64 | |
| 65 | #define VLIB_LOG2_THREAD_STACK_SIZE (20) |
| 66 | #define VLIB_THREAD_STACK_SIZE (1<<VLIB_LOG2_THREAD_STACK_SIZE) |
| 67 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 68 | typedef enum |
| 69 | { |
| 70 | VLIB_FRAME_QUEUE_ELT_DISPATCH_FRAME, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 71 | } vlib_frame_queue_msg_type_t; |
| 72 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 73 | typedef struct |
| 74 | { |
Damjan Marion | b0d680b | 2016-11-04 14:41:44 +0100 | [diff] [blame] | 75 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 76 | volatile u32 valid; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 77 | u32 msg_type; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 78 | u32 n_vectors; |
| 79 | u32 last_n_vectors; |
| 80 | |
| 81 | /* 256 * 4 = 1024 bytes, even mult of cache line size */ |
| 82 | u32 buffer_index[VLIB_FRAME_SIZE]; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 83 | } |
| 84 | vlib_frame_queue_elt_t; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 85 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 86 | typedef struct |
| 87 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 88 | /* First cache line */ |
Damjan Marion | b0d680b | 2016-11-04 14:41:44 +0100 | [diff] [blame] | 89 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 90 | volatile u32 *wait_at_barrier; |
| 91 | volatile u32 *workers_at_barrier; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 92 | |
| 93 | /* Second Cache Line */ |
Damjan Marion | b0d680b | 2016-11-04 14:41:44 +0100 | [diff] [blame] | 94 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 95 | void *thread_mheap; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 96 | u8 *thread_stack; |
| 97 | void (*thread_function) (void *); |
| 98 | void *thread_function_arg; |
| 99 | i64 recursion_level; |
| 100 | elog_track_t elog_track; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 101 | u32 instance_id; |
| 102 | vlib_thread_registration_t *registration; |
| 103 | u8 *name; |
Bud Grise | 42f2006 | 2016-03-16 13:09:46 -0400 | [diff] [blame] | 104 | u64 barrier_sync_count; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 105 | |
| 106 | long lwp; |
Pavel Kotucek | 9876520 | 2016-10-07 08:37:28 +0200 | [diff] [blame] | 107 | int lcore_id; |
| 108 | pthread_t thread_id; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 109 | } vlib_worker_thread_t; |
| 110 | |
Damjan Marion | 6a7acc2 | 2016-12-19 16:28:36 +0100 | [diff] [blame] | 111 | extern vlib_worker_thread_t *vlib_worker_threads; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 112 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 113 | typedef struct |
| 114 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 115 | /* enqueue side */ |
Damjan Marion | b0d680b | 2016-11-04 14:41:44 +0100 | [diff] [blame] | 116 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 117 | volatile u64 tail; |
| 118 | u64 enqueues; |
| 119 | u64 enqueue_ticks; |
| 120 | u64 enqueue_vectors; |
| 121 | u32 enqueue_full_events; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 122 | |
| 123 | /* dequeue side */ |
Damjan Marion | b0d680b | 2016-11-04 14:41:44 +0100 | [diff] [blame] | 124 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 125 | volatile u64 head; |
| 126 | u64 dequeues; |
| 127 | u64 dequeue_ticks; |
| 128 | u64 dequeue_vectors; |
| 129 | u64 trace; |
| 130 | u64 vector_threshold; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 131 | |
| 132 | /* dequeue hint to enqueue side */ |
Damjan Marion | b0d680b | 2016-11-04 14:41:44 +0100 | [diff] [blame] | 133 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline2); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 134 | volatile u64 head_hint; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 135 | |
| 136 | /* read-only, constant, shared */ |
Damjan Marion | b0d680b | 2016-11-04 14:41:44 +0100 | [diff] [blame] | 137 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline3); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 138 | vlib_frame_queue_elt_t *elts; |
| 139 | u32 nelts; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 140 | } |
| 141 | vlib_frame_queue_t; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 142 | |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 143 | typedef struct |
| 144 | { |
| 145 | u32 node_index; |
| 146 | vlib_frame_queue_t **vlib_frame_queues; |
| 147 | |
| 148 | /* for frame queue tracing */ |
| 149 | frame_queue_trace_t *frame_queue_traces; |
| 150 | frame_queue_nelt_counter_t *frame_queue_histogram; |
| 151 | } vlib_frame_queue_main_t; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 152 | |
| 153 | /* Called early, in thread 0's context */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 154 | clib_error_t *vlib_thread_init (vlib_main_t * vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 155 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 156 | vlib_worker_thread_t *vlib_alloc_thread (vlib_main_t * vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 157 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 158 | int vlib_frame_queue_enqueue (vlib_main_t * vm, u32 node_runtime_index, |
| 159 | u32 frame_queue_index, vlib_frame_t * frame, |
| 160 | vlib_frame_queue_msg_type_t type); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 161 | |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 162 | int |
| 163 | vlib_frame_queue_dequeue (vlib_main_t * vm, vlib_frame_queue_main_t * fqm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 164 | |
| 165 | u64 dispatch_node (vlib_main_t * vm, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 166 | vlib_node_runtime_t * node, |
| 167 | vlib_node_type_t type, |
| 168 | vlib_node_state_t dispatch_state, |
| 169 | vlib_frame_t * frame, u64 last_time_stamp); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 170 | |
| 171 | u64 dispatch_pending_node (vlib_main_t * vm, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 172 | vlib_pending_frame_t * p, u64 last_time_stamp); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 173 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 174 | void vlib_worker_thread_node_runtime_update (void); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 175 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 176 | void vlib_create_worker_threads (vlib_main_t * vm, int n, |
| 177 | void (*thread_function) (void *)); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 178 | |
| 179 | void vlib_worker_thread_init (vlib_worker_thread_t * w); |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 180 | u32 vlib_frame_queue_main_init (u32 node_index, u32 frame_queue_nelts); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 181 | |
| 182 | /* Check for a barrier sync request every 30ms */ |
| 183 | #define BARRIER_SYNC_DELAY (0.030000) |
| 184 | |
| 185 | #if CLIB_DEBUG > 0 |
| 186 | /* long barrier timeout, for gdb... */ |
| 187 | #define BARRIER_SYNC_TIMEOUT (600.1) |
| 188 | #else |
| 189 | #define BARRIER_SYNC_TIMEOUT (1.0) |
| 190 | #endif |
| 191 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 192 | void vlib_worker_thread_barrier_sync (vlib_main_t * vm); |
| 193 | void vlib_worker_thread_barrier_release (vlib_main_t * vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 194 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 195 | always_inline void |
| 196 | vlib_smp_unsafe_warning (void) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 197 | { |
| 198 | if (CLIB_DEBUG > 0) |
| 199 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 200 | if (os_get_cpu_number ()) |
| 201 | fformat (stderr, "%s: SMP unsafe warning...\n", __FUNCTION__); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 202 | } |
| 203 | } |
| 204 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 205 | typedef enum |
| 206 | { |
| 207 | VLIB_WORKER_THREAD_FORK_FIXUP_ILLEGAL = 0, |
| 208 | VLIB_WORKER_THREAD_FORK_FIXUP_NEW_SW_IF_INDEX, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 209 | } vlib_fork_fixup_t; |
| 210 | |
| 211 | void vlib_worker_thread_fork_fixup (vlib_fork_fixup_t which); |
| 212 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 213 | static inline void |
| 214 | vlib_worker_thread_barrier_check (void) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 215 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 216 | if (PREDICT_FALSE (*vlib_worker_threads->wait_at_barrier)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 217 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 218 | clib_smp_atomic_add (vlib_worker_threads->workers_at_barrier, 1); |
| 219 | while (*vlib_worker_threads->wait_at_barrier) |
| 220 | ; |
| 221 | clib_smp_atomic_add (vlib_worker_threads->workers_at_barrier, -1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 222 | } |
| 223 | } |
| 224 | |
Dave Barach | 80f54e2 | 2017-03-08 19:08:56 -0500 | [diff] [blame] | 225 | #define foreach_vlib_main(body) \ |
| 226 | do { \ |
| 227 | vlib_main_t ** __vlib_mains = 0, *this_vlib_main; \ |
| 228 | int ii; \ |
| 229 | \ |
| 230 | for (ii = 0; ii < vec_len (vlib_mains); ii++) \ |
| 231 | { \ |
| 232 | this_vlib_main = vlib_mains[ii]; \ |
| 233 | if (this_vlib_main) \ |
| 234 | vec_add1 (__vlib_mains, this_vlib_main); \ |
| 235 | } \ |
| 236 | \ |
| 237 | for (ii = 0; ii < vec_len (__vlib_mains); ii++) \ |
| 238 | { \ |
| 239 | this_vlib_main = __vlib_mains[ii]; \ |
| 240 | /* body uses this_vlib_main... */ \ |
| 241 | (body); \ |
| 242 | } \ |
| 243 | vec_free (__vlib_mains); \ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 244 | } while (0); |
| 245 | |
Pavel Kotucek | 1e76583 | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 246 | #define foreach_sched_policy \ |
| 247 | _(SCHED_OTHER, OTHER, "other") \ |
| 248 | _(SCHED_BATCH, BATCH, "batch") \ |
| 249 | _(SCHED_IDLE, IDLE, "idle") \ |
| 250 | _(SCHED_FIFO, FIFO, "fifo") \ |
| 251 | _(SCHED_RR, RR, "rr") |
| 252 | |
| 253 | typedef enum |
| 254 | { |
| 255 | #define _(v,f,s) SCHED_POLICY_##f = v, |
| 256 | foreach_sched_policy |
| 257 | #undef _ |
| 258 | SCHED_POLICY_N, |
| 259 | } sched_policy_t; |
| 260 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 261 | typedef struct |
| 262 | { |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 263 | clib_error_t *(*vlib_launch_thread_cb) (void *fp, vlib_worker_thread_t * w, |
| 264 | unsigned lcore_id); |
| 265 | clib_error_t *(*vlib_thread_set_lcore_cb) (u32 thread, u16 lcore); |
| 266 | } vlib_thread_callbacks_t; |
| 267 | |
| 268 | typedef struct |
| 269 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 270 | /* Link list of registrations, built by constructors */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 271 | vlib_thread_registration_t *next; |
| 272 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 273 | /* Vector of registrations, w/ non-data-structure clones at the top */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 274 | vlib_thread_registration_t **registrations; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 275 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 276 | uword *thread_registrations_by_name; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 277 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 278 | vlib_worker_thread_t *worker_threads; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 279 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 280 | /* |
| 281 | * Launch all threads as pthreads, |
| 282 | * not eal_rte_launch (strict affinity) threads |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 283 | */ |
| 284 | int use_pthreads; |
| 285 | |
| 286 | /* Number of vlib_main / vnet_main clones */ |
| 287 | u32 n_vlib_mains; |
| 288 | |
| 289 | /* Number of thread stacks to create */ |
| 290 | u32 n_thread_stacks; |
| 291 | |
| 292 | /* Number of pthreads */ |
| 293 | u32 n_pthreads; |
| 294 | |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 295 | /* Number of threads */ |
| 296 | u32 n_threads; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 297 | |
| 298 | /* Number of cores to skip, must match the core mask */ |
| 299 | u32 skip_cores; |
| 300 | |
| 301 | /* Thread prefix name */ |
| 302 | u8 *thread_prefix; |
| 303 | |
| 304 | /* main thread lcore */ |
| 305 | u8 main_lcore; |
| 306 | |
| 307 | /* Bitmap of available CPU cores */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 308 | uword *cpu_core_bitmap; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 309 | |
| 310 | /* Bitmap of available CPU sockets (NUMA nodes) */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 311 | uword *cpu_socket_bitmap; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 312 | |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 313 | /* Worker handoff queues */ |
| 314 | vlib_frame_queue_main_t *frame_queue_mains; |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 315 | |
| 316 | /* worker thread initialization barrier */ |
| 317 | volatile u32 worker_thread_release; |
| 318 | |
Pavel Kotucek | 1e76583 | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 319 | /* scheduling policy */ |
| 320 | u32 sched_policy; |
| 321 | |
| 322 | /* scheduling policy priority */ |
| 323 | u32 sched_priority; |
| 324 | |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 325 | /* callbacks */ |
| 326 | vlib_thread_callbacks_t cb; |
| 327 | int extern_thread_mgmt; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 328 | } vlib_thread_main_t; |
| 329 | |
Damjan Marion | 6a7acc2 | 2016-12-19 16:28:36 +0100 | [diff] [blame] | 330 | extern vlib_thread_main_t vlib_thread_main; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 331 | |
| 332 | #define VLIB_REGISTER_THREAD(x,...) \ |
| 333 | __VA_ARGS__ vlib_thread_registration_t x; \ |
| 334 | static void __vlib_add_thread_registration_##x (void) \ |
| 335 | __attribute__((__constructor__)) ; \ |
| 336 | static void __vlib_add_thread_registration_##x (void) \ |
| 337 | { \ |
| 338 | vlib_thread_main_t * tm = &vlib_thread_main; \ |
| 339 | x.next = tm->next; \ |
| 340 | tm->next = &x; \ |
| 341 | } \ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 342 | __VA_ARGS__ vlib_thread_registration_t x |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 343 | |
Damjan Marion | 6403436 | 2016-11-07 22:19:55 +0100 | [diff] [blame] | 344 | always_inline u32 |
| 345 | vlib_num_workers () |
| 346 | { |
| 347 | return vlib_thread_main.n_vlib_mains - 1; |
| 348 | } |
| 349 | |
| 350 | always_inline u32 |
| 351 | vlib_get_worker_cpu_index (u32 worker_index) |
| 352 | { |
| 353 | return worker_index + 1; |
| 354 | } |
| 355 | |
Damjan Marion | 3a4ed39 | 2016-11-08 13:20:42 +0100 | [diff] [blame] | 356 | always_inline u32 |
| 357 | vlib_get_worker_index (u32 cpu_index) |
| 358 | { |
| 359 | return cpu_index - 1; |
| 360 | } |
| 361 | |
| 362 | always_inline u32 |
| 363 | vlib_get_current_worker_index () |
| 364 | { |
| 365 | return os_get_cpu_number () - 1; |
| 366 | } |
| 367 | |
Damjan Marion | 6403436 | 2016-11-07 22:19:55 +0100 | [diff] [blame] | 368 | always_inline vlib_main_t * |
| 369 | vlib_get_worker_vlib_main (u32 worker_index) |
| 370 | { |
| 371 | vlib_main_t *vm; |
| 372 | vlib_thread_main_t *tm = &vlib_thread_main; |
| 373 | ASSERT (worker_index < tm->n_vlib_mains - 1); |
| 374 | vm = vlib_mains[worker_index + 1]; |
| 375 | ASSERT (vm); |
| 376 | return vm; |
| 377 | } |
| 378 | |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 379 | static inline void |
| 380 | vlib_put_frame_queue_elt (vlib_frame_queue_elt_t * hf) |
| 381 | { |
| 382 | CLIB_MEMORY_BARRIER (); |
| 383 | hf->valid = 1; |
| 384 | } |
| 385 | |
| 386 | static inline vlib_frame_queue_elt_t * |
| 387 | vlib_get_frame_queue_elt (u32 frame_queue_index, u32 index) |
| 388 | { |
| 389 | vlib_frame_queue_t *fq; |
| 390 | vlib_frame_queue_elt_t *elt; |
| 391 | vlib_thread_main_t *tm = &vlib_thread_main; |
| 392 | vlib_frame_queue_main_t *fqm = |
| 393 | vec_elt_at_index (tm->frame_queue_mains, frame_queue_index); |
| 394 | u64 new_tail; |
| 395 | |
| 396 | fq = fqm->vlib_frame_queues[index]; |
| 397 | ASSERT (fq); |
| 398 | |
| 399 | new_tail = __sync_add_and_fetch (&fq->tail, 1); |
| 400 | |
| 401 | /* Wait until a ring slot is available */ |
| 402 | while (new_tail >= fq->head_hint + fq->nelts) |
| 403 | vlib_worker_thread_barrier_check (); |
| 404 | |
| 405 | elt = fq->elts + (new_tail & (fq->nelts - 1)); |
| 406 | |
| 407 | /* this would be very bad... */ |
| 408 | while (elt->valid) |
| 409 | ; |
| 410 | |
| 411 | elt->msg_type = VLIB_FRAME_QUEUE_ELT_DISPATCH_FRAME; |
| 412 | elt->last_n_vectors = elt->n_vectors = 0; |
| 413 | |
| 414 | return elt; |
| 415 | } |
| 416 | |
| 417 | static inline vlib_frame_queue_t * |
| 418 | is_vlib_frame_queue_congested (u32 frame_queue_index, |
| 419 | u32 index, |
| 420 | u32 queue_hi_thresh, |
| 421 | vlib_frame_queue_t ** |
| 422 | handoff_queue_by_worker_index) |
| 423 | { |
| 424 | vlib_frame_queue_t *fq; |
| 425 | vlib_thread_main_t *tm = &vlib_thread_main; |
| 426 | vlib_frame_queue_main_t *fqm = |
| 427 | vec_elt_at_index (tm->frame_queue_mains, frame_queue_index); |
| 428 | |
| 429 | fq = handoff_queue_by_worker_index[index]; |
| 430 | if (fq != (vlib_frame_queue_t *) (~0)) |
| 431 | return fq; |
| 432 | |
| 433 | fq = fqm->vlib_frame_queues[index]; |
| 434 | ASSERT (fq); |
| 435 | |
| 436 | if (PREDICT_FALSE (fq->tail >= (fq->head_hint + queue_hi_thresh))) |
| 437 | { |
| 438 | /* a valid entry in the array will indicate the queue has reached |
| 439 | * the specified threshold and is congested |
| 440 | */ |
| 441 | handoff_queue_by_worker_index[index] = fq; |
| 442 | fq->enqueue_full_events++; |
| 443 | return fq; |
| 444 | } |
| 445 | |
| 446 | return NULL; |
| 447 | } |
| 448 | |
| 449 | static inline vlib_frame_queue_elt_t * |
| 450 | vlib_get_worker_handoff_queue_elt (u32 frame_queue_index, |
| 451 | u32 vlib_worker_index, |
| 452 | vlib_frame_queue_elt_t ** |
| 453 | handoff_queue_elt_by_worker_index) |
| 454 | { |
| 455 | vlib_frame_queue_elt_t *elt; |
| 456 | |
| 457 | if (handoff_queue_elt_by_worker_index[vlib_worker_index]) |
| 458 | return handoff_queue_elt_by_worker_index[vlib_worker_index]; |
| 459 | |
| 460 | elt = vlib_get_frame_queue_elt (frame_queue_index, vlib_worker_index); |
| 461 | |
| 462 | handoff_queue_elt_by_worker_index[vlib_worker_index] = elt; |
| 463 | |
| 464 | return elt; |
| 465 | } |
| 466 | |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 467 | int vlib_thread_cb_register (struct vlib_main_t *vm, |
| 468 | vlib_thread_callbacks_t * cb); |
| 469 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 470 | #endif /* included_vlib_threads_h */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 471 | |
| 472 | /* |
| 473 | * fd.io coding-style-patch-verification: ON |
| 474 | * |
| 475 | * Local Variables: |
| 476 | * eval: (c-set-style "gnu") |
| 477 | * End: |
| 478 | */ |