Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2015 Cisco and/or its affiliates. |
| 3 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | * you may not use this file except in compliance with the License. |
| 5 | * You may obtain a copy of the License at: |
| 6 | * |
| 7 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | * |
| 9 | * Unless required by applicable law or agreed to in writing, software |
| 10 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | * See the License for the specific language governing permissions and |
| 13 | * limitations under the License. |
| 14 | */ |
| 15 | #ifndef included_vlib_threads_h |
| 16 | #define included_vlib_threads_h |
| 17 | |
| 18 | #include <vlib/main.h> |
Pavel Kotucek | 1e76583 | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 19 | #include <linux/sched.h> |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 20 | |
Damjan Marion | 6a7acc2 | 2016-12-19 16:28:36 +0100 | [diff] [blame] | 21 | extern vlib_main_t **vlib_mains; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 22 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 23 | void vlib_set_thread_name (char *name); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 24 | |
| 25 | /* arg is actually a vlib__thread_t * */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 26 | typedef void (vlib_thread_function_t) (void *arg); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 27 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 28 | typedef struct vlib_thread_registration_ |
| 29 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 30 | /* constructor generated list of thread registrations */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 31 | struct vlib_thread_registration_ *next; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 32 | |
| 33 | /* config parameters */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 34 | char *name; |
| 35 | char *short_name; |
| 36 | vlib_thread_function_t *function; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 37 | uword mheap_size; |
| 38 | int fixed_count; |
| 39 | u32 count; |
| 40 | int no_data_structure_clone; |
Bud Grise | 68adab9 | 2016-02-12 10:36:11 -0500 | [diff] [blame] | 41 | u32 frame_queue_nelts; |
| 42 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 43 | /* All threads of this type run on pthreads */ |
| 44 | int use_pthreads; |
| 45 | u32 first_index; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 46 | uword *coremask; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 47 | } vlib_thread_registration_t; |
| 48 | |
Damjan Marion | 3f46baf | 2016-02-06 19:16:21 +0100 | [diff] [blame] | 49 | /* |
| 50 | * Frames have their cpu / vlib_main_t index in the low-order N bits |
| 51 | * Make VLIB_MAX_CPUS a power-of-two, please... |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 52 | */ |
Damjan Marion | 3f46baf | 2016-02-06 19:16:21 +0100 | [diff] [blame] | 53 | |
Christophe Fontaine | fef15b4 | 2016-04-09 12:38:49 +0900 | [diff] [blame] | 54 | #ifndef VLIB_MAX_CPUS |
Damjan Marion | 3f46baf | 2016-02-06 19:16:21 +0100 | [diff] [blame] | 55 | #define VLIB_MAX_CPUS 256 |
Christophe Fontaine | fef15b4 | 2016-04-09 12:38:49 +0900 | [diff] [blame] | 56 | #endif |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 57 | |
Damjan Marion | ce8debf | 2016-02-06 19:16:21 +0100 | [diff] [blame] | 58 | #if VLIB_MAX_CPUS > CLIB_MAX_MHEAPS |
| 59 | #error Please increase number of per-cpu mheaps |
| 60 | #endif |
| 61 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 62 | #define VLIB_CPU_MASK (VLIB_MAX_CPUS - 1) /* 0x3f, max */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 63 | #define VLIB_OFFSET_MASK (~VLIB_CPU_MASK) |
| 64 | |
| 65 | #define VLIB_LOG2_THREAD_STACK_SIZE (20) |
| 66 | #define VLIB_THREAD_STACK_SIZE (1<<VLIB_LOG2_THREAD_STACK_SIZE) |
| 67 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 68 | typedef enum |
| 69 | { |
| 70 | VLIB_FRAME_QUEUE_ELT_DISPATCH_FRAME, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 71 | } vlib_frame_queue_msg_type_t; |
| 72 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 73 | typedef struct |
| 74 | { |
Damjan Marion | b0d680b | 2016-11-04 14:41:44 +0100 | [diff] [blame] | 75 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 76 | volatile u32 valid; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 77 | u32 msg_type; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 78 | u32 n_vectors; |
| 79 | u32 last_n_vectors; |
| 80 | |
| 81 | /* 256 * 4 = 1024 bytes, even mult of cache line size */ |
| 82 | u32 buffer_index[VLIB_FRAME_SIZE]; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 83 | } |
| 84 | vlib_frame_queue_elt_t; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 85 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 86 | typedef struct |
| 87 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 88 | /* First cache line */ |
Damjan Marion | b0d680b | 2016-11-04 14:41:44 +0100 | [diff] [blame] | 89 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 90 | volatile u32 *wait_at_barrier; |
| 91 | volatile u32 *workers_at_barrier; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 92 | |
| 93 | /* Second Cache Line */ |
Damjan Marion | b0d680b | 2016-11-04 14:41:44 +0100 | [diff] [blame] | 94 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 95 | void *thread_mheap; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 96 | u8 *thread_stack; |
| 97 | void (*thread_function) (void *); |
| 98 | void *thread_function_arg; |
| 99 | i64 recursion_level; |
| 100 | elog_track_t elog_track; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 101 | u32 instance_id; |
| 102 | vlib_thread_registration_t *registration; |
| 103 | u8 *name; |
Bud Grise | 42f2006 | 2016-03-16 13:09:46 -0400 | [diff] [blame] | 104 | u64 barrier_sync_count; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 105 | |
| 106 | long lwp; |
Pavel Kotucek | 9876520 | 2016-10-07 08:37:28 +0200 | [diff] [blame] | 107 | int lcore_id; |
| 108 | pthread_t thread_id; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 109 | } vlib_worker_thread_t; |
| 110 | |
Damjan Marion | 6a7acc2 | 2016-12-19 16:28:36 +0100 | [diff] [blame] | 111 | extern vlib_worker_thread_t *vlib_worker_threads; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 112 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 113 | typedef struct |
| 114 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 115 | /* enqueue side */ |
Damjan Marion | b0d680b | 2016-11-04 14:41:44 +0100 | [diff] [blame] | 116 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 117 | volatile u64 tail; |
| 118 | u64 enqueues; |
| 119 | u64 enqueue_ticks; |
| 120 | u64 enqueue_vectors; |
| 121 | u32 enqueue_full_events; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 122 | |
| 123 | /* dequeue side */ |
Damjan Marion | b0d680b | 2016-11-04 14:41:44 +0100 | [diff] [blame] | 124 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 125 | volatile u64 head; |
| 126 | u64 dequeues; |
| 127 | u64 dequeue_ticks; |
| 128 | u64 dequeue_vectors; |
| 129 | u64 trace; |
| 130 | u64 vector_threshold; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 131 | |
| 132 | /* dequeue hint to enqueue side */ |
Damjan Marion | b0d680b | 2016-11-04 14:41:44 +0100 | [diff] [blame] | 133 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline2); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 134 | volatile u64 head_hint; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 135 | |
| 136 | /* read-only, constant, shared */ |
Damjan Marion | b0d680b | 2016-11-04 14:41:44 +0100 | [diff] [blame] | 137 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline3); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 138 | vlib_frame_queue_elt_t *elts; |
| 139 | u32 nelts; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 140 | } |
| 141 | vlib_frame_queue_t; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 142 | |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 143 | typedef struct |
| 144 | { |
| 145 | u32 node_index; |
| 146 | vlib_frame_queue_t **vlib_frame_queues; |
| 147 | |
| 148 | /* for frame queue tracing */ |
| 149 | frame_queue_trace_t *frame_queue_traces; |
| 150 | frame_queue_nelt_counter_t *frame_queue_histogram; |
| 151 | } vlib_frame_queue_main_t; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 152 | |
| 153 | /* Called early, in thread 0's context */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 154 | clib_error_t *vlib_thread_init (vlib_main_t * vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 155 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 156 | vlib_worker_thread_t *vlib_alloc_thread (vlib_main_t * vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 157 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 158 | int vlib_frame_queue_enqueue (vlib_main_t * vm, u32 node_runtime_index, |
| 159 | u32 frame_queue_index, vlib_frame_t * frame, |
| 160 | vlib_frame_queue_msg_type_t type); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 161 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 162 | int vlib_frame_queue_dequeue (int thread_id, |
| 163 | vlib_main_t * vm, vlib_node_main_t * nm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 164 | |
| 165 | u64 dispatch_node (vlib_main_t * vm, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 166 | vlib_node_runtime_t * node, |
| 167 | vlib_node_type_t type, |
| 168 | vlib_node_state_t dispatch_state, |
| 169 | vlib_frame_t * frame, u64 last_time_stamp); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 170 | |
| 171 | u64 dispatch_pending_node (vlib_main_t * vm, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 172 | vlib_pending_frame_t * p, u64 last_time_stamp); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 173 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 174 | void vlib_worker_thread_node_runtime_update (void); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 175 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 176 | void vlib_create_worker_threads (vlib_main_t * vm, int n, |
| 177 | void (*thread_function) (void *)); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 178 | |
| 179 | void vlib_worker_thread_init (vlib_worker_thread_t * w); |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 180 | u32 vlib_frame_queue_main_init (u32 node_index, u32 frame_queue_nelts); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 181 | |
| 182 | /* Check for a barrier sync request every 30ms */ |
| 183 | #define BARRIER_SYNC_DELAY (0.030000) |
| 184 | |
| 185 | #if CLIB_DEBUG > 0 |
| 186 | /* long barrier timeout, for gdb... */ |
| 187 | #define BARRIER_SYNC_TIMEOUT (600.1) |
| 188 | #else |
| 189 | #define BARRIER_SYNC_TIMEOUT (1.0) |
| 190 | #endif |
| 191 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 192 | void vlib_worker_thread_barrier_sync (vlib_main_t * vm); |
| 193 | void vlib_worker_thread_barrier_release (vlib_main_t * vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 194 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 195 | always_inline void |
| 196 | vlib_smp_unsafe_warning (void) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 197 | { |
| 198 | if (CLIB_DEBUG > 0) |
| 199 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 200 | if (os_get_cpu_number ()) |
| 201 | fformat (stderr, "%s: SMP unsafe warning...\n", __FUNCTION__); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 202 | } |
| 203 | } |
| 204 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 205 | typedef enum |
| 206 | { |
| 207 | VLIB_WORKER_THREAD_FORK_FIXUP_ILLEGAL = 0, |
| 208 | VLIB_WORKER_THREAD_FORK_FIXUP_NEW_SW_IF_INDEX, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 209 | } vlib_fork_fixup_t; |
| 210 | |
| 211 | void vlib_worker_thread_fork_fixup (vlib_fork_fixup_t which); |
| 212 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 213 | static inline void |
| 214 | vlib_worker_thread_barrier_check (void) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 215 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 216 | if (PREDICT_FALSE (*vlib_worker_threads->wait_at_barrier)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 217 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 218 | clib_smp_atomic_add (vlib_worker_threads->workers_at_barrier, 1); |
| 219 | while (*vlib_worker_threads->wait_at_barrier) |
| 220 | ; |
| 221 | clib_smp_atomic_add (vlib_worker_threads->workers_at_barrier, -1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 222 | } |
| 223 | } |
| 224 | |
| 225 | #define foreach_vlib_main(body) \ |
| 226 | do { \ |
| 227 | vlib_main_t ** __vlib_mains = 0, *this_vlib_main; \ |
| 228 | int ii; \ |
| 229 | \ |
| 230 | if (vec_len (vlib_mains) == 0) \ |
| 231 | vec_add1 (__vlib_mains, &vlib_global_main); \ |
| 232 | else \ |
| 233 | { \ |
| 234 | for (ii = 0; ii < vec_len (vlib_mains); ii++) \ |
| 235 | { \ |
| 236 | this_vlib_main = vlib_mains[ii]; \ |
| 237 | if (this_vlib_main) \ |
| 238 | vec_add1 (__vlib_mains, this_vlib_main); \ |
| 239 | } \ |
| 240 | } \ |
| 241 | \ |
| 242 | for (ii = 0; ii < vec_len (__vlib_mains); ii++) \ |
| 243 | { \ |
| 244 | this_vlib_main = __vlib_mains[ii]; \ |
| 245 | /* body uses this_vlib_main... */ \ |
| 246 | (body); \ |
| 247 | } \ |
| 248 | vec_free (__vlib_mains); \ |
| 249 | } while (0); |
| 250 | |
Pavel Kotucek | 1e76583 | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 251 | #define foreach_sched_policy \ |
| 252 | _(SCHED_OTHER, OTHER, "other") \ |
| 253 | _(SCHED_BATCH, BATCH, "batch") \ |
| 254 | _(SCHED_IDLE, IDLE, "idle") \ |
| 255 | _(SCHED_FIFO, FIFO, "fifo") \ |
| 256 | _(SCHED_RR, RR, "rr") |
| 257 | |
| 258 | typedef enum |
| 259 | { |
| 260 | #define _(v,f,s) SCHED_POLICY_##f = v, |
| 261 | foreach_sched_policy |
| 262 | #undef _ |
| 263 | SCHED_POLICY_N, |
| 264 | } sched_policy_t; |
| 265 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 266 | typedef struct |
| 267 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 268 | /* Link list of registrations, built by constructors */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 269 | vlib_thread_registration_t *next; |
| 270 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 271 | /* Vector of registrations, w/ non-data-structure clones at the top */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 272 | vlib_thread_registration_t **registrations; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 273 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 274 | uword *thread_registrations_by_name; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 275 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 276 | vlib_worker_thread_t *worker_threads; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 277 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 278 | /* |
| 279 | * Launch all threads as pthreads, |
| 280 | * not eal_rte_launch (strict affinity) threads |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 281 | */ |
| 282 | int use_pthreads; |
| 283 | |
| 284 | /* Number of vlib_main / vnet_main clones */ |
| 285 | u32 n_vlib_mains; |
| 286 | |
| 287 | /* Number of thread stacks to create */ |
| 288 | u32 n_thread_stacks; |
| 289 | |
| 290 | /* Number of pthreads */ |
| 291 | u32 n_pthreads; |
| 292 | |
| 293 | /* Number of DPDK eal threads */ |
| 294 | u32 n_eal_threads; |
| 295 | |
| 296 | /* Number of cores to skip, must match the core mask */ |
| 297 | u32 skip_cores; |
| 298 | |
| 299 | /* Thread prefix name */ |
| 300 | u8 *thread_prefix; |
| 301 | |
| 302 | /* main thread lcore */ |
| 303 | u8 main_lcore; |
| 304 | |
| 305 | /* Bitmap of available CPU cores */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 306 | uword *cpu_core_bitmap; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 307 | |
| 308 | /* Bitmap of available CPU sockets (NUMA nodes) */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 309 | uword *cpu_socket_bitmap; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 310 | |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 311 | /* Worker handoff queues */ |
| 312 | vlib_frame_queue_main_t *frame_queue_mains; |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 313 | |
| 314 | /* worker thread initialization barrier */ |
| 315 | volatile u32 worker_thread_release; |
| 316 | |
Pavel Kotucek | 1e76583 | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 317 | /* scheduling policy */ |
| 318 | u32 sched_policy; |
| 319 | |
| 320 | /* scheduling policy priority */ |
| 321 | u32 sched_priority; |
| 322 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 323 | } vlib_thread_main_t; |
| 324 | |
Damjan Marion | 6a7acc2 | 2016-12-19 16:28:36 +0100 | [diff] [blame] | 325 | extern vlib_thread_main_t vlib_thread_main; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 326 | |
| 327 | #define VLIB_REGISTER_THREAD(x,...) \ |
| 328 | __VA_ARGS__ vlib_thread_registration_t x; \ |
| 329 | static void __vlib_add_thread_registration_##x (void) \ |
| 330 | __attribute__((__constructor__)) ; \ |
| 331 | static void __vlib_add_thread_registration_##x (void) \ |
| 332 | { \ |
| 333 | vlib_thread_main_t * tm = &vlib_thread_main; \ |
| 334 | x.next = tm->next; \ |
| 335 | tm->next = &x; \ |
| 336 | } \ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 337 | __VA_ARGS__ vlib_thread_registration_t x |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 338 | |
Damjan Marion | 6403436 | 2016-11-07 22:19:55 +0100 | [diff] [blame] | 339 | always_inline u32 |
| 340 | vlib_num_workers () |
| 341 | { |
| 342 | return vlib_thread_main.n_vlib_mains - 1; |
| 343 | } |
| 344 | |
| 345 | always_inline u32 |
| 346 | vlib_get_worker_cpu_index (u32 worker_index) |
| 347 | { |
| 348 | return worker_index + 1; |
| 349 | } |
| 350 | |
Damjan Marion | 3a4ed39 | 2016-11-08 13:20:42 +0100 | [diff] [blame] | 351 | always_inline u32 |
| 352 | vlib_get_worker_index (u32 cpu_index) |
| 353 | { |
| 354 | return cpu_index - 1; |
| 355 | } |
| 356 | |
| 357 | always_inline u32 |
| 358 | vlib_get_current_worker_index () |
| 359 | { |
| 360 | return os_get_cpu_number () - 1; |
| 361 | } |
| 362 | |
Damjan Marion | 6403436 | 2016-11-07 22:19:55 +0100 | [diff] [blame] | 363 | always_inline vlib_main_t * |
| 364 | vlib_get_worker_vlib_main (u32 worker_index) |
| 365 | { |
| 366 | vlib_main_t *vm; |
| 367 | vlib_thread_main_t *tm = &vlib_thread_main; |
| 368 | ASSERT (worker_index < tm->n_vlib_mains - 1); |
| 369 | vm = vlib_mains[worker_index + 1]; |
| 370 | ASSERT (vm); |
| 371 | return vm; |
| 372 | } |
| 373 | |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 374 | static inline void |
| 375 | vlib_put_frame_queue_elt (vlib_frame_queue_elt_t * hf) |
| 376 | { |
| 377 | CLIB_MEMORY_BARRIER (); |
| 378 | hf->valid = 1; |
| 379 | } |
| 380 | |
| 381 | static inline vlib_frame_queue_elt_t * |
| 382 | vlib_get_frame_queue_elt (u32 frame_queue_index, u32 index) |
| 383 | { |
| 384 | vlib_frame_queue_t *fq; |
| 385 | vlib_frame_queue_elt_t *elt; |
| 386 | vlib_thread_main_t *tm = &vlib_thread_main; |
| 387 | vlib_frame_queue_main_t *fqm = |
| 388 | vec_elt_at_index (tm->frame_queue_mains, frame_queue_index); |
| 389 | u64 new_tail; |
| 390 | |
| 391 | fq = fqm->vlib_frame_queues[index]; |
| 392 | ASSERT (fq); |
| 393 | |
| 394 | new_tail = __sync_add_and_fetch (&fq->tail, 1); |
| 395 | |
| 396 | /* Wait until a ring slot is available */ |
| 397 | while (new_tail >= fq->head_hint + fq->nelts) |
| 398 | vlib_worker_thread_barrier_check (); |
| 399 | |
| 400 | elt = fq->elts + (new_tail & (fq->nelts - 1)); |
| 401 | |
| 402 | /* this would be very bad... */ |
| 403 | while (elt->valid) |
| 404 | ; |
| 405 | |
| 406 | elt->msg_type = VLIB_FRAME_QUEUE_ELT_DISPATCH_FRAME; |
| 407 | elt->last_n_vectors = elt->n_vectors = 0; |
| 408 | |
| 409 | return elt; |
| 410 | } |
| 411 | |
| 412 | static inline vlib_frame_queue_t * |
| 413 | is_vlib_frame_queue_congested (u32 frame_queue_index, |
| 414 | u32 index, |
| 415 | u32 queue_hi_thresh, |
| 416 | vlib_frame_queue_t ** |
| 417 | handoff_queue_by_worker_index) |
| 418 | { |
| 419 | vlib_frame_queue_t *fq; |
| 420 | vlib_thread_main_t *tm = &vlib_thread_main; |
| 421 | vlib_frame_queue_main_t *fqm = |
| 422 | vec_elt_at_index (tm->frame_queue_mains, frame_queue_index); |
| 423 | |
| 424 | fq = handoff_queue_by_worker_index[index]; |
| 425 | if (fq != (vlib_frame_queue_t *) (~0)) |
| 426 | return fq; |
| 427 | |
| 428 | fq = fqm->vlib_frame_queues[index]; |
| 429 | ASSERT (fq); |
| 430 | |
| 431 | if (PREDICT_FALSE (fq->tail >= (fq->head_hint + queue_hi_thresh))) |
| 432 | { |
| 433 | /* a valid entry in the array will indicate the queue has reached |
| 434 | * the specified threshold and is congested |
| 435 | */ |
| 436 | handoff_queue_by_worker_index[index] = fq; |
| 437 | fq->enqueue_full_events++; |
| 438 | return fq; |
| 439 | } |
| 440 | |
| 441 | return NULL; |
| 442 | } |
| 443 | |
| 444 | static inline vlib_frame_queue_elt_t * |
| 445 | vlib_get_worker_handoff_queue_elt (u32 frame_queue_index, |
| 446 | u32 vlib_worker_index, |
| 447 | vlib_frame_queue_elt_t ** |
| 448 | handoff_queue_elt_by_worker_index) |
| 449 | { |
| 450 | vlib_frame_queue_elt_t *elt; |
| 451 | |
| 452 | if (handoff_queue_elt_by_worker_index[vlib_worker_index]) |
| 453 | return handoff_queue_elt_by_worker_index[vlib_worker_index]; |
| 454 | |
| 455 | elt = vlib_get_frame_queue_elt (frame_queue_index, vlib_worker_index); |
| 456 | |
| 457 | handoff_queue_elt_by_worker_index[vlib_worker_index] = elt; |
| 458 | |
| 459 | return elt; |
| 460 | } |
| 461 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 462 | #endif /* included_vlib_threads_h */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 463 | |
| 464 | /* |
| 465 | * fd.io coding-style-patch-verification: ON |
| 466 | * |
| 467 | * Local Variables: |
| 468 | * eval: (c-set-style "gnu") |
| 469 | * End: |
| 470 | */ |