Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2015 Cisco and/or its affiliates. |
| 3 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | * you may not use this file except in compliance with the License. |
| 5 | * You may obtain a copy of the License at: |
| 6 | * |
| 7 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | * |
| 9 | * Unless required by applicable law or agreed to in writing, software |
| 10 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | * See the License for the specific language governing permissions and |
| 13 | * limitations under the License. |
| 14 | */ |
| 15 | /* |
| 16 | * main.c: main vector processing loop |
| 17 | * |
| 18 | * Copyright (c) 2008 Eliot Dresselhaus |
| 19 | * |
| 20 | * Permission is hereby granted, free of charge, to any person obtaining |
| 21 | * a copy of this software and associated documentation files (the |
| 22 | * "Software"), to deal in the Software without restriction, including |
| 23 | * without limitation the rights to use, copy, modify, merge, publish, |
| 24 | * distribute, sublicense, and/or sell copies of the Software, and to |
| 25 | * permit persons to whom the Software is furnished to do so, subject to |
| 26 | * the following conditions: |
| 27 | * |
| 28 | * The above copyright notice and this permission notice shall be |
| 29 | * included in all copies or substantial portions of the Software. |
| 30 | * |
| 31 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 32 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 33 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 34 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE |
| 35 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION |
| 36 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION |
| 37 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
| 38 | */ |
| 39 | |
| 40 | #include <math.h> |
| 41 | #include <vppinfra/format.h> |
| 42 | #include <vlib/vlib.h> |
| 43 | #include <vlib/threads.h> |
Dave Barach | 5c20a01 | 2017-06-13 08:48:31 -0400 | [diff] [blame] | 44 | #include <vppinfra/tw_timer_1t_3w_1024sl_ov.h> |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 45 | |
Damjan Marion | 04a7f05 | 2017-07-10 15:06:17 +0200 | [diff] [blame] | 46 | #include <vlib/unix/unix.h> |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 47 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 48 | /* Actually allocate a few extra slots of vector data to support |
| 49 | speculative vector enqueues which overflow vector data in next frame. */ |
| 50 | #define VLIB_FRAME_SIZE_ALLOC (VLIB_FRAME_SIZE + 4) |
| 51 | |
| 52 | always_inline u32 |
| 53 | vlib_frame_bytes (u32 n_scalar_bytes, u32 n_vector_bytes) |
| 54 | { |
| 55 | u32 n_bytes; |
| 56 | |
| 57 | /* Make room for vlib_frame_t plus scalar arguments. */ |
| 58 | n_bytes = vlib_frame_vector_byte_offset (n_scalar_bytes); |
| 59 | |
| 60 | /* Make room for vector arguments. |
| 61 | Allocate a few extra slots of vector data to support |
| 62 | speculative vector enqueues which overflow vector data in next frame. */ |
| 63 | #define VLIB_FRAME_SIZE_EXTRA 4 |
| 64 | n_bytes += (VLIB_FRAME_SIZE + VLIB_FRAME_SIZE_EXTRA) * n_vector_bytes; |
| 65 | |
| 66 | /* Magic number is first 32bit number after vector data. |
| 67 | Used to make sure that vector data is never overrun. */ |
| 68 | #define VLIB_FRAME_MAGIC (0xabadc0ed) |
| 69 | n_bytes += sizeof (u32); |
| 70 | |
| 71 | /* Pad to cache line. */ |
| 72 | n_bytes = round_pow2 (n_bytes, CLIB_CACHE_LINE_BYTES); |
| 73 | |
| 74 | return n_bytes; |
| 75 | } |
| 76 | |
| 77 | always_inline u32 * |
| 78 | vlib_frame_find_magic (vlib_frame_t * f, vlib_node_t * node) |
| 79 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 80 | void *p = f; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 81 | |
| 82 | p += vlib_frame_vector_byte_offset (node->scalar_size); |
| 83 | |
| 84 | p += (VLIB_FRAME_SIZE + VLIB_FRAME_SIZE_EXTRA) * node->vector_size; |
| 85 | |
| 86 | return p; |
| 87 | } |
| 88 | |
Dave Barach | 593eedf | 2019-03-10 09:44:51 -0400 | [diff] [blame] | 89 | static inline vlib_frame_size_t * |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 90 | get_frame_size_info (vlib_node_main_t * nm, |
| 91 | u32 n_scalar_bytes, u32 n_vector_bytes) |
| 92 | { |
Dave Barach | 593eedf | 2019-03-10 09:44:51 -0400 | [diff] [blame] | 93 | #ifdef VLIB_SUPPORTS_ARBITRARY_SCALAR_SIZES |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 94 | uword key = (n_scalar_bytes << 16) | n_vector_bytes; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 95 | uword *p, i; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 96 | |
| 97 | p = hash_get (nm->frame_size_hash, key); |
| 98 | if (p) |
| 99 | i = p[0]; |
| 100 | else |
| 101 | { |
| 102 | i = vec_len (nm->frame_sizes); |
| 103 | vec_validate (nm->frame_sizes, i); |
| 104 | hash_set (nm->frame_size_hash, key, i); |
| 105 | } |
| 106 | |
| 107 | return vec_elt_at_index (nm->frame_sizes, i); |
Dave Barach | 593eedf | 2019-03-10 09:44:51 -0400 | [diff] [blame] | 108 | #else |
| 109 | ASSERT (vlib_frame_bytes (n_scalar_bytes, n_vector_bytes) |
| 110 | == (vlib_frame_bytes (0, 4))); |
| 111 | return vec_elt_at_index (nm->frame_sizes, 0); |
| 112 | #endif |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 113 | } |
| 114 | |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 115 | static vlib_frame_t * |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 116 | vlib_frame_alloc_to_node (vlib_main_t * vm, u32 to_node_index, |
| 117 | u32 frame_flags) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 118 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 119 | vlib_node_main_t *nm = &vm->node_main; |
| 120 | vlib_frame_size_t *fs; |
| 121 | vlib_node_t *to_node; |
| 122 | vlib_frame_t *f; |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 123 | u32 l, n, scalar_size, vector_size; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 124 | |
Dave Barach | a8f4ebd | 2021-02-08 07:56:22 -0500 | [diff] [blame] | 125 | ASSERT (vm == vlib_get_main ()); |
| 126 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 127 | to_node = vlib_get_node (vm, to_node_index); |
| 128 | |
| 129 | scalar_size = to_node->scalar_size; |
| 130 | vector_size = to_node->vector_size; |
| 131 | |
| 132 | fs = get_frame_size_info (nm, scalar_size, vector_size); |
| 133 | n = vlib_frame_bytes (scalar_size, vector_size); |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 134 | if ((l = vec_len (fs->free_frames)) > 0) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 135 | { |
| 136 | /* Allocate from end of free list. */ |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 137 | f = fs->free_frames[l - 1]; |
| 138 | _vec_len (fs->free_frames) = l - 1; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 139 | } |
| 140 | else |
| 141 | { |
Damjan Marion | 3f46baf | 2016-02-06 19:16:21 +0100 | [diff] [blame] | 142 | f = clib_mem_alloc_aligned_no_fail (n, VLIB_FRAME_ALIGN); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 143 | } |
| 144 | |
| 145 | /* Poison frame when debugging. */ |
| 146 | if (CLIB_DEBUG > 0) |
Dave Barach | b7b9299 | 2018-10-17 10:38:51 -0400 | [diff] [blame] | 147 | clib_memset (f, 0xfe, n); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 148 | |
| 149 | /* Insert magic number. */ |
| 150 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 151 | u32 *magic; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 152 | |
| 153 | magic = vlib_frame_find_magic (f, to_node); |
| 154 | *magic = VLIB_FRAME_MAGIC; |
| 155 | } |
| 156 | |
Damjan Marion | 633b6fd | 2018-09-14 14:38:53 +0200 | [diff] [blame] | 157 | f->frame_flags = VLIB_FRAME_IS_ALLOCATED | frame_flags; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 158 | f->n_vectors = 0; |
| 159 | f->scalar_size = scalar_size; |
| 160 | f->vector_size = vector_size; |
Damjan Marion | a3d5986 | 2018-11-10 10:23:00 +0100 | [diff] [blame] | 161 | f->flags = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 162 | |
| 163 | fs->n_alloc_frames += 1; |
| 164 | |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 165 | return f; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 166 | } |
| 167 | |
| 168 | /* Allocate a frame for from FROM_NODE to TO_NODE via TO_NEXT_INDEX. |
| 169 | Returns frame index. */ |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 170 | static vlib_frame_t * |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 171 | vlib_frame_alloc (vlib_main_t * vm, vlib_node_runtime_t * from_node_runtime, |
| 172 | u32 to_next_index) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 173 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 174 | vlib_node_t *from_node; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 175 | |
| 176 | from_node = vlib_get_node (vm, from_node_runtime->node_index); |
| 177 | ASSERT (to_next_index < vec_len (from_node->next_nodes)); |
| 178 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 179 | return vlib_frame_alloc_to_node (vm, from_node->next_nodes[to_next_index], |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 180 | /* frame_flags */ 0); |
| 181 | } |
| 182 | |
| 183 | vlib_frame_t * |
| 184 | vlib_get_frame_to_node (vlib_main_t * vm, u32 to_node_index) |
| 185 | { |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 186 | vlib_frame_t *f = vlib_frame_alloc_to_node (vm, to_node_index, |
| 187 | /* frame_flags */ |
| 188 | VLIB_FRAME_FREE_AFTER_DISPATCH); |
| 189 | return vlib_get_frame (vm, f); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 190 | } |
| 191 | |
Dave Barach | c74b43c | 2020-04-09 17:24:07 -0400 | [diff] [blame] | 192 | static inline void |
| 193 | vlib_validate_frame_indices (vlib_frame_t * f) |
| 194 | { |
| 195 | if (CLIB_DEBUG > 0) |
| 196 | { |
| 197 | int i; |
| 198 | u32 *from = vlib_frame_vector_args (f); |
| 199 | |
| 200 | /* Check for bad buffer index values */ |
| 201 | for (i = 0; i < f->n_vectors; i++) |
| 202 | { |
| 203 | if (from[i] == 0) |
| 204 | { |
| 205 | clib_warning ("BUG: buffer index 0 at index %d", i); |
| 206 | ASSERT (0); |
| 207 | } |
| 208 | else if (from[i] == 0xfefefefe) |
| 209 | { |
| 210 | clib_warning ("BUG: frame poison pattern at index %d", i); |
| 211 | ASSERT (0); |
| 212 | } |
| 213 | } |
| 214 | } |
| 215 | } |
| 216 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 217 | void |
| 218 | vlib_put_frame_to_node (vlib_main_t * vm, u32 to_node_index, vlib_frame_t * f) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 219 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 220 | vlib_pending_frame_t *p; |
| 221 | vlib_node_t *to_node; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 222 | |
| 223 | if (f->n_vectors == 0) |
| 224 | return; |
| 225 | |
Dave Barach | a8f4ebd | 2021-02-08 07:56:22 -0500 | [diff] [blame] | 226 | ASSERT (vm == vlib_get_main ()); |
| 227 | |
Dave Barach | c74b43c | 2020-04-09 17:24:07 -0400 | [diff] [blame] | 228 | vlib_validate_frame_indices (f); |
| 229 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 230 | to_node = vlib_get_node (vm, to_node_index); |
| 231 | |
| 232 | vec_add2 (vm->node_main.pending_frames, p, 1); |
| 233 | |
Damjan Marion | 633b6fd | 2018-09-14 14:38:53 +0200 | [diff] [blame] | 234 | f->frame_flags |= VLIB_FRAME_PENDING; |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 235 | p->frame = vlib_get_frame (vm, f); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 236 | p->node_runtime_index = to_node->runtime_index; |
| 237 | p->next_frame_index = VLIB_PENDING_FRAME_NO_NEXT_FRAME; |
| 238 | } |
| 239 | |
| 240 | /* Free given frame. */ |
| 241 | void |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 242 | vlib_frame_free (vlib_main_t * vm, vlib_node_runtime_t * r, vlib_frame_t * f) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 243 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 244 | vlib_node_main_t *nm = &vm->node_main; |
| 245 | vlib_node_t *node; |
| 246 | vlib_frame_size_t *fs; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 247 | |
Dave Barach | a8f4ebd | 2021-02-08 07:56:22 -0500 | [diff] [blame] | 248 | ASSERT (vm == vlib_get_main ()); |
Damjan Marion | 633b6fd | 2018-09-14 14:38:53 +0200 | [diff] [blame] | 249 | ASSERT (f->frame_flags & VLIB_FRAME_IS_ALLOCATED); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 250 | |
| 251 | node = vlib_get_node (vm, r->node_index); |
| 252 | fs = get_frame_size_info (nm, node->scalar_size, node->vector_size); |
| 253 | |
Damjan Marion | 633b6fd | 2018-09-14 14:38:53 +0200 | [diff] [blame] | 254 | ASSERT (f->frame_flags & VLIB_FRAME_IS_ALLOCATED); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 255 | |
| 256 | /* No next frames may point to freed frame. */ |
| 257 | if (CLIB_DEBUG > 0) |
| 258 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 259 | vlib_next_frame_t *nf; |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 260 | vec_foreach (nf, vm->node_main.next_frames) ASSERT (nf->frame != f); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 261 | } |
| 262 | |
Damjan Marion | 296988d | 2019-02-21 20:24:54 +0100 | [diff] [blame] | 263 | f->frame_flags &= ~(VLIB_FRAME_IS_ALLOCATED | VLIB_FRAME_NO_APPEND); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 264 | |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 265 | vec_add1 (fs->free_frames, f); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 266 | ASSERT (fs->n_alloc_frames > 0); |
| 267 | fs->n_alloc_frames -= 1; |
| 268 | } |
| 269 | |
| 270 | static clib_error_t * |
| 271 | show_frame_stats (vlib_main_t * vm, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 272 | unformat_input_t * input, vlib_cli_command_t * cmd) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 273 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 274 | vlib_node_main_t *nm = &vm->node_main; |
| 275 | vlib_frame_size_t *fs; |
| 276 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 277 | vlib_cli_output (vm, "%=6s%=12s%=12s", "Size", "# Alloc", "# Free"); |
| 278 | vec_foreach (fs, nm->frame_sizes) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 279 | { |
| 280 | u32 n_alloc = fs->n_alloc_frames; |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 281 | u32 n_free = vec_len (fs->free_frames); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 282 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 283 | if (n_alloc + n_free > 0) |
| 284 | vlib_cli_output (vm, "%=6d%=12d%=12d", |
| 285 | fs - nm->frame_sizes, n_alloc, n_free); |
| 286 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 287 | |
| 288 | return 0; |
| 289 | } |
| 290 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 291 | /* *INDENT-OFF* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 292 | VLIB_CLI_COMMAND (show_frame_stats_cli, static) = { |
| 293 | .path = "show vlib frame-allocation", |
| 294 | .short_help = "Show node dispatch frame statistics", |
| 295 | .function = show_frame_stats, |
| 296 | }; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 297 | /* *INDENT-ON* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 298 | |
| 299 | /* Change ownership of enqueue rights to given next node. */ |
| 300 | static void |
| 301 | vlib_next_frame_change_ownership (vlib_main_t * vm, |
| 302 | vlib_node_runtime_t * node_runtime, |
| 303 | u32 next_index) |
| 304 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 305 | vlib_node_main_t *nm = &vm->node_main; |
| 306 | vlib_next_frame_t *next_frame; |
| 307 | vlib_node_t *node, *next_node; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 308 | |
| 309 | node = vec_elt (nm->nodes, node_runtime->node_index); |
| 310 | |
| 311 | /* Only internal & input nodes are allowed to call other nodes. */ |
| 312 | ASSERT (node->type == VLIB_NODE_TYPE_INTERNAL |
| 313 | || node->type == VLIB_NODE_TYPE_INPUT |
| 314 | || node->type == VLIB_NODE_TYPE_PROCESS); |
| 315 | |
| 316 | ASSERT (vec_len (node->next_nodes) == node_runtime->n_next_nodes); |
| 317 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 318 | next_frame = |
| 319 | vlib_node_runtime_get_next_frame (vm, node_runtime, next_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 320 | next_node = vec_elt (nm->nodes, node->next_nodes[next_index]); |
| 321 | |
| 322 | if (next_node->owner_node_index != VLIB_INVALID_NODE_INDEX) |
| 323 | { |
| 324 | /* Get frame from previous owner. */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 325 | vlib_next_frame_t *owner_next_frame; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 326 | vlib_next_frame_t tmp; |
| 327 | |
| 328 | owner_next_frame = |
| 329 | vlib_node_get_next_frame (vm, |
| 330 | next_node->owner_node_index, |
| 331 | next_node->owner_next_index); |
| 332 | |
| 333 | /* Swap target next frame with owner's. */ |
| 334 | tmp = owner_next_frame[0]; |
| 335 | owner_next_frame[0] = next_frame[0]; |
| 336 | next_frame[0] = tmp; |
| 337 | |
| 338 | /* |
| 339 | * If next_frame is already pending, we have to track down |
| 340 | * all pending frames and fix their next_frame_index fields. |
| 341 | */ |
| 342 | if (next_frame->flags & VLIB_FRAME_PENDING) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 343 | { |
| 344 | vlib_pending_frame_t *p; |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 345 | if (next_frame->frame != NULL) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 346 | { |
| 347 | vec_foreach (p, nm->pending_frames) |
| 348 | { |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 349 | if (p->frame == next_frame->frame) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 350 | { |
| 351 | p->next_frame_index = |
| 352 | next_frame - vm->node_main.next_frames; |
| 353 | } |
| 354 | } |
| 355 | } |
| 356 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 357 | } |
| 358 | else |
| 359 | { |
| 360 | /* No previous owner. Take ownership. */ |
| 361 | next_frame->flags |= VLIB_FRAME_OWNER; |
| 362 | } |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 363 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 364 | /* Record new owner. */ |
| 365 | next_node->owner_node_index = node->index; |
| 366 | next_node->owner_next_index = next_index; |
| 367 | |
| 368 | /* Now we should be owner. */ |
| 369 | ASSERT (next_frame->flags & VLIB_FRAME_OWNER); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 370 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 371 | |
| 372 | /* Make sure that magic number is still there. |
| 373 | Otherwise, it is likely that caller has overrun frame arguments. */ |
| 374 | always_inline void |
| 375 | validate_frame_magic (vlib_main_t * vm, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 376 | vlib_frame_t * f, vlib_node_t * n, uword next_index) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 377 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 378 | vlib_node_t *next_node = vlib_get_node (vm, n->next_nodes[next_index]); |
| 379 | u32 *magic = vlib_frame_find_magic (f, next_node); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 380 | ASSERT (VLIB_FRAME_MAGIC == magic[0]); |
| 381 | } |
| 382 | |
| 383 | vlib_frame_t * |
| 384 | vlib_get_next_frame_internal (vlib_main_t * vm, |
| 385 | vlib_node_runtime_t * node, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 386 | u32 next_index, u32 allocate_new_next_frame) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 387 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 388 | vlib_frame_t *f; |
| 389 | vlib_next_frame_t *nf; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 390 | u32 n_used; |
| 391 | |
| 392 | nf = vlib_node_runtime_get_next_frame (vm, node, next_index); |
| 393 | |
| 394 | /* Make sure this next frame owns right to enqueue to destination frame. */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 395 | if (PREDICT_FALSE (!(nf->flags & VLIB_FRAME_OWNER))) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 396 | vlib_next_frame_change_ownership (vm, node, next_index); |
| 397 | |
| 398 | /* ??? Don't need valid flag: can use frame_index == ~0 */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 399 | if (PREDICT_FALSE (!(nf->flags & VLIB_FRAME_IS_ALLOCATED))) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 400 | { |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 401 | nf->frame = vlib_frame_alloc (vm, node, next_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 402 | nf->flags |= VLIB_FRAME_IS_ALLOCATED; |
| 403 | } |
| 404 | |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 405 | f = nf->frame; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 406 | |
| 407 | /* Has frame been removed from pending vector (e.g. finished dispatching)? |
| 408 | If so we can reuse frame. */ |
Damjan Marion | 633b6fd | 2018-09-14 14:38:53 +0200 | [diff] [blame] | 409 | if ((nf->flags & VLIB_FRAME_PENDING) |
| 410 | && !(f->frame_flags & VLIB_FRAME_PENDING)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 411 | { |
| 412 | nf->flags &= ~VLIB_FRAME_PENDING; |
| 413 | f->n_vectors = 0; |
Damjan Marion | 9162c2d | 2018-11-20 09:55:10 +0100 | [diff] [blame] | 414 | f->flags = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 415 | } |
| 416 | |
Damjan Marion | 296988d | 2019-02-21 20:24:54 +0100 | [diff] [blame] | 417 | /* Allocate new frame if current one is marked as no-append or |
| 418 | it is already full. */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 419 | n_used = f->n_vectors; |
Damjan Marion | 296988d | 2019-02-21 20:24:54 +0100 | [diff] [blame] | 420 | if (n_used >= VLIB_FRAME_SIZE || (allocate_new_next_frame && n_used > 0) || |
| 421 | (f->frame_flags & VLIB_FRAME_NO_APPEND)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 422 | { |
| 423 | /* Old frame may need to be freed after dispatch, since we'll have |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 424 | two redundant frames from node -> next node. */ |
| 425 | if (!(nf->flags & VLIB_FRAME_NO_FREE_AFTER_DISPATCH)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 426 | { |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 427 | vlib_frame_t *f_old = vlib_get_frame (vm, nf->frame); |
Damjan Marion | 633b6fd | 2018-09-14 14:38:53 +0200 | [diff] [blame] | 428 | f_old->frame_flags |= VLIB_FRAME_FREE_AFTER_DISPATCH; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 429 | } |
| 430 | |
| 431 | /* Allocate new frame to replace full one. */ |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 432 | f = nf->frame = vlib_frame_alloc (vm, node, next_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 433 | n_used = f->n_vectors; |
| 434 | } |
| 435 | |
| 436 | /* Should have free vectors in frame now. */ |
| 437 | ASSERT (n_used < VLIB_FRAME_SIZE); |
| 438 | |
| 439 | if (CLIB_DEBUG > 0) |
| 440 | { |
| 441 | validate_frame_magic (vm, f, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 442 | vlib_get_node (vm, node->node_index), next_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 443 | } |
| 444 | |
| 445 | return f; |
| 446 | } |
| 447 | |
| 448 | static void |
| 449 | vlib_put_next_frame_validate (vlib_main_t * vm, |
| 450 | vlib_node_runtime_t * rt, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 451 | u32 next_index, u32 n_vectors_left) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 452 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 453 | vlib_node_main_t *nm = &vm->node_main; |
| 454 | vlib_next_frame_t *nf; |
| 455 | vlib_frame_t *f; |
| 456 | vlib_node_runtime_t *next_rt; |
| 457 | vlib_node_t *next_node; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 458 | u32 n_before, n_after; |
| 459 | |
| 460 | nf = vlib_node_runtime_get_next_frame (vm, rt, next_index); |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 461 | f = vlib_get_frame (vm, nf->frame); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 462 | |
| 463 | ASSERT (n_vectors_left <= VLIB_FRAME_SIZE); |
Dave Barach | c74b43c | 2020-04-09 17:24:07 -0400 | [diff] [blame] | 464 | |
| 465 | vlib_validate_frame_indices (f); |
| 466 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 467 | n_after = VLIB_FRAME_SIZE - n_vectors_left; |
| 468 | n_before = f->n_vectors; |
| 469 | |
| 470 | ASSERT (n_after >= n_before); |
| 471 | |
| 472 | next_rt = vec_elt_at_index (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL], |
| 473 | nf->node_runtime_index); |
| 474 | next_node = vlib_get_node (vm, next_rt->node_index); |
| 475 | if (n_after > 0 && next_node->validate_frame) |
| 476 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 477 | u8 *msg = next_node->validate_frame (vm, rt, f); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 478 | if (msg) |
| 479 | { |
| 480 | clib_warning ("%v", msg); |
| 481 | ASSERT (0); |
| 482 | } |
| 483 | vec_free (msg); |
| 484 | } |
| 485 | } |
| 486 | |
| 487 | void |
| 488 | vlib_put_next_frame (vlib_main_t * vm, |
| 489 | vlib_node_runtime_t * r, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 490 | u32 next_index, u32 n_vectors_left) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 491 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 492 | vlib_node_main_t *nm = &vm->node_main; |
| 493 | vlib_next_frame_t *nf; |
| 494 | vlib_frame_t *f; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 495 | u32 n_vectors_in_frame; |
| 496 | |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 497 | if (CLIB_DEBUG > 0) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 498 | vlib_put_next_frame_validate (vm, r, next_index, n_vectors_left); |
| 499 | |
| 500 | nf = vlib_node_runtime_get_next_frame (vm, r, next_index); |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 501 | f = vlib_get_frame (vm, nf->frame); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 502 | |
| 503 | /* Make sure that magic number is still there. Otherwise, caller |
| 504 | has overrun frame meta data. */ |
| 505 | if (CLIB_DEBUG > 0) |
| 506 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 507 | vlib_node_t *node = vlib_get_node (vm, r->node_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 508 | validate_frame_magic (vm, f, node, next_index); |
| 509 | } |
| 510 | |
| 511 | /* Convert # of vectors left -> number of vectors there. */ |
| 512 | ASSERT (n_vectors_left <= VLIB_FRAME_SIZE); |
| 513 | n_vectors_in_frame = VLIB_FRAME_SIZE - n_vectors_left; |
| 514 | |
| 515 | f->n_vectors = n_vectors_in_frame; |
| 516 | |
| 517 | /* If vectors were added to frame, add to pending vector. */ |
| 518 | if (PREDICT_TRUE (n_vectors_in_frame > 0)) |
| 519 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 520 | vlib_pending_frame_t *p; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 521 | u32 v0, v1; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 522 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 523 | r->cached_next_index = next_index; |
| 524 | |
Damjan Marion | 633b6fd | 2018-09-14 14:38:53 +0200 | [diff] [blame] | 525 | if (!(f->frame_flags & VLIB_FRAME_PENDING)) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 526 | { |
| 527 | __attribute__ ((unused)) vlib_node_t *node; |
| 528 | vlib_node_t *next_node; |
| 529 | vlib_node_runtime_t *next_runtime; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 530 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 531 | node = vlib_get_node (vm, r->node_index); |
| 532 | next_node = vlib_get_next_node (vm, r->node_index, next_index); |
| 533 | next_runtime = vlib_node_get_runtime (vm, next_node->index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 534 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 535 | vec_add2 (nm->pending_frames, p, 1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 536 | |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 537 | p->frame = nf->frame; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 538 | p->node_runtime_index = nf->node_runtime_index; |
| 539 | p->next_frame_index = nf - nm->next_frames; |
| 540 | nf->flags |= VLIB_FRAME_PENDING; |
Damjan Marion | 633b6fd | 2018-09-14 14:38:53 +0200 | [diff] [blame] | 541 | f->frame_flags |= VLIB_FRAME_PENDING; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 542 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 543 | /* |
| 544 | * If we're going to dispatch this frame on another thread, |
| 545 | * force allocation of a new frame. Otherwise, we create |
| 546 | * a dangling frame reference. Each thread has its own copy of |
| 547 | * the next_frames vector. |
| 548 | */ |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 549 | if (0 && r->thread_index != next_runtime->thread_index) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 550 | { |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 551 | nf->frame = NULL; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 552 | nf->flags &= ~(VLIB_FRAME_PENDING | VLIB_FRAME_IS_ALLOCATED); |
| 553 | } |
| 554 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 555 | |
| 556 | /* Copy trace flag from next_frame and from runtime. */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 557 | nf->flags |= |
| 558 | (nf->flags & VLIB_NODE_FLAG_TRACE) | (r-> |
| 559 | flags & VLIB_NODE_FLAG_TRACE); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 560 | |
| 561 | v0 = nf->vectors_since_last_overflow; |
| 562 | v1 = v0 + n_vectors_in_frame; |
| 563 | nf->vectors_since_last_overflow = v1; |
| 564 | if (PREDICT_FALSE (v1 < v0)) |
| 565 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 566 | vlib_node_t *node = vlib_get_node (vm, r->node_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 567 | vec_elt (node->n_vectors_by_next_node, next_index) += v0; |
| 568 | } |
| 569 | } |
| 570 | } |
| 571 | |
| 572 | /* Sync up runtime (32 bit counters) and main node stats (64 bit counters). */ |
Arthur de Kerhor | 156158f | 2021-02-18 03:09:42 -0800 | [diff] [blame] | 573 | void |
| 574 | vlib_node_runtime_sync_stats_node (vlib_node_t *n, vlib_node_runtime_t *r, |
| 575 | uword n_calls, uword n_vectors, |
| 576 | uword n_clocks) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 577 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 578 | n->stats_total.calls += n_calls + r->calls_since_last_overflow; |
| 579 | n->stats_total.vectors += n_vectors + r->vectors_since_last_overflow; |
| 580 | n->stats_total.clocks += n_clocks + r->clocks_since_last_overflow; |
| 581 | n->stats_total.max_clock = r->max_clock; |
| 582 | n->stats_total.max_clock_n = r->max_clock_n; |
| 583 | |
| 584 | r->calls_since_last_overflow = 0; |
| 585 | r->vectors_since_last_overflow = 0; |
| 586 | r->clocks_since_last_overflow = 0; |
| 587 | } |
| 588 | |
Arthur de Kerhor | 156158f | 2021-02-18 03:09:42 -0800 | [diff] [blame] | 589 | void |
| 590 | vlib_node_runtime_sync_stats (vlib_main_t *vm, vlib_node_runtime_t *r, |
| 591 | uword n_calls, uword n_vectors, uword n_clocks) |
| 592 | { |
| 593 | vlib_node_t *n = vlib_get_node (vm, r->node_index); |
| 594 | vlib_node_runtime_sync_stats_node (n, r, n_calls, n_vectors, n_clocks); |
| 595 | } |
| 596 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 597 | always_inline void __attribute__ ((unused)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 598 | vlib_process_sync_stats (vlib_main_t * vm, |
| 599 | vlib_process_t * p, |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 600 | uword n_calls, uword n_vectors, uword n_clocks) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 601 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 602 | vlib_node_runtime_t *rt = &p->node_runtime; |
| 603 | vlib_node_t *n = vlib_get_node (vm, rt->node_index); |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 604 | vlib_node_runtime_sync_stats (vm, rt, n_calls, n_vectors, n_clocks); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 605 | n->stats_total.suspends += p->n_suspends; |
| 606 | p->n_suspends = 0; |
| 607 | } |
| 608 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 609 | void |
| 610 | vlib_node_sync_stats (vlib_main_t * vm, vlib_node_t * n) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 611 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 612 | vlib_node_runtime_t *rt; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 613 | |
| 614 | if (n->type == VLIB_NODE_TYPE_PROCESS) |
| 615 | { |
| 616 | /* Nothing to do for PROCESS nodes except in main thread */ |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 617 | if (vm != vlib_get_first_main ()) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 618 | return; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 619 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 620 | vlib_process_t *p = vlib_get_process_from_node (vm, n); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 621 | n->stats_total.suspends += p->n_suspends; |
| 622 | p->n_suspends = 0; |
| 623 | rt = &p->node_runtime; |
| 624 | } |
| 625 | else |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 626 | rt = |
| 627 | vec_elt_at_index (vm->node_main.nodes_by_type[n->type], |
| 628 | n->runtime_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 629 | |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 630 | vlib_node_runtime_sync_stats (vm, rt, 0, 0, 0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 631 | |
| 632 | /* Sync up runtime next frame vector counters with main node structure. */ |
| 633 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 634 | vlib_next_frame_t *nf; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 635 | uword i; |
| 636 | for (i = 0; i < rt->n_next_nodes; i++) |
| 637 | { |
| 638 | nf = vlib_node_runtime_get_next_frame (vm, rt, i); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 639 | vec_elt (n->n_vectors_by_next_node, i) += |
| 640 | nf->vectors_since_last_overflow; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 641 | nf->vectors_since_last_overflow = 0; |
| 642 | } |
| 643 | } |
| 644 | } |
| 645 | |
| 646 | always_inline u32 |
| 647 | vlib_node_runtime_update_stats (vlib_main_t * vm, |
| 648 | vlib_node_runtime_t * node, |
| 649 | uword n_calls, |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 650 | uword n_vectors, uword n_clocks) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 651 | { |
| 652 | u32 ca0, ca1, v0, v1, cl0, cl1, r; |
| 653 | |
| 654 | cl0 = cl1 = node->clocks_since_last_overflow; |
| 655 | ca0 = ca1 = node->calls_since_last_overflow; |
| 656 | v0 = v1 = node->vectors_since_last_overflow; |
| 657 | |
| 658 | ca1 = ca0 + n_calls; |
| 659 | v1 = v0 + n_vectors; |
| 660 | cl1 = cl0 + n_clocks; |
| 661 | |
| 662 | node->calls_since_last_overflow = ca1; |
| 663 | node->clocks_since_last_overflow = cl1; |
| 664 | node->vectors_since_last_overflow = v1; |
Dave Barach | 4d1a866 | 2018-09-10 12:31:15 -0400 | [diff] [blame] | 665 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 666 | node->max_clock_n = node->max_clock > n_clocks ? |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 667 | node->max_clock_n : n_vectors; |
| 668 | node->max_clock = node->max_clock > n_clocks ? node->max_clock : n_clocks; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 669 | |
| 670 | r = vlib_node_runtime_update_main_loop_vector_stats (vm, node, n_vectors); |
| 671 | |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 672 | if (PREDICT_FALSE (ca1 < ca0 || v1 < v0 || cl1 < cl0)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 673 | { |
| 674 | node->calls_since_last_overflow = ca0; |
| 675 | node->clocks_since_last_overflow = cl0; |
| 676 | node->vectors_since_last_overflow = v0; |
Dave Barach | 4d1a866 | 2018-09-10 12:31:15 -0400 | [diff] [blame] | 677 | |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 678 | vlib_node_runtime_sync_stats (vm, node, n_calls, n_vectors, n_clocks); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 679 | } |
| 680 | |
| 681 | return r; |
| 682 | } |
| 683 | |
Dave Barach | 17e5d80 | 2019-05-01 11:30:13 -0400 | [diff] [blame] | 684 | always_inline void |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 685 | vlib_process_update_stats (vlib_main_t * vm, |
| 686 | vlib_process_t * p, |
Dave Barach | ec595ef | 2019-01-24 10:34:24 -0500 | [diff] [blame] | 687 | uword n_calls, uword n_vectors, uword n_clocks) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 688 | { |
| 689 | vlib_node_runtime_update_stats (vm, &p->node_runtime, |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 690 | n_calls, n_vectors, n_clocks); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 691 | } |
| 692 | |
| 693 | static clib_error_t * |
| 694 | vlib_cli_elog_clear (vlib_main_t * vm, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 695 | unformat_input_t * input, vlib_cli_command_t * cmd) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 696 | { |
Damjan Marion | f553a2c | 2021-03-26 13:45:37 +0100 | [diff] [blame] | 697 | elog_reset_buffer (&vlib_global_main.elog_main); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 698 | return 0; |
| 699 | } |
| 700 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 701 | /* *INDENT-OFF* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 702 | VLIB_CLI_COMMAND (elog_clear_cli, static) = { |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 703 | .path = "event-logger clear", |
| 704 | .short_help = "Clear the event log", |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 705 | .function = vlib_cli_elog_clear, |
| 706 | }; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 707 | /* *INDENT-ON* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 708 | |
| 709 | #ifdef CLIB_UNIX |
| 710 | static clib_error_t * |
| 711 | elog_save_buffer (vlib_main_t * vm, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 712 | unformat_input_t * input, vlib_cli_command_t * cmd) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 713 | { |
Damjan Marion | f553a2c | 2021-03-26 13:45:37 +0100 | [diff] [blame] | 714 | elog_main_t *em = &vlib_global_main.elog_main; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 715 | char *file, *chroot_file; |
| 716 | clib_error_t *error = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 717 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 718 | if (!unformat (input, "%s", &file)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 719 | { |
| 720 | vlib_cli_output (vm, "expected file name, got `%U'", |
| 721 | format_unformat_error, input); |
| 722 | return 0; |
| 723 | } |
| 724 | |
| 725 | /* It's fairly hard to get "../oopsie" through unformat; just in case */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 726 | if (strstr (file, "..") || index (file, '/')) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 727 | { |
| 728 | vlib_cli_output (vm, "illegal characters in filename '%s'", file); |
| 729 | return 0; |
| 730 | } |
| 731 | |
| 732 | chroot_file = (char *) format (0, "/tmp/%s%c", file, 0); |
| 733 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 734 | vec_free (file); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 735 | |
| 736 | vlib_cli_output (vm, "Saving %wd of %wd events to %s", |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 737 | elog_n_events_in_buffer (em), |
| 738 | elog_buffer_capacity (em), chroot_file); |
| 739 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 740 | vlib_worker_thread_barrier_sync (vm); |
Dave Barach | 903fd51 | 2017-04-01 11:07:40 -0400 | [diff] [blame] | 741 | error = elog_write_file (em, chroot_file, 1 /* flush ring */ ); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 742 | vlib_worker_thread_barrier_release (vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 743 | vec_free (chroot_file); |
| 744 | return error; |
| 745 | } |
| 746 | |
Dave Barach | 8148131 | 2017-05-16 09:08:14 -0400 | [diff] [blame] | 747 | void |
Dave Barach | 27d978c | 2020-11-03 09:59:06 -0500 | [diff] [blame] | 748 | vlib_post_mortem_dump (void) |
Dave Barach | 8148131 | 2017-05-16 09:08:14 -0400 | [diff] [blame] | 749 | { |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 750 | vlib_global_main_t *vgm = vlib_get_global_main (); |
Dave Barach | 27d978c | 2020-11-03 09:59:06 -0500 | [diff] [blame] | 751 | |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 752 | for (int i = 0; i < vec_len (vgm->post_mortem_callbacks); i++) |
| 753 | (vgm->post_mortem_callbacks[i]) (); |
Dave Barach | 8148131 | 2017-05-16 09:08:14 -0400 | [diff] [blame] | 754 | } |
| 755 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 756 | /* *INDENT-OFF* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 757 | VLIB_CLI_COMMAND (elog_save_cli, static) = { |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 758 | .path = "event-logger save", |
| 759 | .short_help = "event-logger save <filename> (saves log in /tmp/<filename>)", |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 760 | .function = elog_save_buffer, |
| 761 | }; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 762 | /* *INDENT-ON* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 763 | |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 764 | static clib_error_t * |
| 765 | elog_stop (vlib_main_t * vm, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 766 | unformat_input_t * input, vlib_cli_command_t * cmd) |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 767 | { |
Damjan Marion | f553a2c | 2021-03-26 13:45:37 +0100 | [diff] [blame] | 768 | elog_main_t *em = &vlib_global_main.elog_main; |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 769 | |
| 770 | em->n_total_events_disable_limit = em->n_total_events; |
| 771 | |
| 772 | vlib_cli_output (vm, "Stopped the event logger..."); |
| 773 | return 0; |
| 774 | } |
| 775 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 776 | /* *INDENT-OFF* */ |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 777 | VLIB_CLI_COMMAND (elog_stop_cli, static) = { |
| 778 | .path = "event-logger stop", |
| 779 | .short_help = "Stop the event-logger", |
| 780 | .function = elog_stop, |
| 781 | }; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 782 | /* *INDENT-ON* */ |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 783 | |
| 784 | static clib_error_t * |
| 785 | elog_restart (vlib_main_t * vm, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 786 | unformat_input_t * input, vlib_cli_command_t * cmd) |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 787 | { |
Damjan Marion | f553a2c | 2021-03-26 13:45:37 +0100 | [diff] [blame] | 788 | elog_main_t *em = &vlib_global_main.elog_main; |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 789 | |
| 790 | em->n_total_events_disable_limit = ~0; |
| 791 | |
| 792 | vlib_cli_output (vm, "Restarted the event logger..."); |
| 793 | return 0; |
| 794 | } |
| 795 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 796 | /* *INDENT-OFF* */ |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 797 | VLIB_CLI_COMMAND (elog_restart_cli, static) = { |
| 798 | .path = "event-logger restart", |
| 799 | .short_help = "Restart the event-logger", |
| 800 | .function = elog_restart, |
| 801 | }; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 802 | /* *INDENT-ON* */ |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 803 | |
| 804 | static clib_error_t * |
Dave Barach | bc867c3 | 2020-11-25 10:07:09 -0500 | [diff] [blame] | 805 | elog_resize_command_fn (vlib_main_t * vm, |
| 806 | unformat_input_t * input, vlib_cli_command_t * cmd) |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 807 | { |
Damjan Marion | f553a2c | 2021-03-26 13:45:37 +0100 | [diff] [blame] | 808 | elog_main_t *em = &vlib_global_main.elog_main; |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 809 | u32 tmp; |
| 810 | |
| 811 | /* Stop the parade */ |
Damjan Marion | f553a2c | 2021-03-26 13:45:37 +0100 | [diff] [blame] | 812 | elog_reset_buffer (em); |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 813 | |
| 814 | if (unformat (input, "%d", &tmp)) |
| 815 | { |
| 816 | elog_alloc (em, tmp); |
| 817 | em->n_total_events_disable_limit = ~0; |
| 818 | } |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 819 | else |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 820 | return clib_error_return (0, "Must specify how many events in the ring"); |
| 821 | |
| 822 | vlib_cli_output (vm, "Resized ring and restarted the event logger..."); |
| 823 | return 0; |
| 824 | } |
| 825 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 826 | /* *INDENT-OFF* */ |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 827 | VLIB_CLI_COMMAND (elog_resize_cli, static) = { |
| 828 | .path = "event-logger resize", |
| 829 | .short_help = "event-logger resize <nnn>", |
Dave Barach | bc867c3 | 2020-11-25 10:07:09 -0500 | [diff] [blame] | 830 | .function = elog_resize_command_fn, |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 831 | }; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 832 | /* *INDENT-ON* */ |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 833 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 834 | #endif /* CLIB_UNIX */ |
| 835 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 836 | static void |
| 837 | elog_show_buffer_internal (vlib_main_t * vm, u32 n_events_to_show) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 838 | { |
Damjan Marion | f553a2c | 2021-03-26 13:45:37 +0100 | [diff] [blame] | 839 | elog_main_t *em = &vlib_global_main.elog_main; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 840 | elog_event_t *e, *es; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 841 | f64 dt; |
| 842 | |
| 843 | /* Show events in VLIB time since log clock starts after VLIB clock. */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 844 | dt = (em->init_time.cpu - vm->clib_time.init_cpu_time) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 845 | * vm->clib_time.seconds_per_clock; |
| 846 | |
| 847 | es = elog_peek_events (em); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 848 | vlib_cli_output (vm, "%d of %d events in buffer, logger %s", vec_len (es), |
| 849 | em->event_ring_size, |
| 850 | em->n_total_events < em->n_total_events_disable_limit ? |
| 851 | "running" : "stopped"); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 852 | vec_foreach (e, es) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 853 | { |
| 854 | vlib_cli_output (vm, "%18.9f: %U", |
| 855 | e->time + dt, format_elog_event, em, e); |
| 856 | n_events_to_show--; |
| 857 | if (n_events_to_show == 0) |
| 858 | break; |
| 859 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 860 | vec_free (es); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 861 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 862 | } |
| 863 | |
| 864 | static clib_error_t * |
| 865 | elog_show_buffer (vlib_main_t * vm, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 866 | unformat_input_t * input, vlib_cli_command_t * cmd) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 867 | { |
| 868 | u32 n_events_to_show; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 869 | clib_error_t *error = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 870 | |
| 871 | n_events_to_show = 250; |
| 872 | while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) |
| 873 | { |
| 874 | if (unformat (input, "%d", &n_events_to_show)) |
| 875 | ; |
| 876 | else if (unformat (input, "all")) |
| 877 | n_events_to_show = ~0; |
| 878 | else |
| 879 | return unformat_parse_error (input); |
| 880 | } |
| 881 | elog_show_buffer_internal (vm, n_events_to_show); |
| 882 | return error; |
| 883 | } |
| 884 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 885 | /* *INDENT-OFF* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 886 | VLIB_CLI_COMMAND (elog_show_cli, static) = { |
| 887 | .path = "show event-logger", |
| 888 | .short_help = "Show event logger info", |
| 889 | .function = elog_show_buffer, |
| 890 | }; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 891 | /* *INDENT-ON* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 892 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 893 | void |
| 894 | vlib_gdb_show_event_log (void) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 895 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 896 | elog_show_buffer_internal (vlib_get_main (), (u32) ~ 0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 897 | } |
| 898 | |
Dave Barach | fb6e59d | 2016-03-26 18:45:42 -0400 | [diff] [blame] | 899 | static inline void |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 900 | vlib_elog_main_loop_event (vlib_main_t * vm, |
| 901 | u32 node_index, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 902 | u64 time, u32 n_vectors, u32 is_return) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 903 | { |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 904 | vlib_main_t *evm = vlib_get_first_main (); |
Damjan Marion | f553a2c | 2021-03-26 13:45:37 +0100 | [diff] [blame] | 905 | elog_main_t *em = vlib_get_elog_main (); |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 906 | int enabled = evm->elog_trace_graph_dispatch | |
| 907 | evm->elog_trace_graph_circuit; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 908 | |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 909 | if (PREDICT_FALSE (enabled && n_vectors)) |
| 910 | { |
| 911 | if (PREDICT_FALSE (!elog_is_enabled (em))) |
| 912 | { |
| 913 | evm->elog_trace_graph_dispatch = 0; |
| 914 | evm->elog_trace_graph_circuit = 0; |
| 915 | return; |
| 916 | } |
| 917 | if (PREDICT_TRUE |
| 918 | (evm->elog_trace_graph_dispatch || |
| 919 | (evm->elog_trace_graph_circuit && |
| 920 | node_index == evm->elog_trace_graph_circuit_node_index))) |
| 921 | { |
| 922 | elog_track (em, |
| 923 | /* event type */ |
| 924 | vec_elt_at_index (is_return |
| 925 | ? evm->node_return_elog_event_types |
| 926 | : evm->node_call_elog_event_types, |
| 927 | node_index), |
| 928 | /* track */ |
| 929 | (vm->thread_index ? |
| 930 | &vlib_worker_threads[vm->thread_index].elog_track |
| 931 | : &em->default_track), |
| 932 | /* data to log */ n_vectors); |
| 933 | } |
| 934 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 935 | } |
| 936 | |
Dave Barach | 7bee773 | 2017-10-18 18:48:11 -0400 | [diff] [blame] | 937 | static inline void |
| 938 | add_trajectory_trace (vlib_buffer_t * b, u32 node_index) |
| 939 | { |
| 940 | #if VLIB_BUFFER_TRACE_TRAJECTORY > 0 |
Benoît Ganne | f89bbbe | 2021-03-04 14:31:03 +0100 | [diff] [blame] | 941 | if (PREDICT_FALSE (b->trajectory_nb >= VLIB_BUFFER_TRACE_TRAJECTORY_MAX)) |
| 942 | return; |
| 943 | b->trajectory_trace[b->trajectory_nb] = node_index; |
| 944 | b->trajectory_nb++; |
Dave Barach | 7bee773 | 2017-10-18 18:48:11 -0400 | [diff] [blame] | 945 | #endif |
| 946 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 947 | |
Damjan Marion | 9a332e1 | 2017-03-28 15:11:20 +0200 | [diff] [blame] | 948 | static_always_inline u64 |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 949 | dispatch_node (vlib_main_t * vm, |
| 950 | vlib_node_runtime_t * node, |
| 951 | vlib_node_type_t type, |
| 952 | vlib_node_state_t dispatch_state, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 953 | vlib_frame_t * frame, u64 last_time_stamp) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 954 | { |
| 955 | uword n, v; |
| 956 | u64 t; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 957 | vlib_node_main_t *nm = &vm->node_main; |
| 958 | vlib_next_frame_t *nf; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 959 | |
| 960 | if (CLIB_DEBUG > 0) |
| 961 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 962 | vlib_node_t *n = vlib_get_node (vm, node->node_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 963 | ASSERT (n->type == type); |
| 964 | } |
| 965 | |
| 966 | /* Only non-internal nodes may be disabled. */ |
| 967 | if (type != VLIB_NODE_TYPE_INTERNAL && node->state != dispatch_state) |
| 968 | { |
| 969 | ASSERT (type != VLIB_NODE_TYPE_INTERNAL); |
| 970 | return last_time_stamp; |
| 971 | } |
| 972 | |
| 973 | if ((type == VLIB_NODE_TYPE_PRE_INPUT || type == VLIB_NODE_TYPE_INPUT) |
| 974 | && dispatch_state != VLIB_NODE_STATE_INTERRUPT) |
| 975 | { |
| 976 | u32 c = node->input_main_loops_per_call; |
| 977 | /* Only call node when count reaches zero. */ |
| 978 | if (c) |
| 979 | { |
| 980 | node->input_main_loops_per_call = c - 1; |
| 981 | return last_time_stamp; |
| 982 | } |
| 983 | } |
| 984 | |
| 985 | /* Speculatively prefetch next frames. */ |
| 986 | if (node->n_next_nodes > 0) |
| 987 | { |
| 988 | nf = vec_elt_at_index (nm->next_frames, node->next_frame_index); |
| 989 | CLIB_PREFETCH (nf, 4 * sizeof (nf[0]), WRITE); |
| 990 | } |
| 991 | |
| 992 | vm->cpu_time_last_node_dispatch = last_time_stamp; |
| 993 | |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 994 | vlib_elog_main_loop_event (vm, node->node_index, |
| 995 | last_time_stamp, frame ? frame->n_vectors : 0, |
| 996 | /* is_after */ 0); |
| 997 | |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 998 | vlib_node_runtime_perf_counter (vm, node, frame, 0, last_time_stamp, |
| 999 | VLIB_NODE_RUNTIME_PERF_BEFORE); |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1000 | |
| 1001 | /* |
| 1002 | * Turn this on if you run into |
| 1003 | * "bad monkey" contexts, and you want to know exactly |
| 1004 | * which nodes they've visited... See ixge.c... |
| 1005 | */ |
| 1006 | if (VLIB_BUFFER_TRACE_TRAJECTORY && frame) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1007 | { |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1008 | int i; |
| 1009 | u32 *from; |
| 1010 | from = vlib_frame_vector_args (frame); |
| 1011 | for (i = 0; i < frame->n_vectors; i++) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1012 | { |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1013 | vlib_buffer_t *b = vlib_get_buffer (vm, from[i]); |
| 1014 | add_trajectory_trace (b, node->node_index); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1015 | } |
Damjan Marion | 8b60fb0 | 2020-11-27 20:15:17 +0100 | [diff] [blame] | 1016 | if (PREDICT_TRUE (vm->dispatch_wrapper_fn == 0)) |
| 1017 | n = node->function (vm, node, frame); |
| 1018 | else |
| 1019 | n = vm->dispatch_wrapper_fn (vm, node, frame); |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1020 | } |
| 1021 | else |
| 1022 | { |
Damjan Marion | 8b60fb0 | 2020-11-27 20:15:17 +0100 | [diff] [blame] | 1023 | if (PREDICT_TRUE (vm->dispatch_wrapper_fn == 0)) |
| 1024 | n = node->function (vm, node, frame); |
| 1025 | else |
| 1026 | n = vm->dispatch_wrapper_fn (vm, node, frame); |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1027 | } |
| 1028 | |
| 1029 | t = clib_cpu_time_now (); |
| 1030 | |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 1031 | vlib_node_runtime_perf_counter (vm, node, frame, n, t, |
| 1032 | VLIB_NODE_RUNTIME_PERF_AFTER); |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1033 | |
| 1034 | vlib_elog_main_loop_event (vm, node->node_index, t, n, 1 /* is_after */ ); |
| 1035 | |
| 1036 | vm->main_loop_vectors_processed += n; |
| 1037 | vm->main_loop_nodes_processed += n > 0; |
| 1038 | |
| 1039 | v = vlib_node_runtime_update_stats (vm, node, |
| 1040 | /* n_calls */ 1, |
| 1041 | /* n_vectors */ n, |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 1042 | /* n_clocks */ t - last_time_stamp); |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1043 | |
Florin Coras | 982e44f | 2021-03-19 13:12:41 -0700 | [diff] [blame] | 1044 | /* When in adaptive mode and vector rate crosses threshold switch to |
| 1045 | polling mode and vice versa. */ |
| 1046 | if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_ADAPTIVE_MODE)) |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1047 | { |
| 1048 | /* *INDENT-OFF* */ |
| 1049 | ELOG_TYPE_DECLARE (e) = |
| 1050 | { |
| 1051 | .function = (char *) __FUNCTION__, |
| 1052 | .format = "%s vector length %d, switching to %s", |
| 1053 | .format_args = "T4i4t4", |
| 1054 | .n_enum_strings = 2, |
| 1055 | .enum_strings = { |
| 1056 | "interrupt", "polling", |
| 1057 | }, |
| 1058 | }; |
| 1059 | /* *INDENT-ON* */ |
| 1060 | struct |
| 1061 | { |
| 1062 | u32 node_name, vector_length, is_polling; |
| 1063 | } *ed; |
| 1064 | |
| 1065 | if ((dispatch_state == VLIB_NODE_STATE_INTERRUPT |
| 1066 | && v >= nm->polling_threshold_vector_length) && |
| 1067 | !(node->flags & |
| 1068 | VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE)) |
Dave Barach | 3ae2873 | 2018-11-16 17:19:00 -0500 | [diff] [blame] | 1069 | { |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1070 | vlib_node_t *n = vlib_get_node (vm, node->node_index); |
| 1071 | n->state = VLIB_NODE_STATE_POLLING; |
| 1072 | node->state = VLIB_NODE_STATE_POLLING; |
| 1073 | node->flags &= |
| 1074 | ~VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE; |
| 1075 | node->flags |= VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE; |
| 1076 | nm->input_node_counts_by_state[VLIB_NODE_STATE_INTERRUPT] -= 1; |
| 1077 | nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] += 1; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1078 | |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 1079 | if (PREDICT_FALSE ( |
| 1080 | vlib_get_first_main ()->elog_trace_graph_dispatch)) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1081 | { |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1082 | vlib_worker_thread_t *w = vlib_worker_threads |
| 1083 | + vm->thread_index; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1084 | |
Steven | 6aa75af | 2017-02-24 10:03:22 -0800 | [diff] [blame] | 1085 | ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e, |
| 1086 | w->elog_track); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1087 | ed->node_name = n->name_elog_string; |
| 1088 | ed->vector_length = v; |
| 1089 | ed->is_polling = 1; |
| 1090 | } |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1091 | } |
| 1092 | else if (dispatch_state == VLIB_NODE_STATE_POLLING |
| 1093 | && v <= nm->interrupt_threshold_vector_length) |
| 1094 | { |
| 1095 | vlib_node_t *n = vlib_get_node (vm, node->node_index); |
| 1096 | if (node->flags & |
| 1097 | VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1098 | { |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1099 | /* Switch to interrupt mode after dispatch in polling one more time. |
| 1100 | This allows driver to re-enable interrupts. */ |
| 1101 | n->state = VLIB_NODE_STATE_INTERRUPT; |
| 1102 | node->state = VLIB_NODE_STATE_INTERRUPT; |
| 1103 | node->flags &= |
| 1104 | ~VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE; |
| 1105 | nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] -= 1; |
| 1106 | nm->input_node_counts_by_state[VLIB_NODE_STATE_INTERRUPT] += 1; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1107 | |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1108 | } |
| 1109 | else |
| 1110 | { |
| 1111 | vlib_worker_thread_t *w = vlib_worker_threads |
| 1112 | + vm->thread_index; |
| 1113 | node->flags |= |
| 1114 | VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE; |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 1115 | if (PREDICT_FALSE ( |
| 1116 | vlib_get_first_main ()->elog_trace_graph_dispatch)) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1117 | { |
Steven | 6aa75af | 2017-02-24 10:03:22 -0800 | [diff] [blame] | 1118 | ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e, |
| 1119 | w->elog_track); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1120 | ed->node_name = n->name_elog_string; |
| 1121 | ed->vector_length = v; |
| 1122 | ed->is_polling = 0; |
| 1123 | } |
| 1124 | } |
| 1125 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1126 | } |
| 1127 | |
| 1128 | return t; |
| 1129 | } |
| 1130 | |
Damjan Marion | 9a332e1 | 2017-03-28 15:11:20 +0200 | [diff] [blame] | 1131 | static u64 |
Dave Barach | a626999 | 2017-06-07 08:18:49 -0400 | [diff] [blame] | 1132 | dispatch_pending_node (vlib_main_t * vm, uword pending_frame_index, |
| 1133 | u64 last_time_stamp) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1134 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1135 | vlib_node_main_t *nm = &vm->node_main; |
| 1136 | vlib_frame_t *f; |
Dave Barach | 11fb09e | 2020-08-06 12:10:09 -0400 | [diff] [blame] | 1137 | vlib_next_frame_t *nf, nf_placeholder; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1138 | vlib_node_runtime_t *n; |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 1139 | vlib_frame_t *restore_frame; |
Dave Barach | a626999 | 2017-06-07 08:18:49 -0400 | [diff] [blame] | 1140 | vlib_pending_frame_t *p; |
| 1141 | |
| 1142 | /* See comment below about dangling references to nm->pending_frames */ |
| 1143 | p = nm->pending_frames + pending_frame_index; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1144 | |
| 1145 | n = vec_elt_at_index (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL], |
| 1146 | p->node_runtime_index); |
| 1147 | |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 1148 | f = vlib_get_frame (vm, p->frame); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1149 | if (p->next_frame_index == VLIB_PENDING_FRAME_NO_NEXT_FRAME) |
| 1150 | { |
Dave Barach | 11fb09e | 2020-08-06 12:10:09 -0400 | [diff] [blame] | 1151 | /* No next frame: so use placeholder on stack. */ |
| 1152 | nf = &nf_placeholder; |
Damjan Marion | 633b6fd | 2018-09-14 14:38:53 +0200 | [diff] [blame] | 1153 | nf->flags = f->frame_flags & VLIB_NODE_FLAG_TRACE; |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 1154 | nf->frame = NULL; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1155 | } |
| 1156 | else |
| 1157 | nf = vec_elt_at_index (nm->next_frames, p->next_frame_index); |
| 1158 | |
Damjan Marion | 633b6fd | 2018-09-14 14:38:53 +0200 | [diff] [blame] | 1159 | ASSERT (f->frame_flags & VLIB_FRAME_IS_ALLOCATED); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1160 | |
| 1161 | /* Force allocation of new frame while current frame is being |
| 1162 | dispatched. */ |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 1163 | restore_frame = NULL; |
| 1164 | if (nf->frame == p->frame) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1165 | { |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 1166 | nf->frame = NULL; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1167 | nf->flags &= ~VLIB_FRAME_IS_ALLOCATED; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1168 | if (!(n->flags & VLIB_NODE_FLAG_FRAME_NO_FREE_AFTER_DISPATCH)) |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 1169 | restore_frame = p->frame; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1170 | } |
| 1171 | |
| 1172 | /* Frame must be pending. */ |
Damjan Marion | 633b6fd | 2018-09-14 14:38:53 +0200 | [diff] [blame] | 1173 | ASSERT (f->frame_flags & VLIB_FRAME_PENDING); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1174 | ASSERT (f->n_vectors > 0); |
| 1175 | |
| 1176 | /* Copy trace flag from next frame to node. |
| 1177 | Trace flag indicates that at least one vector in the dispatched |
| 1178 | frame is traced. */ |
| 1179 | n->flags &= ~VLIB_NODE_FLAG_TRACE; |
| 1180 | n->flags |= (nf->flags & VLIB_FRAME_TRACE) ? VLIB_NODE_FLAG_TRACE : 0; |
| 1181 | nf->flags &= ~VLIB_FRAME_TRACE; |
| 1182 | |
| 1183 | last_time_stamp = dispatch_node (vm, n, |
| 1184 | VLIB_NODE_TYPE_INTERNAL, |
| 1185 | VLIB_NODE_STATE_POLLING, |
| 1186 | f, last_time_stamp); |
Dave Barach | a8df85c | 2019-10-01 13:34:23 -0400 | [diff] [blame] | 1187 | /* Internal node vector-rate accounting, for summary stats */ |
| 1188 | vm->internal_node_vectors += f->n_vectors; |
| 1189 | vm->internal_node_calls++; |
| 1190 | vm->internal_node_last_vectors_per_main_loop = |
| 1191 | (f->n_vectors > vm->internal_node_last_vectors_per_main_loop) ? |
| 1192 | f->n_vectors : vm->internal_node_last_vectors_per_main_loop; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1193 | |
Damjan Marion | 296988d | 2019-02-21 20:24:54 +0100 | [diff] [blame] | 1194 | f->frame_flags &= ~(VLIB_FRAME_PENDING | VLIB_FRAME_NO_APPEND); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1195 | |
| 1196 | /* Frame is ready to be used again, so restore it. */ |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 1197 | if (restore_frame != NULL) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1198 | { |
Dave Barach | a626999 | 2017-06-07 08:18:49 -0400 | [diff] [blame] | 1199 | /* |
| 1200 | * We musn't restore a frame that is flagged to be freed. This |
| 1201 | * shouldn't happen since frames to be freed post dispatch are |
| 1202 | * those used when the to-node frame becomes full i.e. they form a |
| 1203 | * sort of queue of frames to a single node. If we get here then |
| 1204 | * the to-node frame and the pending frame *were* the same, and so |
| 1205 | * we removed the to-node frame. Therefore this frame is no |
| 1206 | * longer part of the queue for that node and hence it cannot be |
| 1207 | * it's overspill. |
Neale Ranns | 8817061 | 2016-11-22 08:29:51 +0000 | [diff] [blame] | 1208 | */ |
Damjan Marion | 633b6fd | 2018-09-14 14:38:53 +0200 | [diff] [blame] | 1209 | ASSERT (!(f->frame_flags & VLIB_FRAME_FREE_AFTER_DISPATCH)); |
Neale Ranns | 8817061 | 2016-11-22 08:29:51 +0000 | [diff] [blame] | 1210 | |
Dave Barach | a626999 | 2017-06-07 08:18:49 -0400 | [diff] [blame] | 1211 | /* |
| 1212 | * NB: dispatching node n can result in the creation and scheduling |
| 1213 | * of new frames, and hence in the reallocation of nm->pending_frames. |
| 1214 | * Recompute p, or no supper. This was broken for more than 10 years. |
| 1215 | */ |
| 1216 | p = nm->pending_frames + pending_frame_index; |
| 1217 | |
| 1218 | /* |
| 1219 | * p->next_frame_index can change during node dispatch if node |
| 1220 | * function decides to change graph hook up. |
| 1221 | */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1222 | nf = vec_elt_at_index (nm->next_frames, p->next_frame_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1223 | nf->flags |= VLIB_FRAME_IS_ALLOCATED; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1224 | |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 1225 | if (NULL == nf->frame) |
Neale Ranns | 8817061 | 2016-11-22 08:29:51 +0000 | [diff] [blame] | 1226 | { |
| 1227 | /* no new frame has been assigned to this node, use the saved one */ |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 1228 | nf->frame = restore_frame; |
Neale Ranns | 8817061 | 2016-11-22 08:29:51 +0000 | [diff] [blame] | 1229 | f->n_vectors = 0; |
| 1230 | } |
| 1231 | else |
| 1232 | { |
| 1233 | /* The node has gained a frame, implying packets from the current frame |
| 1234 | were re-queued to this same node. we don't need the saved one |
| 1235 | anymore */ |
| 1236 | vlib_frame_free (vm, n, f); |
| 1237 | } |
| 1238 | } |
| 1239 | else |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1240 | { |
Damjan Marion | 633b6fd | 2018-09-14 14:38:53 +0200 | [diff] [blame] | 1241 | if (f->frame_flags & VLIB_FRAME_FREE_AFTER_DISPATCH) |
Neale Ranns | 8817061 | 2016-11-22 08:29:51 +0000 | [diff] [blame] | 1242 | { |
| 1243 | ASSERT (!(n->flags & VLIB_NODE_FLAG_FRAME_NO_FREE_AFTER_DISPATCH)); |
| 1244 | vlib_frame_free (vm, n, f); |
| 1245 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1246 | } |
| 1247 | |
| 1248 | return last_time_stamp; |
| 1249 | } |
| 1250 | |
| 1251 | always_inline uword |
| 1252 | vlib_process_stack_is_valid (vlib_process_t * p) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1253 | { |
| 1254 | return p->stack[0] == VLIB_PROCESS_STACK_MAGIC; |
| 1255 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1256 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1257 | typedef struct |
| 1258 | { |
| 1259 | vlib_main_t *vm; |
| 1260 | vlib_process_t *process; |
| 1261 | vlib_frame_t *frame; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1262 | } vlib_process_bootstrap_args_t; |
| 1263 | |
| 1264 | /* Called in process stack. */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1265 | static uword |
| 1266 | vlib_process_bootstrap (uword _a) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1267 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1268 | vlib_process_bootstrap_args_t *a; |
| 1269 | vlib_main_t *vm; |
| 1270 | vlib_node_runtime_t *node; |
| 1271 | vlib_frame_t *f; |
| 1272 | vlib_process_t *p; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1273 | uword n; |
| 1274 | |
| 1275 | a = uword_to_pointer (_a, vlib_process_bootstrap_args_t *); |
| 1276 | |
| 1277 | vm = a->vm; |
| 1278 | p = a->process; |
Damjan Marion | cea4652 | 2020-05-21 16:47:05 +0200 | [diff] [blame] | 1279 | vlib_process_finish_switch_stack (vm); |
| 1280 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1281 | f = a->frame; |
| 1282 | node = &p->node_runtime; |
| 1283 | |
| 1284 | n = node->function (vm, node, f); |
| 1285 | |
| 1286 | ASSERT (vlib_process_stack_is_valid (p)); |
| 1287 | |
Damjan Marion | cea4652 | 2020-05-21 16:47:05 +0200 | [diff] [blame] | 1288 | vlib_process_start_switch_stack (vm, 0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1289 | clib_longjmp (&p->return_longjmp, n); |
| 1290 | |
| 1291 | return n; |
| 1292 | } |
| 1293 | |
| 1294 | /* Called in main stack. */ |
| 1295 | static_always_inline uword |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1296 | vlib_process_startup (vlib_main_t * vm, vlib_process_t * p, vlib_frame_t * f) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1297 | { |
| 1298 | vlib_process_bootstrap_args_t a; |
| 1299 | uword r; |
| 1300 | |
| 1301 | a.vm = vm; |
| 1302 | a.process = p; |
| 1303 | a.frame = f; |
| 1304 | |
| 1305 | r = clib_setjmp (&p->return_longjmp, VLIB_PROCESS_RETURN_LONGJMP_RETURN); |
| 1306 | if (r == VLIB_PROCESS_RETURN_LONGJMP_RETURN) |
Damjan Marion | cea4652 | 2020-05-21 16:47:05 +0200 | [diff] [blame] | 1307 | { |
| 1308 | vlib_process_start_switch_stack (vm, p); |
| 1309 | r = clib_calljmp (vlib_process_bootstrap, pointer_to_uword (&a), |
| 1310 | (void *) p->stack + (1 << p->log2_n_stack_bytes)); |
| 1311 | } |
| 1312 | else |
| 1313 | vlib_process_finish_switch_stack (vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1314 | |
| 1315 | return r; |
| 1316 | } |
| 1317 | |
| 1318 | static_always_inline uword |
Damjan Marion | cea4652 | 2020-05-21 16:47:05 +0200 | [diff] [blame] | 1319 | vlib_process_resume (vlib_main_t * vm, vlib_process_t * p) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1320 | { |
| 1321 | uword r; |
| 1322 | p->flags &= ~(VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK |
| 1323 | | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT |
| 1324 | | VLIB_PROCESS_RESUME_PENDING); |
| 1325 | r = clib_setjmp (&p->return_longjmp, VLIB_PROCESS_RETURN_LONGJMP_RETURN); |
| 1326 | if (r == VLIB_PROCESS_RETURN_LONGJMP_RETURN) |
Damjan Marion | cea4652 | 2020-05-21 16:47:05 +0200 | [diff] [blame] | 1327 | { |
| 1328 | vlib_process_start_switch_stack (vm, p); |
| 1329 | clib_longjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_RESUME); |
| 1330 | } |
| 1331 | else |
| 1332 | vlib_process_finish_switch_stack (vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1333 | return r; |
| 1334 | } |
| 1335 | |
| 1336 | static u64 |
| 1337 | dispatch_process (vlib_main_t * vm, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1338 | vlib_process_t * p, vlib_frame_t * f, u64 last_time_stamp) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1339 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1340 | vlib_node_main_t *nm = &vm->node_main; |
| 1341 | vlib_node_runtime_t *node_runtime = &p->node_runtime; |
| 1342 | vlib_node_t *node = vlib_get_node (vm, node_runtime->node_index); |
Florin Coras | fd542f1 | 2018-05-16 19:28:24 -0700 | [diff] [blame] | 1343 | u32 old_process_index; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1344 | u64 t; |
| 1345 | uword n_vectors, is_suspend; |
| 1346 | |
| 1347 | if (node->state != VLIB_NODE_STATE_POLLING |
| 1348 | || (p->flags & (VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK |
| 1349 | | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT))) |
| 1350 | return last_time_stamp; |
| 1351 | |
| 1352 | p->flags |= VLIB_PROCESS_IS_RUNNING; |
| 1353 | |
| 1354 | t = last_time_stamp; |
| 1355 | vlib_elog_main_loop_event (vm, node_runtime->node_index, t, |
| 1356 | f ? f->n_vectors : 0, /* is_after */ 0); |
| 1357 | |
| 1358 | /* Save away current process for suspend. */ |
Florin Coras | fd542f1 | 2018-05-16 19:28:24 -0700 | [diff] [blame] | 1359 | old_process_index = nm->current_process_index; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1360 | nm->current_process_index = node->runtime_index; |
| 1361 | |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 1362 | vlib_node_runtime_perf_counter (vm, node_runtime, f, 0, last_time_stamp, |
| 1363 | VLIB_NODE_RUNTIME_PERF_BEFORE); |
| 1364 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1365 | n_vectors = vlib_process_startup (vm, p, f); |
| 1366 | |
Florin Coras | fd542f1 | 2018-05-16 19:28:24 -0700 | [diff] [blame] | 1367 | nm->current_process_index = old_process_index; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1368 | |
| 1369 | ASSERT (n_vectors != VLIB_PROCESS_RETURN_LONGJMP_RETURN); |
| 1370 | is_suspend = n_vectors == VLIB_PROCESS_RETURN_LONGJMP_SUSPEND; |
| 1371 | if (is_suspend) |
| 1372 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1373 | vlib_pending_frame_t *pf; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1374 | |
| 1375 | n_vectors = 0; |
| 1376 | pool_get (nm->suspended_process_frames, pf); |
| 1377 | pf->node_runtime_index = node->runtime_index; |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 1378 | pf->frame = f; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1379 | pf->next_frame_index = ~0; |
| 1380 | |
| 1381 | p->n_suspends += 1; |
| 1382 | p->suspended_process_frame_index = pf - nm->suspended_process_frames; |
| 1383 | |
| 1384 | if (p->flags & VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK) |
Dave Barach | 5c20a01 | 2017-06-13 08:48:31 -0400 | [diff] [blame] | 1385 | { |
| 1386 | TWT (tw_timer_wheel) * tw = |
| 1387 | (TWT (tw_timer_wheel) *) nm->timing_wheel; |
| 1388 | p->stop_timer_handle = |
| 1389 | TW (tw_timer_start) (tw, |
| 1390 | vlib_timing_wheel_data_set_suspended_process |
| 1391 | (node->runtime_index) /* [sic] pool idex */ , |
| 1392 | 0 /* timer_id */ , |
| 1393 | p->resume_clock_interval); |
| 1394 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1395 | } |
| 1396 | else |
| 1397 | p->flags &= ~VLIB_PROCESS_IS_RUNNING; |
| 1398 | |
| 1399 | t = clib_cpu_time_now (); |
| 1400 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1401 | vlib_elog_main_loop_event (vm, node_runtime->node_index, t, is_suspend, |
| 1402 | /* is_after */ 1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1403 | |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 1404 | vlib_node_runtime_perf_counter (vm, node_runtime, f, n_vectors, t, |
| 1405 | VLIB_NODE_RUNTIME_PERF_AFTER); |
| 1406 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1407 | vlib_process_update_stats (vm, p, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1408 | /* n_calls */ !is_suspend, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1409 | /* n_vectors */ n_vectors, |
Dave Barach | ec595ef | 2019-01-24 10:34:24 -0500 | [diff] [blame] | 1410 | /* n_clocks */ t - last_time_stamp); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1411 | |
| 1412 | return t; |
| 1413 | } |
| 1414 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1415 | void |
| 1416 | vlib_start_process (vlib_main_t * vm, uword process_index) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1417 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1418 | vlib_node_main_t *nm = &vm->node_main; |
| 1419 | vlib_process_t *p = vec_elt (nm->processes, process_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1420 | dispatch_process (vm, p, /* frame */ 0, /* cpu_time_now */ 0); |
| 1421 | } |
| 1422 | |
| 1423 | static u64 |
| 1424 | dispatch_suspended_process (vlib_main_t * vm, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1425 | uword process_index, u64 last_time_stamp) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1426 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1427 | vlib_node_main_t *nm = &vm->node_main; |
| 1428 | vlib_node_runtime_t *node_runtime; |
| 1429 | vlib_node_t *node; |
| 1430 | vlib_frame_t *f; |
| 1431 | vlib_process_t *p; |
| 1432 | vlib_pending_frame_t *pf; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1433 | u64 t, n_vectors, is_suspend; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1434 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1435 | t = last_time_stamp; |
| 1436 | |
| 1437 | p = vec_elt (nm->processes, process_index); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1438 | if (PREDICT_FALSE (!(p->flags & VLIB_PROCESS_IS_RUNNING))) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1439 | return last_time_stamp; |
| 1440 | |
| 1441 | ASSERT (p->flags & (VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK |
| 1442 | | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT)); |
| 1443 | |
Florin Coras | 221d6f1 | 2018-11-07 20:46:38 -0800 | [diff] [blame] | 1444 | pf = pool_elt_at_index (nm->suspended_process_frames, |
| 1445 | p->suspended_process_frame_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1446 | |
| 1447 | node_runtime = &p->node_runtime; |
| 1448 | node = vlib_get_node (vm, node_runtime->node_index); |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 1449 | f = pf->frame; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1450 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1451 | vlib_elog_main_loop_event (vm, node_runtime->node_index, t, |
| 1452 | f ? f->n_vectors : 0, /* is_after */ 0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1453 | |
| 1454 | /* Save away current process for suspend. */ |
| 1455 | nm->current_process_index = node->runtime_index; |
| 1456 | |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 1457 | vlib_node_runtime_perf_counter (vm, node_runtime, f, 0, last_time_stamp, |
| 1458 | VLIB_NODE_RUNTIME_PERF_BEFORE); |
| 1459 | |
Damjan Marion | cea4652 | 2020-05-21 16:47:05 +0200 | [diff] [blame] | 1460 | n_vectors = vlib_process_resume (vm, p); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1461 | t = clib_cpu_time_now (); |
| 1462 | |
| 1463 | nm->current_process_index = ~0; |
| 1464 | |
| 1465 | is_suspend = n_vectors == VLIB_PROCESS_RETURN_LONGJMP_SUSPEND; |
| 1466 | if (is_suspend) |
| 1467 | { |
| 1468 | /* Suspend it again. */ |
| 1469 | n_vectors = 0; |
| 1470 | p->n_suspends += 1; |
| 1471 | if (p->flags & VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK) |
Dave Barach | 5c20a01 | 2017-06-13 08:48:31 -0400 | [diff] [blame] | 1472 | { |
| 1473 | p->stop_timer_handle = |
| 1474 | TW (tw_timer_start) ((TWT (tw_timer_wheel) *) nm->timing_wheel, |
| 1475 | vlib_timing_wheel_data_set_suspended_process |
| 1476 | (node->runtime_index) /* [sic] pool idex */ , |
| 1477 | 0 /* timer_id */ , |
| 1478 | p->resume_clock_interval); |
| 1479 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1480 | } |
| 1481 | else |
| 1482 | { |
| 1483 | p->flags &= ~VLIB_PROCESS_IS_RUNNING; |
Florin Coras | 221d6f1 | 2018-11-07 20:46:38 -0800 | [diff] [blame] | 1484 | pool_put_index (nm->suspended_process_frames, |
| 1485 | p->suspended_process_frame_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1486 | p->suspended_process_frame_index = ~0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1487 | } |
| 1488 | |
| 1489 | t = clib_cpu_time_now (); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1490 | vlib_elog_main_loop_event (vm, node_runtime->node_index, t, !is_suspend, |
| 1491 | /* is_after */ 1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1492 | |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 1493 | vlib_node_runtime_perf_counter (vm, node_runtime, f, n_vectors, t, |
| 1494 | VLIB_NODE_RUNTIME_PERF_AFTER); |
| 1495 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1496 | vlib_process_update_stats (vm, p, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1497 | /* n_calls */ !is_suspend, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1498 | /* n_vectors */ n_vectors, |
Dave Barach | ec595ef | 2019-01-24 10:34:24 -0500 | [diff] [blame] | 1499 | /* n_clocks */ t - last_time_stamp); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1500 | |
| 1501 | return t; |
| 1502 | } |
| 1503 | |
Dave Barach | 2877eee | 2017-12-15 12:22:57 -0500 | [diff] [blame] | 1504 | void vl_api_send_pending_rpc_requests (vlib_main_t *) __attribute__ ((weak)); |
| 1505 | void |
| 1506 | vl_api_send_pending_rpc_requests (vlib_main_t * vm) |
| 1507 | { |
| 1508 | } |
| 1509 | |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1510 | static_always_inline void |
| 1511 | vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1512 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1513 | vlib_node_main_t *nm = &vm->node_main; |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1514 | vlib_thread_main_t *tm = vlib_get_thread_main (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1515 | uword i; |
| 1516 | u64 cpu_time_now; |
Dave Barach | 000a029 | 2020-02-17 17:07:12 -0500 | [diff] [blame] | 1517 | f64 now; |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1518 | vlib_frame_queue_main_t *fqm; |
Dave Barach | 80965f5 | 2019-03-11 09:57:38 -0400 | [diff] [blame] | 1519 | u32 frame_queue_check_counter = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1520 | |
| 1521 | /* Initialize pending node vector. */ |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1522 | if (is_main) |
| 1523 | { |
| 1524 | vec_resize (nm->pending_frames, 32); |
| 1525 | _vec_len (nm->pending_frames) = 0; |
| 1526 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1527 | |
| 1528 | /* Mark time of main loop start. */ |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1529 | if (is_main) |
| 1530 | { |
| 1531 | cpu_time_now = vm->clib_time.last_cpu_time; |
| 1532 | vm->cpu_time_main_loop_start = cpu_time_now; |
| 1533 | } |
| 1534 | else |
| 1535 | cpu_time_now = clib_cpu_time_now (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1536 | |
Damjan Marion | 2c2b640 | 2017-03-28 14:16:15 +0200 | [diff] [blame] | 1537 | /* Pre-allocate interupt runtime indices and lock. */ |
Damjan Marion | 9410053 | 2020-11-06 23:25:57 +0100 | [diff] [blame] | 1538 | vec_alloc_aligned (nm->pending_interrupts, 1, CLIB_CACHE_LINE_BYTES); |
Damjan Marion | 2c2b640 | 2017-03-28 14:16:15 +0200 | [diff] [blame] | 1539 | |
| 1540 | /* Pre-allocate expired nodes. */ |
Steven | 7312cc7 | 2017-03-15 21:18:55 -0700 | [diff] [blame] | 1541 | if (!nm->polling_threshold_vector_length) |
| 1542 | nm->polling_threshold_vector_length = 10; |
| 1543 | if (!nm->interrupt_threshold_vector_length) |
| 1544 | nm->interrupt_threshold_vector_length = 5; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1545 | |
Damjan Marion | 29c0b33 | 2019-01-28 13:41:27 +0100 | [diff] [blame] | 1546 | vm->cpu_id = clib_get_current_cpu_id (); |
| 1547 | vm->numa_node = clib_get_current_numa_node (); |
Florin Coras | 4c95995 | 2020-02-09 18:09:31 +0000 | [diff] [blame] | 1548 | os_set_numa_index (vm->numa_node); |
Damjan Marion | 29c0b33 | 2019-01-28 13:41:27 +0100 | [diff] [blame] | 1549 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1550 | /* Start all processes. */ |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1551 | if (is_main) |
| 1552 | { |
| 1553 | uword i; |
Dave Barach | c602b38 | 2019-06-03 19:48:22 -0400 | [diff] [blame] | 1554 | |
| 1555 | /* |
| 1556 | * Perform an initial barrier sync. Pays no attention to |
| 1557 | * the barrier sync hold-down timer scheme, which won't work |
| 1558 | * at this point in time. |
| 1559 | */ |
| 1560 | vlib_worker_thread_initial_barrier_sync_and_release (vm); |
| 1561 | |
Steven | f3b5364 | 2017-05-01 14:03:02 -0700 | [diff] [blame] | 1562 | nm->current_process_index = ~0; |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1563 | for (i = 0; i < vec_len (nm->processes); i++) |
| 1564 | cpu_time_now = dispatch_process (vm, nm->processes[i], /* frame */ 0, |
| 1565 | cpu_time_now); |
| 1566 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1567 | |
| 1568 | while (1) |
| 1569 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1570 | vlib_node_runtime_t *n; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1571 | |
Dave Barach | 2877eee | 2017-12-15 12:22:57 -0500 | [diff] [blame] | 1572 | if (PREDICT_FALSE (_vec_len (vm->pending_rpc_requests) > 0)) |
Dave Barach | f6c68d7 | 2018-11-01 08:12:52 -0400 | [diff] [blame] | 1573 | { |
| 1574 | if (!is_main) |
| 1575 | vl_api_send_pending_rpc_requests (vm); |
| 1576 | } |
Dave Barach | 2877eee | 2017-12-15 12:22:57 -0500 | [diff] [blame] | 1577 | |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1578 | if (!is_main) |
Damjan Marion | f6e6c78 | 2020-09-17 09:54:07 +0200 | [diff] [blame] | 1579 | vlib_worker_thread_barrier_check (); |
| 1580 | |
| 1581 | if (PREDICT_FALSE (vm->check_frame_queues + frame_queue_check_counter)) |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1582 | { |
Damjan Marion | f6e6c78 | 2020-09-17 09:54:07 +0200 | [diff] [blame] | 1583 | u32 processed = 0; |
Damjan Marion | eee099e | 2021-05-01 14:56:13 +0200 | [diff] [blame] | 1584 | vlib_frame_queue_dequeue_fn_t *fn = |
| 1585 | vlib_buffer_func_main.frame_queue_dequeue_fn; |
Damjan Marion | f6e6c78 | 2020-09-17 09:54:07 +0200 | [diff] [blame] | 1586 | |
| 1587 | if (vm->check_frame_queues) |
Dave Barach | 80965f5 | 2019-03-11 09:57:38 -0400 | [diff] [blame] | 1588 | { |
Damjan Marion | f6e6c78 | 2020-09-17 09:54:07 +0200 | [diff] [blame] | 1589 | frame_queue_check_counter = 100; |
| 1590 | vm->check_frame_queues = 0; |
Dave Barach | 80965f5 | 2019-03-11 09:57:38 -0400 | [diff] [blame] | 1591 | } |
Damjan Marion | f6e6c78 | 2020-09-17 09:54:07 +0200 | [diff] [blame] | 1592 | |
| 1593 | vec_foreach (fqm, tm->frame_queue_mains) |
Damjan Marion | eee099e | 2021-05-01 14:56:13 +0200 | [diff] [blame] | 1594 | processed += (fn) (vm, fqm); |
Damjan Marion | f6e6c78 | 2020-09-17 09:54:07 +0200 | [diff] [blame] | 1595 | |
| 1596 | /* No handoff queue work found? */ |
| 1597 | if (processed) |
| 1598 | frame_queue_check_counter = 100; |
| 1599 | else |
| 1600 | frame_queue_check_counter--; |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1601 | } |
| 1602 | |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 1603 | if (PREDICT_FALSE (vec_len (vm->worker_thread_main_loop_callbacks))) |
| 1604 | clib_call_callbacks (vm->worker_thread_main_loop_callbacks, vm, |
| 1605 | cpu_time_now); |
| 1606 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1607 | /* Process pre-input nodes. */ |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 1608 | cpu_time_now = clib_cpu_time_now (); |
Damjan Marion | ceab788 | 2018-01-19 20:56:12 +0100 | [diff] [blame] | 1609 | vec_foreach (n, nm->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT]) |
| 1610 | cpu_time_now = dispatch_node (vm, n, |
| 1611 | VLIB_NODE_TYPE_PRE_INPUT, |
| 1612 | VLIB_NODE_STATE_POLLING, |
| 1613 | /* frame */ 0, |
| 1614 | cpu_time_now); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1615 | |
| 1616 | /* Next process input nodes. */ |
| 1617 | vec_foreach (n, nm->nodes_by_type[VLIB_NODE_TYPE_INPUT]) |
| 1618 | cpu_time_now = dispatch_node (vm, n, |
| 1619 | VLIB_NODE_TYPE_INPUT, |
| 1620 | VLIB_NODE_STATE_POLLING, |
| 1621 | /* frame */ 0, |
| 1622 | cpu_time_now); |
| 1623 | |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1624 | if (PREDICT_TRUE (is_main && vm->queue_signal_pending == 0)) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1625 | vm->queue_signal_callback (vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1626 | |
Damjan Marion | 9410053 | 2020-11-06 23:25:57 +0100 | [diff] [blame] | 1627 | if (__atomic_load_n (nm->pending_interrupts, __ATOMIC_ACQUIRE)) |
Damjan Marion | 0b31630 | 2020-09-09 18:55:16 +0200 | [diff] [blame] | 1628 | { |
Damjan Marion | 9410053 | 2020-11-06 23:25:57 +0100 | [diff] [blame] | 1629 | int int_num = -1; |
| 1630 | *nm->pending_interrupts = 0; |
Dave Barach | d47c509 | 2018-01-19 13:09:20 -0500 | [diff] [blame] | 1631 | |
Damjan Marion | 9410053 | 2020-11-06 23:25:57 +0100 | [diff] [blame] | 1632 | while ((int_num = |
| 1633 | clib_interrupt_get_next (nm->interrupts, int_num)) != -1) |
| 1634 | { |
| 1635 | vlib_node_runtime_t *n; |
| 1636 | clib_interrupt_clear (nm->interrupts, int_num); |
| 1637 | n = vec_elt_at_index (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT], |
| 1638 | int_num); |
| 1639 | cpu_time_now = dispatch_node (vm, n, VLIB_NODE_TYPE_INPUT, |
| 1640 | VLIB_NODE_STATE_INTERRUPT, |
| 1641 | /* frame */ 0, cpu_time_now); |
| 1642 | } |
Damjan Marion | 1033b49 | 2020-06-03 12:20:41 +0200 | [diff] [blame] | 1643 | } |
| 1644 | |
Dave Barach | e324898 | 2018-08-14 13:47:58 -0400 | [diff] [blame] | 1645 | /* Input nodes may have added work to the pending vector. |
| 1646 | Process pending vector until there is nothing left. |
| 1647 | All pending vectors will be processed from input -> output. */ |
| 1648 | for (i = 0; i < _vec_len (nm->pending_frames); i++) |
| 1649 | cpu_time_now = dispatch_pending_node (vm, i, cpu_time_now); |
| 1650 | /* Reset pending vector for next iteration. */ |
| 1651 | _vec_len (nm->pending_frames) = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1652 | |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1653 | if (is_main) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1654 | { |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1655 | /* *INDENT-OFF* */ |
| 1656 | ELOG_TYPE_DECLARE (es) = |
| 1657 | { |
| 1658 | .format = "process tw start", |
| 1659 | .format_args = "", |
| 1660 | }; |
| 1661 | ELOG_TYPE_DECLARE (ee) = |
| 1662 | { |
| 1663 | .format = "process tw end: %d", |
| 1664 | .format_args = "i4", |
| 1665 | }; |
| 1666 | /* *INDENT-ON* */ |
| 1667 | |
| 1668 | struct |
| 1669 | { |
| 1670 | int nready_procs; |
| 1671 | } *ed; |
| 1672 | |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1673 | /* Check if process nodes have expired from timing wheel. */ |
Dave Barach | 5c20a01 | 2017-06-13 08:48:31 -0400 | [diff] [blame] | 1674 | ASSERT (nm->data_from_advancing_timing_wheel != 0); |
| 1675 | |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1676 | if (PREDICT_FALSE (vm->elog_trace_graph_dispatch)) |
| 1677 | ed = ELOG_DATA (&vlib_global_main.elog_main, es); |
| 1678 | |
Dave Barach | 5c20a01 | 2017-06-13 08:48:31 -0400 | [diff] [blame] | 1679 | nm->data_from_advancing_timing_wheel = |
| 1680 | TW (tw_timer_expire_timers_vec) |
| 1681 | ((TWT (tw_timer_wheel) *) nm->timing_wheel, vlib_time_now (vm), |
| 1682 | nm->data_from_advancing_timing_wheel); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1683 | |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1684 | ASSERT (nm->data_from_advancing_timing_wheel != 0); |
Dave Barach | 5c20a01 | 2017-06-13 08:48:31 -0400 | [diff] [blame] | 1685 | |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1686 | if (PREDICT_FALSE (vm->elog_trace_graph_dispatch)) |
| 1687 | { |
| 1688 | ed = ELOG_DATA (&vlib_global_main.elog_main, ee); |
| 1689 | ed->nready_procs = |
| 1690 | _vec_len (nm->data_from_advancing_timing_wheel); |
| 1691 | } |
| 1692 | |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1693 | if (PREDICT_FALSE |
| 1694 | (_vec_len (nm->data_from_advancing_timing_wheel) > 0)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1695 | { |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1696 | uword i; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1697 | |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1698 | for (i = 0; i < _vec_len (nm->data_from_advancing_timing_wheel); |
| 1699 | i++) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1700 | { |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1701 | u32 d = nm->data_from_advancing_timing_wheel[i]; |
| 1702 | u32 di = vlib_timing_wheel_data_get_index (d); |
| 1703 | |
| 1704 | if (vlib_timing_wheel_data_is_timed_event (d)) |
| 1705 | { |
| 1706 | vlib_signal_timed_event_data_t *te = |
| 1707 | pool_elt_at_index (nm->signal_timed_event_data_pool, |
| 1708 | di); |
| 1709 | vlib_node_t *n = |
| 1710 | vlib_get_node (vm, te->process_node_index); |
| 1711 | vlib_process_t *p = |
| 1712 | vec_elt (nm->processes, n->runtime_index); |
| 1713 | void *data; |
| 1714 | data = |
| 1715 | vlib_process_signal_event_helper (nm, n, p, |
| 1716 | te->event_type_index, |
| 1717 | te->n_data_elts, |
| 1718 | te->n_data_elt_bytes); |
| 1719 | if (te->n_data_bytes < sizeof (te->inline_event_data)) |
Dave Barach | 178cf49 | 2018-11-13 16:34:13 -0500 | [diff] [blame] | 1720 | clib_memcpy_fast (data, te->inline_event_data, |
| 1721 | te->n_data_bytes); |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1722 | else |
| 1723 | { |
Dave Barach | 178cf49 | 2018-11-13 16:34:13 -0500 | [diff] [blame] | 1724 | clib_memcpy_fast (data, te->event_data_as_vector, |
| 1725 | te->n_data_bytes); |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1726 | vec_free (te->event_data_as_vector); |
| 1727 | } |
| 1728 | pool_put (nm->signal_timed_event_data_pool, te); |
| 1729 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1730 | else |
| 1731 | { |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1732 | cpu_time_now = clib_cpu_time_now (); |
| 1733 | cpu_time_now = |
| 1734 | dispatch_suspended_process (vm, di, cpu_time_now); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1735 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1736 | } |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1737 | _vec_len (nm->data_from_advancing_timing_wheel) = 0; |
| 1738 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1739 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1740 | vlib_increment_main_loop_counter (vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1741 | /* Record time stamp in case there are no enabled nodes and above |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1742 | calls do not update time stamp. */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1743 | cpu_time_now = clib_cpu_time_now (); |
Dave Barach | 000a029 | 2020-02-17 17:07:12 -0500 | [diff] [blame] | 1744 | vm->loops_this_reporting_interval++; |
| 1745 | now = clib_time_now_internal (&vm->clib_time, cpu_time_now); |
| 1746 | /* Time to update loops_per_second? */ |
| 1747 | if (PREDICT_FALSE (now >= vm->loop_interval_end)) |
| 1748 | { |
| 1749 | /* Next sample ends in 20ms */ |
| 1750 | if (vm->loop_interval_start) |
| 1751 | { |
| 1752 | f64 this_loops_per_second; |
| 1753 | |
| 1754 | this_loops_per_second = |
| 1755 | ((f64) vm->loops_this_reporting_interval) / (now - |
| 1756 | vm->loop_interval_start); |
| 1757 | |
| 1758 | vm->loops_per_second = |
| 1759 | vm->loops_per_second * vm->damping_constant + |
| 1760 | (1.0 - vm->damping_constant) * this_loops_per_second; |
| 1761 | if (vm->loops_per_second != 0.0) |
| 1762 | vm->seconds_per_loop = 1.0 / vm->loops_per_second; |
| 1763 | else |
| 1764 | vm->seconds_per_loop = 0.0; |
| 1765 | } |
| 1766 | /* New interval starts now, and ends in 20ms */ |
| 1767 | vm->loop_interval_start = now; |
| 1768 | vm->loop_interval_end = now + 2e-4; |
| 1769 | vm->loops_this_reporting_interval = 0; |
| 1770 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1771 | } |
| 1772 | } |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1773 | |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1774 | static void |
| 1775 | vlib_main_loop (vlib_main_t * vm) |
| 1776 | { |
| 1777 | vlib_main_or_worker_loop (vm, /* is_main */ 1); |
| 1778 | } |
| 1779 | |
| 1780 | void |
| 1781 | vlib_worker_loop (vlib_main_t * vm) |
| 1782 | { |
| 1783 | vlib_main_or_worker_loop (vm, /* is_main */ 0); |
| 1784 | } |
| 1785 | |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 1786 | vlib_global_main_t vlib_global_main; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1787 | |
Damjan Marion | 25ab6c5 | 2021-03-05 14:41:25 +0100 | [diff] [blame] | 1788 | void |
| 1789 | vlib_add_del_post_mortem_callback (void *cb, int is_add) |
| 1790 | { |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 1791 | vlib_global_main_t *vgm = vlib_get_global_main (); |
Damjan Marion | 25ab6c5 | 2021-03-05 14:41:25 +0100 | [diff] [blame] | 1792 | int i; |
| 1793 | |
| 1794 | if (is_add == 0) |
| 1795 | { |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 1796 | for (i = vec_len (vgm->post_mortem_callbacks) - 1; i >= 0; i--) |
| 1797 | if (vgm->post_mortem_callbacks[i] == cb) |
| 1798 | vec_del1 (vgm->post_mortem_callbacks, i); |
Damjan Marion | 25ab6c5 | 2021-03-05 14:41:25 +0100 | [diff] [blame] | 1799 | return; |
| 1800 | } |
| 1801 | |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 1802 | for (i = 0; i < vec_len (vgm->post_mortem_callbacks); i++) |
| 1803 | if (vgm->post_mortem_callbacks[i] == cb) |
Damjan Marion | 25ab6c5 | 2021-03-05 14:41:25 +0100 | [diff] [blame] | 1804 | return; |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 1805 | vec_add1 (vgm->post_mortem_callbacks, cb); |
Damjan Marion | 25ab6c5 | 2021-03-05 14:41:25 +0100 | [diff] [blame] | 1806 | } |
| 1807 | |
| 1808 | static void |
| 1809 | elog_post_mortem_dump (void) |
| 1810 | { |
Damjan Marion | f553a2c | 2021-03-26 13:45:37 +0100 | [diff] [blame] | 1811 | elog_main_t *em = vlib_get_elog_main (); |
Damjan Marion | 25ab6c5 | 2021-03-05 14:41:25 +0100 | [diff] [blame] | 1812 | |
| 1813 | u8 *filename; |
| 1814 | clib_error_t *error; |
| 1815 | |
| 1816 | filename = format (0, "/tmp/elog_post_mortem.%d%c", getpid (), 0); |
| 1817 | error = elog_write_file (em, (char *) filename, 1 /* flush ring */); |
| 1818 | if (error) |
| 1819 | clib_error_report (error); |
| 1820 | /* |
| 1821 | * We're in the middle of crashing. Don't try to free the filename. |
| 1822 | */ |
| 1823 | } |
| 1824 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1825 | static clib_error_t * |
| 1826 | vlib_main_configure (vlib_main_t * vm, unformat_input_t * input) |
| 1827 | { |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 1828 | vlib_global_main_t *vgm = vlib_get_global_main (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1829 | int turn_on_mem_trace = 0; |
| 1830 | |
| 1831 | while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) |
| 1832 | { |
| 1833 | if (unformat (input, "memory-trace")) |
| 1834 | turn_on_mem_trace = 1; |
| 1835 | |
| 1836 | else if (unformat (input, "elog-events %d", |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 1837 | &vgm->configured_elog_ring_size)) |
| 1838 | vgm->configured_elog_ring_size = |
| 1839 | 1 << max_log2 (vgm->configured_elog_ring_size); |
Dave Barach | 8148131 | 2017-05-16 09:08:14 -0400 | [diff] [blame] | 1840 | else if (unformat (input, "elog-post-mortem-dump")) |
Damjan Marion | 25ab6c5 | 2021-03-05 14:41:25 +0100 | [diff] [blame] | 1841 | vlib_add_del_post_mortem_callback (elog_post_mortem_dump, |
| 1842 | /* is_add */ 1); |
Dave Barach | c74b43c | 2020-04-09 17:24:07 -0400 | [diff] [blame] | 1843 | else if (unformat (input, "buffer-alloc-success-rate %f", |
| 1844 | &vm->buffer_alloc_success_rate)) |
| 1845 | { |
| 1846 | if (VLIB_BUFFER_ALLOC_FAULT_INJECTOR == 0) |
| 1847 | return clib_error_return |
| 1848 | (0, "Buffer fault injection not configured"); |
| 1849 | } |
| 1850 | else if (unformat (input, "buffer-alloc-success-seed %u", |
| 1851 | &vm->buffer_alloc_success_seed)) |
| 1852 | { |
| 1853 | if (VLIB_BUFFER_ALLOC_FAULT_INJECTOR == 0) |
| 1854 | return clib_error_return |
| 1855 | (0, "Buffer fault injection not configured"); |
| 1856 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1857 | else |
| 1858 | return unformat_parse_error (input); |
| 1859 | } |
| 1860 | |
| 1861 | unformat_free (input); |
| 1862 | |
| 1863 | /* Enable memory trace as early as possible. */ |
| 1864 | if (turn_on_mem_trace) |
| 1865 | clib_mem_trace (1); |
| 1866 | |
| 1867 | return 0; |
| 1868 | } |
| 1869 | |
| 1870 | VLIB_EARLY_CONFIG_FUNCTION (vlib_main_configure, "vlib"); |
| 1871 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1872 | static void |
Dave Barach | 11fb09e | 2020-08-06 12:10:09 -0400 | [diff] [blame] | 1873 | placeholder_queue_signal_callback (vlib_main_t * vm) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1874 | { |
| 1875 | } |
Dave Barach | 16c75df | 2016-05-31 14:05:46 -0400 | [diff] [blame] | 1876 | |
Dave Barach | 1f80658 | 2018-06-14 09:18:21 -0400 | [diff] [blame] | 1877 | #define foreach_weak_reference_stub \ |
| 1878 | _(vlib_map_stat_segment_init) \ |
| 1879 | _(vpe_api_init) \ |
| 1880 | _(vlibmemory_init) \ |
| 1881 | _(map_api_segment_init) |
| 1882 | |
| 1883 | #define _(name) \ |
| 1884 | clib_error_t *name (vlib_main_t *vm) __attribute__((weak)); \ |
| 1885 | clib_error_t *name (vlib_main_t *vm) { return 0; } |
| 1886 | foreach_weak_reference_stub; |
| 1887 | #undef _ |
| 1888 | |
Dave Barach | b09f4d0 | 2019-07-15 16:00:03 -0400 | [diff] [blame] | 1889 | void vl_api_set_elog_main (elog_main_t * m) __attribute__ ((weak)); |
| 1890 | void |
| 1891 | vl_api_set_elog_main (elog_main_t * m) |
| 1892 | { |
| 1893 | clib_warning ("STUB"); |
| 1894 | } |
| 1895 | |
| 1896 | int vl_api_set_elog_trace_api_messages (int enable) __attribute__ ((weak)); |
| 1897 | int |
| 1898 | vl_api_set_elog_trace_api_messages (int enable) |
| 1899 | { |
| 1900 | clib_warning ("STUB"); |
| 1901 | return 0; |
| 1902 | } |
| 1903 | |
| 1904 | int vl_api_get_elog_trace_api_messages (void) __attribute__ ((weak)); |
| 1905 | int |
| 1906 | vl_api_get_elog_trace_api_messages (void) |
| 1907 | { |
| 1908 | clib_warning ("STUB"); |
| 1909 | return 0; |
| 1910 | } |
| 1911 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1912 | /* Main function. */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1913 | int |
Eyal Bari | d334a6b | 2016-09-19 10:23:39 +0300 | [diff] [blame] | 1914 | vlib_main (vlib_main_t * volatile vm, unformat_input_t * input) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1915 | { |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 1916 | vlib_global_main_t *vgm = vlib_get_global_main (); |
Eyal Bari | d334a6b | 2016-09-19 10:23:39 +0300 | [diff] [blame] | 1917 | clib_error_t *volatile error; |
Dave Barach | 5c20a01 | 2017-06-13 08:48:31 -0400 | [diff] [blame] | 1918 | vlib_node_main_t *nm = &vm->node_main; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1919 | |
Dave Barach | 11fb09e | 2020-08-06 12:10:09 -0400 | [diff] [blame] | 1920 | vm->queue_signal_callback = placeholder_queue_signal_callback; |
Dave Barach | 16c75df | 2016-05-31 14:05:46 -0400 | [diff] [blame] | 1921 | |
Dave Barach | bc867c3 | 2020-11-25 10:07:09 -0500 | [diff] [blame] | 1922 | /* Reconfigure event log which is enabled very early */ |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 1923 | if (vgm->configured_elog_ring_size && |
| 1924 | vgm->configured_elog_ring_size != vgm->elog_main.event_ring_size) |
| 1925 | elog_resize (&vgm->elog_main, vgm->configured_elog_ring_size); |
Damjan Marion | f553a2c | 2021-03-26 13:45:37 +0100 | [diff] [blame] | 1926 | vl_api_set_elog_main (vlib_get_elog_main ()); |
Dave Barach | b09f4d0 | 2019-07-15 16:00:03 -0400 | [diff] [blame] | 1927 | (void) vl_api_set_elog_trace_api_messages (1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1928 | |
| 1929 | /* Default name. */ |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 1930 | if (!vgm->name) |
| 1931 | vgm->name = "VLIB"; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1932 | |
Damjan Marion | 68b4da6 | 2018-09-30 18:26:20 +0200 | [diff] [blame] | 1933 | if ((error = vlib_physmem_init (vm))) |
Damjan Marion | 04a7f05 | 2017-07-10 15:06:17 +0200 | [diff] [blame] | 1934 | { |
Damjan Marion | 49d66f1 | 2017-07-20 18:10:35 +0200 | [diff] [blame] | 1935 | clib_error_report (error); |
| 1936 | goto done; |
Damjan Marion | 04a7f05 | 2017-07-10 15:06:17 +0200 | [diff] [blame] | 1937 | } |
Damjan Marion | 49d66f1 | 2017-07-20 18:10:35 +0200 | [diff] [blame] | 1938 | |
Filip Tehlar | d2bbdef | 2019-02-22 05:05:53 -0800 | [diff] [blame] | 1939 | if ((error = vlib_map_stat_segment_init (vm))) |
| 1940 | { |
| 1941 | clib_error_report (error); |
| 1942 | goto done; |
| 1943 | } |
| 1944 | |
Damjan Marion | 49d66f1 | 2017-07-20 18:10:35 +0200 | [diff] [blame] | 1945 | if ((error = vlib_buffer_main_init (vm))) |
Damjan Marion | 04a7f05 | 2017-07-10 15:06:17 +0200 | [diff] [blame] | 1946 | { |
Damjan Marion | 49d66f1 | 2017-07-20 18:10:35 +0200 | [diff] [blame] | 1947 | clib_error_report (error); |
| 1948 | goto done; |
Damjan Marion | 04a7f05 | 2017-07-10 15:06:17 +0200 | [diff] [blame] | 1949 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1950 | |
| 1951 | if ((error = vlib_thread_init (vm))) |
| 1952 | { |
| 1953 | clib_error_report (error); |
| 1954 | goto done; |
| 1955 | } |
| 1956 | |
Damjan Marion | a31698b | 2021-03-10 14:35:28 +0100 | [diff] [blame] | 1957 | /* Register node ifunction variants */ |
| 1958 | vlib_register_all_node_march_variants (vm); |
| 1959 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1960 | /* Register static nodes so that init functions may use them. */ |
| 1961 | vlib_register_all_static_nodes (vm); |
| 1962 | |
| 1963 | /* Set seed for random number generator. |
| 1964 | Allow user to specify seed to make random sequence deterministic. */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1965 | if (!unformat (input, "seed %wd", &vm->random_seed)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1966 | vm->random_seed = clib_cpu_time_now (); |
| 1967 | clib_random_buffer_init (&vm->random_buffer, vm->random_seed); |
| 1968 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1969 | /* Initialize node graph. */ |
| 1970 | if ((error = vlib_node_main_init (vm))) |
| 1971 | { |
| 1972 | /* Arrange for graph hook up error to not be fatal when debugging. */ |
| 1973 | if (CLIB_DEBUG > 0) |
| 1974 | clib_error_report (error); |
| 1975 | else |
| 1976 | goto done; |
| 1977 | } |
| 1978 | |
Dave Barach | 1f80658 | 2018-06-14 09:18:21 -0400 | [diff] [blame] | 1979 | /* Direct call / weak reference, for vlib standalone use-cases */ |
| 1980 | if ((error = vpe_api_init (vm))) |
Dave Barach | 048a4e5 | 2018-06-01 18:52:25 -0400 | [diff] [blame] | 1981 | { |
| 1982 | clib_error_report (error); |
| 1983 | goto done; |
| 1984 | } |
| 1985 | |
Dave Barach | 1f80658 | 2018-06-14 09:18:21 -0400 | [diff] [blame] | 1986 | if ((error = vlibmemory_init (vm))) |
Dave Barach | 048a4e5 | 2018-06-01 18:52:25 -0400 | [diff] [blame] | 1987 | { |
| 1988 | clib_error_report (error); |
| 1989 | goto done; |
| 1990 | } |
| 1991 | |
Dave Barach | 1f80658 | 2018-06-14 09:18:21 -0400 | [diff] [blame] | 1992 | if ((error = map_api_segment_init (vm))) |
Dave Barach | 048a4e5 | 2018-06-01 18:52:25 -0400 | [diff] [blame] | 1993 | { |
| 1994 | clib_error_report (error); |
| 1995 | goto done; |
| 1996 | } |
| 1997 | |
Ole Troan | 964f93e | 2016-06-10 13:22:36 +0200 | [diff] [blame] | 1998 | /* See unix/main.c; most likely already set up */ |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 1999 | if (vgm->init_functions_called == 0) |
| 2000 | vgm->init_functions_called = hash_create (0, /* value bytes */ 0); |
Ole Troan | 964f93e | 2016-06-10 13:22:36 +0200 | [diff] [blame] | 2001 | if ((error = vlib_call_all_init_functions (vm))) |
| 2002 | goto done; |
| 2003 | |
Dave Barach | 5c20a01 | 2017-06-13 08:48:31 -0400 | [diff] [blame] | 2004 | nm->timing_wheel = clib_mem_alloc_aligned (sizeof (TWT (tw_timer_wheel)), |
| 2005 | CLIB_CACHE_LINE_BYTES); |
| 2006 | |
| 2007 | vec_validate (nm->data_from_advancing_timing_wheel, 10); |
| 2008 | _vec_len (nm->data_from_advancing_timing_wheel) = 0; |
| 2009 | |
| 2010 | /* Create the process timing wheel */ |
| 2011 | TW (tw_timer_wheel_init) ((TWT (tw_timer_wheel) *) nm->timing_wheel, |
| 2012 | 0 /* no callback */ , |
| 2013 | 10e-6 /* timer period 10us */ , |
| 2014 | ~0 /* max expirations per call */ ); |
| 2015 | |
Dave Barach | 2877eee | 2017-12-15 12:22:57 -0500 | [diff] [blame] | 2016 | vec_validate (vm->pending_rpc_requests, 0); |
| 2017 | _vec_len (vm->pending_rpc_requests) = 0; |
Dave Barach | f6c68d7 | 2018-11-01 08:12:52 -0400 | [diff] [blame] | 2018 | vec_validate (vm->processing_rpc_requests, 0); |
| 2019 | _vec_len (vm->processing_rpc_requests) = 0; |
Dave Barach | 2877eee | 2017-12-15 12:22:57 -0500 | [diff] [blame] | 2020 | |
Dave Barach | c74b43c | 2020-04-09 17:24:07 -0400 | [diff] [blame] | 2021 | /* Default params for the buffer allocator fault injector, if configured */ |
| 2022 | if (VLIB_BUFFER_ALLOC_FAULT_INJECTOR > 0) |
| 2023 | { |
| 2024 | vm->buffer_alloc_success_seed = 0xdeaddabe; |
| 2025 | vm->buffer_alloc_success_rate = 0.80; |
| 2026 | } |
| 2027 | |
Dave Barach | d1e17d0 | 2019-03-21 18:01:48 -0400 | [diff] [blame] | 2028 | if ((error = vlib_call_all_config_functions (vm, input, 0 /* is_early */ ))) |
| 2029 | goto done; |
| 2030 | |
Dave Barach | 000a029 | 2020-02-17 17:07:12 -0500 | [diff] [blame] | 2031 | /* |
| 2032 | * Use exponential smoothing, with a half-life of 1 second |
| 2033 | * reported_rate(t) = reported_rate(t-1) * K + rate(t)*(1-K) |
| 2034 | * |
| 2035 | * Sample every 20ms, aka 50 samples per second |
| 2036 | * K = exp (-1.0/20.0); |
| 2037 | * K = 0.95 |
| 2038 | */ |
| 2039 | vm->damping_constant = exp (-1.0 / 20.0); |
| 2040 | |
Dave Barach | c602b38 | 2019-06-03 19:48:22 -0400 | [diff] [blame] | 2041 | /* Sort per-thread init functions before we start threads */ |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 2042 | vlib_sort_init_exit_functions (&vgm->worker_init_function_registrations); |
Dave Barach | c602b38 | 2019-06-03 19:48:22 -0400 | [diff] [blame] | 2043 | |
Dave Barach | d1e17d0 | 2019-03-21 18:01:48 -0400 | [diff] [blame] | 2044 | /* Call all main loop enter functions. */ |
| 2045 | { |
| 2046 | clib_error_t *sub_error; |
| 2047 | sub_error = vlib_call_all_main_loop_enter_functions (vm); |
| 2048 | if (sub_error) |
| 2049 | clib_error_report (sub_error); |
| 2050 | } |
| 2051 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2052 | switch (clib_setjmp (&vm->main_loop_exit, VLIB_MAIN_LOOP_EXIT_NONE)) |
| 2053 | { |
| 2054 | case VLIB_MAIN_LOOP_EXIT_NONE: |
| 2055 | vm->main_loop_exit_set = 1; |
| 2056 | break; |
| 2057 | |
| 2058 | case VLIB_MAIN_LOOP_EXIT_CLI: |
| 2059 | goto done; |
| 2060 | |
| 2061 | default: |
| 2062 | error = vm->main_loop_error; |
| 2063 | goto done; |
| 2064 | } |
| 2065 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2066 | vlib_main_loop (vm); |
| 2067 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 2068 | done: |
Kommula Shiva Shankar | ced43e2 | 2021-01-28 13:05:59 +0530 | [diff] [blame] | 2069 | vlib_worker_thread_barrier_sync (vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2070 | /* Call all exit functions. */ |
| 2071 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 2072 | clib_error_t *sub_error; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2073 | sub_error = vlib_call_all_main_loop_exit_functions (vm); |
| 2074 | if (sub_error) |
| 2075 | clib_error_report (sub_error); |
| 2076 | } |
Kommula Shiva Shankar | ced43e2 | 2021-01-28 13:05:59 +0530 | [diff] [blame] | 2077 | vlib_worker_thread_barrier_release (vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2078 | |
| 2079 | if (error) |
| 2080 | clib_error_report (error); |
| 2081 | |
Pierre Pfister | c26cc72 | 2021-09-10 16:38:03 +0200 | [diff] [blame^] | 2082 | return vm->main_loop_exit_status; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2083 | } |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 2084 | |
Dave Barach | ab1a50c | 2020-10-06 14:08:16 -0400 | [diff] [blame] | 2085 | vlib_main_t * |
| 2086 | vlib_get_main_not_inline (void) |
| 2087 | { |
| 2088 | return vlib_get_main (); |
| 2089 | } |
| 2090 | |
| 2091 | elog_main_t * |
| 2092 | vlib_get_elog_main_not_inline () |
| 2093 | { |
| 2094 | return &vlib_global_main.elog_main; |
| 2095 | } |
| 2096 | |
Pierre Pfister | c26cc72 | 2021-09-10 16:38:03 +0200 | [diff] [blame^] | 2097 | void |
| 2098 | vlib_exit_with_status (vlib_main_t *vm, int status) |
| 2099 | { |
| 2100 | vm->main_loop_exit_status = status; |
| 2101 | __atomic_store_n (&vm->main_loop_exit_now, 1, __ATOMIC_RELEASE); |
| 2102 | } |
| 2103 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 2104 | /* |
| 2105 | * fd.io coding-style-patch-verification: ON |
| 2106 | * |
| 2107 | * Local Variables: |
| 2108 | * eval: (c-set-style "gnu") |
| 2109 | * End: |
| 2110 | */ |