Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2015 Cisco and/or its affiliates. |
| 3 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | * you may not use this file except in compliance with the License. |
| 5 | * You may obtain a copy of the License at: |
| 6 | * |
| 7 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | * |
| 9 | * Unless required by applicable law or agreed to in writing, software |
| 10 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | * See the License for the specific language governing permissions and |
| 13 | * limitations under the License. |
| 14 | */ |
| 15 | /* |
| 16 | * main.c: main vector processing loop |
| 17 | * |
| 18 | * Copyright (c) 2008 Eliot Dresselhaus |
| 19 | * |
| 20 | * Permission is hereby granted, free of charge, to any person obtaining |
| 21 | * a copy of this software and associated documentation files (the |
| 22 | * "Software"), to deal in the Software without restriction, including |
| 23 | * without limitation the rights to use, copy, modify, merge, publish, |
| 24 | * distribute, sublicense, and/or sell copies of the Software, and to |
| 25 | * permit persons to whom the Software is furnished to do so, subject to |
| 26 | * the following conditions: |
| 27 | * |
| 28 | * The above copyright notice and this permission notice shall be |
| 29 | * included in all copies or substantial portions of the Software. |
| 30 | * |
| 31 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 32 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 33 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 34 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE |
| 35 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION |
| 36 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION |
| 37 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
| 38 | */ |
| 39 | |
| 40 | #include <math.h> |
| 41 | #include <vppinfra/format.h> |
| 42 | #include <vlib/vlib.h> |
| 43 | #include <vlib/threads.h> |
Dave Barach | 5c20a01 | 2017-06-13 08:48:31 -0400 | [diff] [blame] | 44 | #include <vppinfra/tw_timer_1t_3w_1024sl_ov.h> |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 45 | |
Damjan Marion | 04a7f05 | 2017-07-10 15:06:17 +0200 | [diff] [blame] | 46 | #include <vlib/unix/unix.h> |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 47 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 48 | /* Actually allocate a few extra slots of vector data to support |
| 49 | speculative vector enqueues which overflow vector data in next frame. */ |
| 50 | #define VLIB_FRAME_SIZE_ALLOC (VLIB_FRAME_SIZE + 4) |
| 51 | |
| 52 | always_inline u32 |
| 53 | vlib_frame_bytes (u32 n_scalar_bytes, u32 n_vector_bytes) |
| 54 | { |
| 55 | u32 n_bytes; |
| 56 | |
| 57 | /* Make room for vlib_frame_t plus scalar arguments. */ |
| 58 | n_bytes = vlib_frame_vector_byte_offset (n_scalar_bytes); |
| 59 | |
| 60 | /* Make room for vector arguments. |
| 61 | Allocate a few extra slots of vector data to support |
| 62 | speculative vector enqueues which overflow vector data in next frame. */ |
| 63 | #define VLIB_FRAME_SIZE_EXTRA 4 |
| 64 | n_bytes += (VLIB_FRAME_SIZE + VLIB_FRAME_SIZE_EXTRA) * n_vector_bytes; |
| 65 | |
| 66 | /* Magic number is first 32bit number after vector data. |
| 67 | Used to make sure that vector data is never overrun. */ |
| 68 | #define VLIB_FRAME_MAGIC (0xabadc0ed) |
| 69 | n_bytes += sizeof (u32); |
| 70 | |
| 71 | /* Pad to cache line. */ |
| 72 | n_bytes = round_pow2 (n_bytes, CLIB_CACHE_LINE_BYTES); |
| 73 | |
| 74 | return n_bytes; |
| 75 | } |
| 76 | |
| 77 | always_inline u32 * |
| 78 | vlib_frame_find_magic (vlib_frame_t * f, vlib_node_t * node) |
| 79 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 80 | void *p = f; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 81 | |
| 82 | p += vlib_frame_vector_byte_offset (node->scalar_size); |
| 83 | |
| 84 | p += (VLIB_FRAME_SIZE + VLIB_FRAME_SIZE_EXTRA) * node->vector_size; |
| 85 | |
| 86 | return p; |
| 87 | } |
| 88 | |
Dave Barach | 593eedf | 2019-03-10 09:44:51 -0400 | [diff] [blame] | 89 | static inline vlib_frame_size_t * |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 90 | get_frame_size_info (vlib_node_main_t * nm, |
| 91 | u32 n_scalar_bytes, u32 n_vector_bytes) |
| 92 | { |
Dave Barach | 593eedf | 2019-03-10 09:44:51 -0400 | [diff] [blame] | 93 | #ifdef VLIB_SUPPORTS_ARBITRARY_SCALAR_SIZES |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 94 | uword key = (n_scalar_bytes << 16) | n_vector_bytes; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 95 | uword *p, i; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 96 | |
| 97 | p = hash_get (nm->frame_size_hash, key); |
| 98 | if (p) |
| 99 | i = p[0]; |
| 100 | else |
| 101 | { |
| 102 | i = vec_len (nm->frame_sizes); |
| 103 | vec_validate (nm->frame_sizes, i); |
| 104 | hash_set (nm->frame_size_hash, key, i); |
| 105 | } |
| 106 | |
| 107 | return vec_elt_at_index (nm->frame_sizes, i); |
Dave Barach | 593eedf | 2019-03-10 09:44:51 -0400 | [diff] [blame] | 108 | #else |
| 109 | ASSERT (vlib_frame_bytes (n_scalar_bytes, n_vector_bytes) |
| 110 | == (vlib_frame_bytes (0, 4))); |
| 111 | return vec_elt_at_index (nm->frame_sizes, 0); |
| 112 | #endif |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 113 | } |
| 114 | |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 115 | static vlib_frame_t * |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 116 | vlib_frame_alloc_to_node (vlib_main_t * vm, u32 to_node_index, |
| 117 | u32 frame_flags) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 118 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 119 | vlib_node_main_t *nm = &vm->node_main; |
| 120 | vlib_frame_size_t *fs; |
| 121 | vlib_node_t *to_node; |
| 122 | vlib_frame_t *f; |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 123 | u32 l, n, scalar_size, vector_size; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 124 | |
Dave Barach | a8f4ebd | 2021-02-08 07:56:22 -0500 | [diff] [blame] | 125 | ASSERT (vm == vlib_get_main ()); |
| 126 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 127 | to_node = vlib_get_node (vm, to_node_index); |
| 128 | |
| 129 | scalar_size = to_node->scalar_size; |
| 130 | vector_size = to_node->vector_size; |
| 131 | |
| 132 | fs = get_frame_size_info (nm, scalar_size, vector_size); |
| 133 | n = vlib_frame_bytes (scalar_size, vector_size); |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 134 | if ((l = vec_len (fs->free_frames)) > 0) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 135 | { |
| 136 | /* Allocate from end of free list. */ |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 137 | f = fs->free_frames[l - 1]; |
| 138 | _vec_len (fs->free_frames) = l - 1; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 139 | } |
| 140 | else |
| 141 | { |
Damjan Marion | 3f46baf | 2016-02-06 19:16:21 +0100 | [diff] [blame] | 142 | f = clib_mem_alloc_aligned_no_fail (n, VLIB_FRAME_ALIGN); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 143 | } |
| 144 | |
| 145 | /* Poison frame when debugging. */ |
| 146 | if (CLIB_DEBUG > 0) |
Dave Barach | b7b9299 | 2018-10-17 10:38:51 -0400 | [diff] [blame] | 147 | clib_memset (f, 0xfe, n); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 148 | |
| 149 | /* Insert magic number. */ |
| 150 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 151 | u32 *magic; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 152 | |
| 153 | magic = vlib_frame_find_magic (f, to_node); |
| 154 | *magic = VLIB_FRAME_MAGIC; |
| 155 | } |
| 156 | |
Damjan Marion | 633b6fd | 2018-09-14 14:38:53 +0200 | [diff] [blame] | 157 | f->frame_flags = VLIB_FRAME_IS_ALLOCATED | frame_flags; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 158 | f->n_vectors = 0; |
| 159 | f->scalar_size = scalar_size; |
| 160 | f->vector_size = vector_size; |
Damjan Marion | a3d5986 | 2018-11-10 10:23:00 +0100 | [diff] [blame] | 161 | f->flags = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 162 | |
| 163 | fs->n_alloc_frames += 1; |
| 164 | |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 165 | return f; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 166 | } |
| 167 | |
| 168 | /* Allocate a frame for from FROM_NODE to TO_NODE via TO_NEXT_INDEX. |
| 169 | Returns frame index. */ |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 170 | static vlib_frame_t * |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 171 | vlib_frame_alloc (vlib_main_t * vm, vlib_node_runtime_t * from_node_runtime, |
| 172 | u32 to_next_index) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 173 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 174 | vlib_node_t *from_node; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 175 | |
| 176 | from_node = vlib_get_node (vm, from_node_runtime->node_index); |
| 177 | ASSERT (to_next_index < vec_len (from_node->next_nodes)); |
| 178 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 179 | return vlib_frame_alloc_to_node (vm, from_node->next_nodes[to_next_index], |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 180 | /* frame_flags */ 0); |
| 181 | } |
| 182 | |
| 183 | vlib_frame_t * |
| 184 | vlib_get_frame_to_node (vlib_main_t * vm, u32 to_node_index) |
| 185 | { |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 186 | vlib_frame_t *f = vlib_frame_alloc_to_node (vm, to_node_index, |
| 187 | /* frame_flags */ |
| 188 | VLIB_FRAME_FREE_AFTER_DISPATCH); |
| 189 | return vlib_get_frame (vm, f); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 190 | } |
| 191 | |
Dave Barach | c74b43c | 2020-04-09 17:24:07 -0400 | [diff] [blame] | 192 | static inline void |
| 193 | vlib_validate_frame_indices (vlib_frame_t * f) |
| 194 | { |
| 195 | if (CLIB_DEBUG > 0) |
| 196 | { |
| 197 | int i; |
| 198 | u32 *from = vlib_frame_vector_args (f); |
| 199 | |
| 200 | /* Check for bad buffer index values */ |
| 201 | for (i = 0; i < f->n_vectors; i++) |
| 202 | { |
| 203 | if (from[i] == 0) |
| 204 | { |
| 205 | clib_warning ("BUG: buffer index 0 at index %d", i); |
| 206 | ASSERT (0); |
| 207 | } |
| 208 | else if (from[i] == 0xfefefefe) |
| 209 | { |
| 210 | clib_warning ("BUG: frame poison pattern at index %d", i); |
| 211 | ASSERT (0); |
| 212 | } |
| 213 | } |
| 214 | } |
| 215 | } |
| 216 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 217 | void |
| 218 | vlib_put_frame_to_node (vlib_main_t * vm, u32 to_node_index, vlib_frame_t * f) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 219 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 220 | vlib_pending_frame_t *p; |
| 221 | vlib_node_t *to_node; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 222 | |
| 223 | if (f->n_vectors == 0) |
| 224 | return; |
| 225 | |
Dave Barach | a8f4ebd | 2021-02-08 07:56:22 -0500 | [diff] [blame] | 226 | ASSERT (vm == vlib_get_main ()); |
| 227 | |
Dave Barach | c74b43c | 2020-04-09 17:24:07 -0400 | [diff] [blame] | 228 | vlib_validate_frame_indices (f); |
| 229 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 230 | to_node = vlib_get_node (vm, to_node_index); |
| 231 | |
| 232 | vec_add2 (vm->node_main.pending_frames, p, 1); |
| 233 | |
Damjan Marion | 633b6fd | 2018-09-14 14:38:53 +0200 | [diff] [blame] | 234 | f->frame_flags |= VLIB_FRAME_PENDING; |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 235 | p->frame = vlib_get_frame (vm, f); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 236 | p->node_runtime_index = to_node->runtime_index; |
| 237 | p->next_frame_index = VLIB_PENDING_FRAME_NO_NEXT_FRAME; |
| 238 | } |
| 239 | |
| 240 | /* Free given frame. */ |
| 241 | void |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 242 | vlib_frame_free (vlib_main_t * vm, vlib_node_runtime_t * r, vlib_frame_t * f) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 243 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 244 | vlib_node_main_t *nm = &vm->node_main; |
| 245 | vlib_node_t *node; |
| 246 | vlib_frame_size_t *fs; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 247 | |
Dave Barach | a8f4ebd | 2021-02-08 07:56:22 -0500 | [diff] [blame] | 248 | ASSERT (vm == vlib_get_main ()); |
Damjan Marion | 633b6fd | 2018-09-14 14:38:53 +0200 | [diff] [blame] | 249 | ASSERT (f->frame_flags & VLIB_FRAME_IS_ALLOCATED); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 250 | |
| 251 | node = vlib_get_node (vm, r->node_index); |
| 252 | fs = get_frame_size_info (nm, node->scalar_size, node->vector_size); |
| 253 | |
Damjan Marion | 633b6fd | 2018-09-14 14:38:53 +0200 | [diff] [blame] | 254 | ASSERT (f->frame_flags & VLIB_FRAME_IS_ALLOCATED); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 255 | |
| 256 | /* No next frames may point to freed frame. */ |
| 257 | if (CLIB_DEBUG > 0) |
| 258 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 259 | vlib_next_frame_t *nf; |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 260 | vec_foreach (nf, vm->node_main.next_frames) ASSERT (nf->frame != f); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 261 | } |
| 262 | |
Damjan Marion | 296988d | 2019-02-21 20:24:54 +0100 | [diff] [blame] | 263 | f->frame_flags &= ~(VLIB_FRAME_IS_ALLOCATED | VLIB_FRAME_NO_APPEND); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 264 | |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 265 | vec_add1 (fs->free_frames, f); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 266 | ASSERT (fs->n_alloc_frames > 0); |
| 267 | fs->n_alloc_frames -= 1; |
| 268 | } |
| 269 | |
| 270 | static clib_error_t * |
| 271 | show_frame_stats (vlib_main_t * vm, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 272 | unformat_input_t * input, vlib_cli_command_t * cmd) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 273 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 274 | vlib_node_main_t *nm = &vm->node_main; |
| 275 | vlib_frame_size_t *fs; |
| 276 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 277 | vlib_cli_output (vm, "%=6s%=12s%=12s", "Size", "# Alloc", "# Free"); |
| 278 | vec_foreach (fs, nm->frame_sizes) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 279 | { |
| 280 | u32 n_alloc = fs->n_alloc_frames; |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 281 | u32 n_free = vec_len (fs->free_frames); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 282 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 283 | if (n_alloc + n_free > 0) |
| 284 | vlib_cli_output (vm, "%=6d%=12d%=12d", |
| 285 | fs - nm->frame_sizes, n_alloc, n_free); |
| 286 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 287 | |
| 288 | return 0; |
| 289 | } |
| 290 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 291 | /* *INDENT-OFF* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 292 | VLIB_CLI_COMMAND (show_frame_stats_cli, static) = { |
| 293 | .path = "show vlib frame-allocation", |
| 294 | .short_help = "Show node dispatch frame statistics", |
| 295 | .function = show_frame_stats, |
| 296 | }; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 297 | /* *INDENT-ON* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 298 | |
| 299 | /* Change ownership of enqueue rights to given next node. */ |
| 300 | static void |
| 301 | vlib_next_frame_change_ownership (vlib_main_t * vm, |
| 302 | vlib_node_runtime_t * node_runtime, |
| 303 | u32 next_index) |
| 304 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 305 | vlib_node_main_t *nm = &vm->node_main; |
| 306 | vlib_next_frame_t *next_frame; |
| 307 | vlib_node_t *node, *next_node; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 308 | |
| 309 | node = vec_elt (nm->nodes, node_runtime->node_index); |
| 310 | |
| 311 | /* Only internal & input nodes are allowed to call other nodes. */ |
| 312 | ASSERT (node->type == VLIB_NODE_TYPE_INTERNAL |
| 313 | || node->type == VLIB_NODE_TYPE_INPUT |
| 314 | || node->type == VLIB_NODE_TYPE_PROCESS); |
| 315 | |
| 316 | ASSERT (vec_len (node->next_nodes) == node_runtime->n_next_nodes); |
| 317 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 318 | next_frame = |
| 319 | vlib_node_runtime_get_next_frame (vm, node_runtime, next_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 320 | next_node = vec_elt (nm->nodes, node->next_nodes[next_index]); |
| 321 | |
| 322 | if (next_node->owner_node_index != VLIB_INVALID_NODE_INDEX) |
| 323 | { |
| 324 | /* Get frame from previous owner. */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 325 | vlib_next_frame_t *owner_next_frame; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 326 | vlib_next_frame_t tmp; |
| 327 | |
| 328 | owner_next_frame = |
| 329 | vlib_node_get_next_frame (vm, |
| 330 | next_node->owner_node_index, |
| 331 | next_node->owner_next_index); |
| 332 | |
| 333 | /* Swap target next frame with owner's. */ |
| 334 | tmp = owner_next_frame[0]; |
| 335 | owner_next_frame[0] = next_frame[0]; |
| 336 | next_frame[0] = tmp; |
| 337 | |
| 338 | /* |
| 339 | * If next_frame is already pending, we have to track down |
| 340 | * all pending frames and fix their next_frame_index fields. |
| 341 | */ |
| 342 | if (next_frame->flags & VLIB_FRAME_PENDING) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 343 | { |
| 344 | vlib_pending_frame_t *p; |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 345 | if (next_frame->frame != NULL) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 346 | { |
| 347 | vec_foreach (p, nm->pending_frames) |
| 348 | { |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 349 | if (p->frame == next_frame->frame) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 350 | { |
| 351 | p->next_frame_index = |
| 352 | next_frame - vm->node_main.next_frames; |
| 353 | } |
| 354 | } |
| 355 | } |
| 356 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 357 | } |
| 358 | else |
| 359 | { |
| 360 | /* No previous owner. Take ownership. */ |
| 361 | next_frame->flags |= VLIB_FRAME_OWNER; |
| 362 | } |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 363 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 364 | /* Record new owner. */ |
| 365 | next_node->owner_node_index = node->index; |
| 366 | next_node->owner_next_index = next_index; |
| 367 | |
| 368 | /* Now we should be owner. */ |
| 369 | ASSERT (next_frame->flags & VLIB_FRAME_OWNER); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 370 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 371 | |
| 372 | /* Make sure that magic number is still there. |
| 373 | Otherwise, it is likely that caller has overrun frame arguments. */ |
| 374 | always_inline void |
| 375 | validate_frame_magic (vlib_main_t * vm, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 376 | vlib_frame_t * f, vlib_node_t * n, uword next_index) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 377 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 378 | vlib_node_t *next_node = vlib_get_node (vm, n->next_nodes[next_index]); |
| 379 | u32 *magic = vlib_frame_find_magic (f, next_node); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 380 | ASSERT (VLIB_FRAME_MAGIC == magic[0]); |
| 381 | } |
| 382 | |
| 383 | vlib_frame_t * |
| 384 | vlib_get_next_frame_internal (vlib_main_t * vm, |
| 385 | vlib_node_runtime_t * node, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 386 | u32 next_index, u32 allocate_new_next_frame) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 387 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 388 | vlib_frame_t *f; |
| 389 | vlib_next_frame_t *nf; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 390 | u32 n_used; |
| 391 | |
| 392 | nf = vlib_node_runtime_get_next_frame (vm, node, next_index); |
| 393 | |
| 394 | /* Make sure this next frame owns right to enqueue to destination frame. */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 395 | if (PREDICT_FALSE (!(nf->flags & VLIB_FRAME_OWNER))) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 396 | vlib_next_frame_change_ownership (vm, node, next_index); |
| 397 | |
| 398 | /* ??? Don't need valid flag: can use frame_index == ~0 */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 399 | if (PREDICT_FALSE (!(nf->flags & VLIB_FRAME_IS_ALLOCATED))) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 400 | { |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 401 | nf->frame = vlib_frame_alloc (vm, node, next_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 402 | nf->flags |= VLIB_FRAME_IS_ALLOCATED; |
| 403 | } |
| 404 | |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 405 | f = nf->frame; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 406 | |
| 407 | /* Has frame been removed from pending vector (e.g. finished dispatching)? |
| 408 | If so we can reuse frame. */ |
Damjan Marion | 633b6fd | 2018-09-14 14:38:53 +0200 | [diff] [blame] | 409 | if ((nf->flags & VLIB_FRAME_PENDING) |
| 410 | && !(f->frame_flags & VLIB_FRAME_PENDING)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 411 | { |
| 412 | nf->flags &= ~VLIB_FRAME_PENDING; |
| 413 | f->n_vectors = 0; |
Damjan Marion | 9162c2d | 2018-11-20 09:55:10 +0100 | [diff] [blame] | 414 | f->flags = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 415 | } |
| 416 | |
Damjan Marion | 296988d | 2019-02-21 20:24:54 +0100 | [diff] [blame] | 417 | /* Allocate new frame if current one is marked as no-append or |
| 418 | it is already full. */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 419 | n_used = f->n_vectors; |
Damjan Marion | 296988d | 2019-02-21 20:24:54 +0100 | [diff] [blame] | 420 | if (n_used >= VLIB_FRAME_SIZE || (allocate_new_next_frame && n_used > 0) || |
| 421 | (f->frame_flags & VLIB_FRAME_NO_APPEND)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 422 | { |
| 423 | /* Old frame may need to be freed after dispatch, since we'll have |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 424 | two redundant frames from node -> next node. */ |
| 425 | if (!(nf->flags & VLIB_FRAME_NO_FREE_AFTER_DISPATCH)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 426 | { |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 427 | vlib_frame_t *f_old = vlib_get_frame (vm, nf->frame); |
Damjan Marion | 633b6fd | 2018-09-14 14:38:53 +0200 | [diff] [blame] | 428 | f_old->frame_flags |= VLIB_FRAME_FREE_AFTER_DISPATCH; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 429 | } |
| 430 | |
| 431 | /* Allocate new frame to replace full one. */ |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 432 | f = nf->frame = vlib_frame_alloc (vm, node, next_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 433 | n_used = f->n_vectors; |
| 434 | } |
| 435 | |
| 436 | /* Should have free vectors in frame now. */ |
| 437 | ASSERT (n_used < VLIB_FRAME_SIZE); |
| 438 | |
| 439 | if (CLIB_DEBUG > 0) |
| 440 | { |
| 441 | validate_frame_magic (vm, f, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 442 | vlib_get_node (vm, node->node_index), next_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 443 | } |
| 444 | |
| 445 | return f; |
| 446 | } |
| 447 | |
| 448 | static void |
| 449 | vlib_put_next_frame_validate (vlib_main_t * vm, |
| 450 | vlib_node_runtime_t * rt, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 451 | u32 next_index, u32 n_vectors_left) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 452 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 453 | vlib_node_main_t *nm = &vm->node_main; |
| 454 | vlib_next_frame_t *nf; |
| 455 | vlib_frame_t *f; |
| 456 | vlib_node_runtime_t *next_rt; |
| 457 | vlib_node_t *next_node; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 458 | u32 n_before, n_after; |
| 459 | |
| 460 | nf = vlib_node_runtime_get_next_frame (vm, rt, next_index); |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 461 | f = vlib_get_frame (vm, nf->frame); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 462 | |
| 463 | ASSERT (n_vectors_left <= VLIB_FRAME_SIZE); |
Dave Barach | c74b43c | 2020-04-09 17:24:07 -0400 | [diff] [blame] | 464 | |
| 465 | vlib_validate_frame_indices (f); |
| 466 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 467 | n_after = VLIB_FRAME_SIZE - n_vectors_left; |
| 468 | n_before = f->n_vectors; |
| 469 | |
| 470 | ASSERT (n_after >= n_before); |
| 471 | |
| 472 | next_rt = vec_elt_at_index (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL], |
| 473 | nf->node_runtime_index); |
| 474 | next_node = vlib_get_node (vm, next_rt->node_index); |
| 475 | if (n_after > 0 && next_node->validate_frame) |
| 476 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 477 | u8 *msg = next_node->validate_frame (vm, rt, f); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 478 | if (msg) |
| 479 | { |
| 480 | clib_warning ("%v", msg); |
| 481 | ASSERT (0); |
| 482 | } |
| 483 | vec_free (msg); |
| 484 | } |
| 485 | } |
| 486 | |
| 487 | void |
| 488 | vlib_put_next_frame (vlib_main_t * vm, |
| 489 | vlib_node_runtime_t * r, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 490 | u32 next_index, u32 n_vectors_left) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 491 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 492 | vlib_node_main_t *nm = &vm->node_main; |
| 493 | vlib_next_frame_t *nf; |
| 494 | vlib_frame_t *f; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 495 | u32 n_vectors_in_frame; |
| 496 | |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 497 | if (CLIB_DEBUG > 0) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 498 | vlib_put_next_frame_validate (vm, r, next_index, n_vectors_left); |
| 499 | |
| 500 | nf = vlib_node_runtime_get_next_frame (vm, r, next_index); |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 501 | f = vlib_get_frame (vm, nf->frame); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 502 | |
| 503 | /* Make sure that magic number is still there. Otherwise, caller |
| 504 | has overrun frame meta data. */ |
| 505 | if (CLIB_DEBUG > 0) |
| 506 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 507 | vlib_node_t *node = vlib_get_node (vm, r->node_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 508 | validate_frame_magic (vm, f, node, next_index); |
| 509 | } |
| 510 | |
| 511 | /* Convert # of vectors left -> number of vectors there. */ |
| 512 | ASSERT (n_vectors_left <= VLIB_FRAME_SIZE); |
| 513 | n_vectors_in_frame = VLIB_FRAME_SIZE - n_vectors_left; |
| 514 | |
| 515 | f->n_vectors = n_vectors_in_frame; |
| 516 | |
| 517 | /* If vectors were added to frame, add to pending vector. */ |
| 518 | if (PREDICT_TRUE (n_vectors_in_frame > 0)) |
| 519 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 520 | vlib_pending_frame_t *p; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 521 | u32 v0, v1; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 522 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 523 | r->cached_next_index = next_index; |
| 524 | |
Damjan Marion | 633b6fd | 2018-09-14 14:38:53 +0200 | [diff] [blame] | 525 | if (!(f->frame_flags & VLIB_FRAME_PENDING)) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 526 | { |
| 527 | __attribute__ ((unused)) vlib_node_t *node; |
| 528 | vlib_node_t *next_node; |
| 529 | vlib_node_runtime_t *next_runtime; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 530 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 531 | node = vlib_get_node (vm, r->node_index); |
| 532 | next_node = vlib_get_next_node (vm, r->node_index, next_index); |
| 533 | next_runtime = vlib_node_get_runtime (vm, next_node->index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 534 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 535 | vec_add2 (nm->pending_frames, p, 1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 536 | |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 537 | p->frame = nf->frame; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 538 | p->node_runtime_index = nf->node_runtime_index; |
| 539 | p->next_frame_index = nf - nm->next_frames; |
| 540 | nf->flags |= VLIB_FRAME_PENDING; |
Damjan Marion | 633b6fd | 2018-09-14 14:38:53 +0200 | [diff] [blame] | 541 | f->frame_flags |= VLIB_FRAME_PENDING; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 542 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 543 | /* |
| 544 | * If we're going to dispatch this frame on another thread, |
| 545 | * force allocation of a new frame. Otherwise, we create |
| 546 | * a dangling frame reference. Each thread has its own copy of |
| 547 | * the next_frames vector. |
| 548 | */ |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 549 | if (0 && r->thread_index != next_runtime->thread_index) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 550 | { |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 551 | nf->frame = NULL; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 552 | nf->flags &= ~(VLIB_FRAME_PENDING | VLIB_FRAME_IS_ALLOCATED); |
| 553 | } |
| 554 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 555 | |
| 556 | /* Copy trace flag from next_frame and from runtime. */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 557 | nf->flags |= |
| 558 | (nf->flags & VLIB_NODE_FLAG_TRACE) | (r-> |
| 559 | flags & VLIB_NODE_FLAG_TRACE); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 560 | |
| 561 | v0 = nf->vectors_since_last_overflow; |
| 562 | v1 = v0 + n_vectors_in_frame; |
| 563 | nf->vectors_since_last_overflow = v1; |
| 564 | if (PREDICT_FALSE (v1 < v0)) |
| 565 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 566 | vlib_node_t *node = vlib_get_node (vm, r->node_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 567 | vec_elt (node->n_vectors_by_next_node, next_index) += v0; |
| 568 | } |
| 569 | } |
| 570 | } |
| 571 | |
| 572 | /* Sync up runtime (32 bit counters) and main node stats (64 bit counters). */ |
| 573 | never_inline void |
| 574 | vlib_node_runtime_sync_stats (vlib_main_t * vm, |
| 575 | vlib_node_runtime_t * r, |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 576 | uword n_calls, uword n_vectors, uword n_clocks) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 577 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 578 | vlib_node_t *n = vlib_get_node (vm, r->node_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 579 | |
| 580 | n->stats_total.calls += n_calls + r->calls_since_last_overflow; |
| 581 | n->stats_total.vectors += n_vectors + r->vectors_since_last_overflow; |
| 582 | n->stats_total.clocks += n_clocks + r->clocks_since_last_overflow; |
| 583 | n->stats_total.max_clock = r->max_clock; |
| 584 | n->stats_total.max_clock_n = r->max_clock_n; |
| 585 | |
| 586 | r->calls_since_last_overflow = 0; |
| 587 | r->vectors_since_last_overflow = 0; |
| 588 | r->clocks_since_last_overflow = 0; |
| 589 | } |
| 590 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 591 | always_inline void __attribute__ ((unused)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 592 | vlib_process_sync_stats (vlib_main_t * vm, |
| 593 | vlib_process_t * p, |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 594 | uword n_calls, uword n_vectors, uword n_clocks) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 595 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 596 | vlib_node_runtime_t *rt = &p->node_runtime; |
| 597 | vlib_node_t *n = vlib_get_node (vm, rt->node_index); |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 598 | vlib_node_runtime_sync_stats (vm, rt, n_calls, n_vectors, n_clocks); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 599 | n->stats_total.suspends += p->n_suspends; |
| 600 | p->n_suspends = 0; |
| 601 | } |
| 602 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 603 | void |
| 604 | vlib_node_sync_stats (vlib_main_t * vm, vlib_node_t * n) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 605 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 606 | vlib_node_runtime_t *rt; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 607 | |
| 608 | if (n->type == VLIB_NODE_TYPE_PROCESS) |
| 609 | { |
| 610 | /* Nothing to do for PROCESS nodes except in main thread */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 611 | if (vm != &vlib_global_main) |
| 612 | return; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 613 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 614 | vlib_process_t *p = vlib_get_process_from_node (vm, n); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 615 | n->stats_total.suspends += p->n_suspends; |
| 616 | p->n_suspends = 0; |
| 617 | rt = &p->node_runtime; |
| 618 | } |
| 619 | else |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 620 | rt = |
| 621 | vec_elt_at_index (vm->node_main.nodes_by_type[n->type], |
| 622 | n->runtime_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 623 | |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 624 | vlib_node_runtime_sync_stats (vm, rt, 0, 0, 0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 625 | |
| 626 | /* Sync up runtime next frame vector counters with main node structure. */ |
| 627 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 628 | vlib_next_frame_t *nf; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 629 | uword i; |
| 630 | for (i = 0; i < rt->n_next_nodes; i++) |
| 631 | { |
| 632 | nf = vlib_node_runtime_get_next_frame (vm, rt, i); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 633 | vec_elt (n->n_vectors_by_next_node, i) += |
| 634 | nf->vectors_since_last_overflow; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 635 | nf->vectors_since_last_overflow = 0; |
| 636 | } |
| 637 | } |
| 638 | } |
| 639 | |
| 640 | always_inline u32 |
| 641 | vlib_node_runtime_update_stats (vlib_main_t * vm, |
| 642 | vlib_node_runtime_t * node, |
| 643 | uword n_calls, |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 644 | uword n_vectors, uword n_clocks) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 645 | { |
| 646 | u32 ca0, ca1, v0, v1, cl0, cl1, r; |
| 647 | |
| 648 | cl0 = cl1 = node->clocks_since_last_overflow; |
| 649 | ca0 = ca1 = node->calls_since_last_overflow; |
| 650 | v0 = v1 = node->vectors_since_last_overflow; |
| 651 | |
| 652 | ca1 = ca0 + n_calls; |
| 653 | v1 = v0 + n_vectors; |
| 654 | cl1 = cl0 + n_clocks; |
| 655 | |
| 656 | node->calls_since_last_overflow = ca1; |
| 657 | node->clocks_since_last_overflow = cl1; |
| 658 | node->vectors_since_last_overflow = v1; |
Dave Barach | 4d1a866 | 2018-09-10 12:31:15 -0400 | [diff] [blame] | 659 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 660 | node->max_clock_n = node->max_clock > n_clocks ? |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 661 | node->max_clock_n : n_vectors; |
| 662 | node->max_clock = node->max_clock > n_clocks ? node->max_clock : n_clocks; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 663 | |
| 664 | r = vlib_node_runtime_update_main_loop_vector_stats (vm, node, n_vectors); |
| 665 | |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 666 | if (PREDICT_FALSE (ca1 < ca0 || v1 < v0 || cl1 < cl0)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 667 | { |
| 668 | node->calls_since_last_overflow = ca0; |
| 669 | node->clocks_since_last_overflow = cl0; |
| 670 | node->vectors_since_last_overflow = v0; |
Dave Barach | 4d1a866 | 2018-09-10 12:31:15 -0400 | [diff] [blame] | 671 | |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 672 | vlib_node_runtime_sync_stats (vm, node, n_calls, n_vectors, n_clocks); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 673 | } |
| 674 | |
| 675 | return r; |
| 676 | } |
| 677 | |
Dave Barach | 17e5d80 | 2019-05-01 11:30:13 -0400 | [diff] [blame] | 678 | always_inline void |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 679 | vlib_process_update_stats (vlib_main_t * vm, |
| 680 | vlib_process_t * p, |
Dave Barach | ec595ef | 2019-01-24 10:34:24 -0500 | [diff] [blame] | 681 | uword n_calls, uword n_vectors, uword n_clocks) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 682 | { |
| 683 | vlib_node_runtime_update_stats (vm, &p->node_runtime, |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 684 | n_calls, n_vectors, n_clocks); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 685 | } |
| 686 | |
| 687 | static clib_error_t * |
| 688 | vlib_cli_elog_clear (vlib_main_t * vm, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 689 | unformat_input_t * input, vlib_cli_command_t * cmd) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 690 | { |
| 691 | elog_reset_buffer (&vm->elog_main); |
| 692 | return 0; |
| 693 | } |
| 694 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 695 | /* *INDENT-OFF* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 696 | VLIB_CLI_COMMAND (elog_clear_cli, static) = { |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 697 | .path = "event-logger clear", |
| 698 | .short_help = "Clear the event log", |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 699 | .function = vlib_cli_elog_clear, |
| 700 | }; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 701 | /* *INDENT-ON* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 702 | |
| 703 | #ifdef CLIB_UNIX |
| 704 | static clib_error_t * |
| 705 | elog_save_buffer (vlib_main_t * vm, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 706 | unformat_input_t * input, vlib_cli_command_t * cmd) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 707 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 708 | elog_main_t *em = &vm->elog_main; |
| 709 | char *file, *chroot_file; |
| 710 | clib_error_t *error = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 711 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 712 | if (!unformat (input, "%s", &file)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 713 | { |
| 714 | vlib_cli_output (vm, "expected file name, got `%U'", |
| 715 | format_unformat_error, input); |
| 716 | return 0; |
| 717 | } |
| 718 | |
| 719 | /* It's fairly hard to get "../oopsie" through unformat; just in case */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 720 | if (strstr (file, "..") || index (file, '/')) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 721 | { |
| 722 | vlib_cli_output (vm, "illegal characters in filename '%s'", file); |
| 723 | return 0; |
| 724 | } |
| 725 | |
| 726 | chroot_file = (char *) format (0, "/tmp/%s%c", file, 0); |
| 727 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 728 | vec_free (file); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 729 | |
| 730 | vlib_cli_output (vm, "Saving %wd of %wd events to %s", |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 731 | elog_n_events_in_buffer (em), |
| 732 | elog_buffer_capacity (em), chroot_file); |
| 733 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 734 | vlib_worker_thread_barrier_sync (vm); |
Dave Barach | 903fd51 | 2017-04-01 11:07:40 -0400 | [diff] [blame] | 735 | error = elog_write_file (em, chroot_file, 1 /* flush ring */ ); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 736 | vlib_worker_thread_barrier_release (vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 737 | vec_free (chroot_file); |
| 738 | return error; |
| 739 | } |
| 740 | |
Dave Barach | 8148131 | 2017-05-16 09:08:14 -0400 | [diff] [blame] | 741 | void |
Dave Barach | 27d978c | 2020-11-03 09:59:06 -0500 | [diff] [blame] | 742 | vlib_post_mortem_dump (void) |
Dave Barach | 8148131 | 2017-05-16 09:08:14 -0400 | [diff] [blame] | 743 | { |
| 744 | vlib_main_t *vm = &vlib_global_main; |
| 745 | elog_main_t *em = &vm->elog_main; |
Dave Barach | 27d978c | 2020-11-03 09:59:06 -0500 | [diff] [blame] | 746 | |
Dave Barach | 8148131 | 2017-05-16 09:08:14 -0400 | [diff] [blame] | 747 | u8 *filename; |
| 748 | clib_error_t *error; |
| 749 | |
Dave Barach | 27d978c | 2020-11-03 09:59:06 -0500 | [diff] [blame] | 750 | if ((vm->elog_post_mortem_dump + vm->dispatch_pcap_postmortem) == 0) |
Dave Barach | 8148131 | 2017-05-16 09:08:14 -0400 | [diff] [blame] | 751 | return; |
| 752 | |
Dave Barach | 27d978c | 2020-11-03 09:59:06 -0500 | [diff] [blame] | 753 | if (vm->dispatch_pcap_postmortem) |
| 754 | { |
| 755 | clib_error_t *error; |
| 756 | pcap_main_t *pm = &vm->dispatch_pcap_main; |
| 757 | |
| 758 | pm->n_packets_to_capture = pm->n_packets_captured; |
| 759 | pm->file_name = (char *) format (0, "/tmp/dispatch_post_mortem.%d%c", |
| 760 | getpid (), 0); |
| 761 | error = pcap_write (pm); |
| 762 | pcap_close (pm); |
| 763 | if (error) |
| 764 | clib_error_report (error); |
| 765 | /* |
| 766 | * We're in the middle of crashing. Don't try to free the filename. |
| 767 | */ |
| 768 | } |
| 769 | |
| 770 | if (vm->elog_post_mortem_dump) |
| 771 | { |
| 772 | filename = format (0, "/tmp/elog_post_mortem.%d%c", getpid (), 0); |
| 773 | error = elog_write_file (em, (char *) filename, 1 /* flush ring */ ); |
| 774 | if (error) |
| 775 | clib_error_report (error); |
| 776 | /* |
| 777 | * We're in the middle of crashing. Don't try to free the filename. |
| 778 | */ |
| 779 | } |
Dave Barach | 8148131 | 2017-05-16 09:08:14 -0400 | [diff] [blame] | 780 | } |
| 781 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 782 | /* *INDENT-OFF* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 783 | VLIB_CLI_COMMAND (elog_save_cli, static) = { |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 784 | .path = "event-logger save", |
| 785 | .short_help = "event-logger save <filename> (saves log in /tmp/<filename>)", |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 786 | .function = elog_save_buffer, |
| 787 | }; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 788 | /* *INDENT-ON* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 789 | |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 790 | static clib_error_t * |
| 791 | elog_stop (vlib_main_t * vm, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 792 | unformat_input_t * input, vlib_cli_command_t * cmd) |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 793 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 794 | elog_main_t *em = &vm->elog_main; |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 795 | |
| 796 | em->n_total_events_disable_limit = em->n_total_events; |
| 797 | |
| 798 | vlib_cli_output (vm, "Stopped the event logger..."); |
| 799 | return 0; |
| 800 | } |
| 801 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 802 | /* *INDENT-OFF* */ |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 803 | VLIB_CLI_COMMAND (elog_stop_cli, static) = { |
| 804 | .path = "event-logger stop", |
| 805 | .short_help = "Stop the event-logger", |
| 806 | .function = elog_stop, |
| 807 | }; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 808 | /* *INDENT-ON* */ |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 809 | |
| 810 | static clib_error_t * |
| 811 | elog_restart (vlib_main_t * vm, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 812 | unformat_input_t * input, vlib_cli_command_t * cmd) |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 813 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 814 | elog_main_t *em = &vm->elog_main; |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 815 | |
| 816 | em->n_total_events_disable_limit = ~0; |
| 817 | |
| 818 | vlib_cli_output (vm, "Restarted the event logger..."); |
| 819 | return 0; |
| 820 | } |
| 821 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 822 | /* *INDENT-OFF* */ |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 823 | VLIB_CLI_COMMAND (elog_restart_cli, static) = { |
| 824 | .path = "event-logger restart", |
| 825 | .short_help = "Restart the event-logger", |
| 826 | .function = elog_restart, |
| 827 | }; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 828 | /* *INDENT-ON* */ |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 829 | |
| 830 | static clib_error_t * |
Dave Barach | bc867c3 | 2020-11-25 10:07:09 -0500 | [diff] [blame] | 831 | elog_resize_command_fn (vlib_main_t * vm, |
| 832 | unformat_input_t * input, vlib_cli_command_t * cmd) |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 833 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 834 | elog_main_t *em = &vm->elog_main; |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 835 | u32 tmp; |
| 836 | |
| 837 | /* Stop the parade */ |
| 838 | elog_reset_buffer (&vm->elog_main); |
| 839 | |
| 840 | if (unformat (input, "%d", &tmp)) |
| 841 | { |
| 842 | elog_alloc (em, tmp); |
| 843 | em->n_total_events_disable_limit = ~0; |
| 844 | } |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 845 | else |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 846 | return clib_error_return (0, "Must specify how many events in the ring"); |
| 847 | |
| 848 | vlib_cli_output (vm, "Resized ring and restarted the event logger..."); |
| 849 | return 0; |
| 850 | } |
| 851 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 852 | /* *INDENT-OFF* */ |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 853 | VLIB_CLI_COMMAND (elog_resize_cli, static) = { |
| 854 | .path = "event-logger resize", |
| 855 | .short_help = "event-logger resize <nnn>", |
Dave Barach | bc867c3 | 2020-11-25 10:07:09 -0500 | [diff] [blame] | 856 | .function = elog_resize_command_fn, |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 857 | }; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 858 | /* *INDENT-ON* */ |
Dave Barach | e5389bb | 2016-03-28 17:12:19 -0400 | [diff] [blame] | 859 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 860 | #endif /* CLIB_UNIX */ |
| 861 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 862 | static void |
| 863 | elog_show_buffer_internal (vlib_main_t * vm, u32 n_events_to_show) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 864 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 865 | elog_main_t *em = &vm->elog_main; |
| 866 | elog_event_t *e, *es; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 867 | f64 dt; |
| 868 | |
| 869 | /* Show events in VLIB time since log clock starts after VLIB clock. */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 870 | dt = (em->init_time.cpu - vm->clib_time.init_cpu_time) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 871 | * vm->clib_time.seconds_per_clock; |
| 872 | |
| 873 | es = elog_peek_events (em); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 874 | vlib_cli_output (vm, "%d of %d events in buffer, logger %s", vec_len (es), |
| 875 | em->event_ring_size, |
| 876 | em->n_total_events < em->n_total_events_disable_limit ? |
| 877 | "running" : "stopped"); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 878 | vec_foreach (e, es) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 879 | { |
| 880 | vlib_cli_output (vm, "%18.9f: %U", |
| 881 | e->time + dt, format_elog_event, em, e); |
| 882 | n_events_to_show--; |
| 883 | if (n_events_to_show == 0) |
| 884 | break; |
| 885 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 886 | vec_free (es); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 887 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 888 | } |
| 889 | |
| 890 | static clib_error_t * |
| 891 | elog_show_buffer (vlib_main_t * vm, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 892 | unformat_input_t * input, vlib_cli_command_t * cmd) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 893 | { |
| 894 | u32 n_events_to_show; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 895 | clib_error_t *error = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 896 | |
| 897 | n_events_to_show = 250; |
| 898 | while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) |
| 899 | { |
| 900 | if (unformat (input, "%d", &n_events_to_show)) |
| 901 | ; |
| 902 | else if (unformat (input, "all")) |
| 903 | n_events_to_show = ~0; |
| 904 | else |
| 905 | return unformat_parse_error (input); |
| 906 | } |
| 907 | elog_show_buffer_internal (vm, n_events_to_show); |
| 908 | return error; |
| 909 | } |
| 910 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 911 | /* *INDENT-OFF* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 912 | VLIB_CLI_COMMAND (elog_show_cli, static) = { |
| 913 | .path = "show event-logger", |
| 914 | .short_help = "Show event logger info", |
| 915 | .function = elog_show_buffer, |
| 916 | }; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 917 | /* *INDENT-ON* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 918 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 919 | void |
| 920 | vlib_gdb_show_event_log (void) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 921 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 922 | elog_show_buffer_internal (vlib_get_main (), (u32) ~ 0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 923 | } |
| 924 | |
Dave Barach | fb6e59d | 2016-03-26 18:45:42 -0400 | [diff] [blame] | 925 | static inline void |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 926 | vlib_elog_main_loop_event (vlib_main_t * vm, |
| 927 | u32 node_index, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 928 | u64 time, u32 n_vectors, u32 is_return) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 929 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 930 | vlib_main_t *evm = &vlib_global_main; |
| 931 | elog_main_t *em = &evm->elog_main; |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 932 | int enabled = evm->elog_trace_graph_dispatch | |
| 933 | evm->elog_trace_graph_circuit; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 934 | |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 935 | if (PREDICT_FALSE (enabled && n_vectors)) |
| 936 | { |
| 937 | if (PREDICT_FALSE (!elog_is_enabled (em))) |
| 938 | { |
| 939 | evm->elog_trace_graph_dispatch = 0; |
| 940 | evm->elog_trace_graph_circuit = 0; |
| 941 | return; |
| 942 | } |
| 943 | if (PREDICT_TRUE |
| 944 | (evm->elog_trace_graph_dispatch || |
| 945 | (evm->elog_trace_graph_circuit && |
| 946 | node_index == evm->elog_trace_graph_circuit_node_index))) |
| 947 | { |
| 948 | elog_track (em, |
| 949 | /* event type */ |
| 950 | vec_elt_at_index (is_return |
| 951 | ? evm->node_return_elog_event_types |
| 952 | : evm->node_call_elog_event_types, |
| 953 | node_index), |
| 954 | /* track */ |
| 955 | (vm->thread_index ? |
| 956 | &vlib_worker_threads[vm->thread_index].elog_track |
| 957 | : &em->default_track), |
| 958 | /* data to log */ n_vectors); |
| 959 | } |
| 960 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 961 | } |
| 962 | |
Dave Barach | 7bee773 | 2017-10-18 18:48:11 -0400 | [diff] [blame] | 963 | #if VLIB_BUFFER_TRACE_TRAJECTORY > 0 |
| 964 | void (*vlib_buffer_trace_trajectory_cb) (vlib_buffer_t * b, u32 node_index); |
| 965 | void (*vlib_buffer_trace_trajectory_init_cb) (vlib_buffer_t * b); |
| 966 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 967 | void |
Dave Barach | 7bee773 | 2017-10-18 18:48:11 -0400 | [diff] [blame] | 968 | vlib_buffer_trace_trajectory_init (vlib_buffer_t * b) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 969 | { |
Dave Barach | 7bee773 | 2017-10-18 18:48:11 -0400 | [diff] [blame] | 970 | if (PREDICT_TRUE (vlib_buffer_trace_trajectory_init_cb != 0)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 971 | { |
Dave Barach | 7bee773 | 2017-10-18 18:48:11 -0400 | [diff] [blame] | 972 | (*vlib_buffer_trace_trajectory_init_cb) (b); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 973 | } |
| 974 | } |
| 975 | |
Dave Barach | 7bee773 | 2017-10-18 18:48:11 -0400 | [diff] [blame] | 976 | #endif |
| 977 | |
| 978 | static inline void |
| 979 | add_trajectory_trace (vlib_buffer_t * b, u32 node_index) |
| 980 | { |
| 981 | #if VLIB_BUFFER_TRACE_TRAJECTORY > 0 |
| 982 | if (PREDICT_TRUE (vlib_buffer_trace_trajectory_cb != 0)) |
| 983 | { |
| 984 | (*vlib_buffer_trace_trajectory_cb) (b, node_index); |
| 985 | } |
| 986 | #endif |
| 987 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 988 | |
Dave Barach | 7fff3d2 | 2018-11-27 16:52:59 -0500 | [diff] [blame] | 989 | u8 *format_vnet_buffer_flags (u8 * s, va_list * args) __attribute__ ((weak)); |
| 990 | u8 * |
| 991 | format_vnet_buffer_flags (u8 * s, va_list * args) |
| 992 | { |
| 993 | s = format (s, "BUG STUB %s", __FUNCTION__); |
| 994 | return s; |
| 995 | } |
| 996 | |
| 997 | u8 *format_vnet_buffer_opaque (u8 * s, va_list * args) __attribute__ ((weak)); |
| 998 | u8 * |
| 999 | format_vnet_buffer_opaque (u8 * s, va_list * args) |
| 1000 | { |
| 1001 | s = format (s, "BUG STUB %s", __FUNCTION__); |
| 1002 | return s; |
| 1003 | } |
| 1004 | |
| 1005 | u8 *format_vnet_buffer_opaque2 (u8 * s, va_list * args) |
| 1006 | __attribute__ ((weak)); |
| 1007 | u8 * |
| 1008 | format_vnet_buffer_opaque2 (u8 * s, va_list * args) |
| 1009 | { |
| 1010 | s = format (s, "BUG STUB %s", __FUNCTION__); |
| 1011 | return s; |
| 1012 | } |
| 1013 | |
| 1014 | static u8 * |
| 1015 | format_buffer_metadata (u8 * s, va_list * args) |
| 1016 | { |
| 1017 | vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *); |
| 1018 | |
| 1019 | s = format (s, "flags: %U\n", format_vnet_buffer_flags, b); |
| 1020 | s = format (s, "current_data: %d, current_length: %d\n", |
| 1021 | (i32) (b->current_data), (i32) (b->current_length)); |
Dave Barach | d7b3066 | 2019-10-24 18:10:10 -0400 | [diff] [blame] | 1022 | s = format |
| 1023 | (s, |
| 1024 | "current_config_index/punt_reason: %d, flow_id: %x, next_buffer: %x\n", |
| 1025 | b->current_config_index, b->flow_id, b->next_buffer); |
| 1026 | s = |
| 1027 | format (s, "error: %d, ref_count: %d, buffer_pool_index: %d\n", |
| 1028 | (u32) (b->error), (u32) (b->ref_count), |
| 1029 | (u32) (b->buffer_pool_index)); |
| 1030 | s = |
| 1031 | format (s, "trace_handle: 0x%x, len_not_first_buf: %d\n", b->trace_handle, |
| 1032 | b->total_length_not_including_first_buffer); |
Dave Barach | 7fff3d2 | 2018-11-27 16:52:59 -0500 | [diff] [blame] | 1033 | return s; |
| 1034 | } |
| 1035 | |
| 1036 | #define A(x) vec_add1(vm->pcap_buffer, (x)) |
| 1037 | |
Dave Barach | 3ae2873 | 2018-11-16 17:19:00 -0500 | [diff] [blame] | 1038 | static void |
| 1039 | dispatch_pcap_trace (vlib_main_t * vm, |
| 1040 | vlib_node_runtime_t * node, vlib_frame_t * frame) |
| 1041 | { |
| 1042 | int i; |
| 1043 | vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **bufp, *b; |
Dave Barach | 349cd1a | 2019-10-18 14:44:05 -0400 | [diff] [blame] | 1044 | pcap_main_t *pm = &vlib_global_main.dispatch_pcap_main; |
Dave Barach | 1201a80 | 2018-11-20 12:08:39 -0500 | [diff] [blame] | 1045 | vlib_trace_main_t *tm = &vm->trace_main; |
Dave Barach | 3ae2873 | 2018-11-16 17:19:00 -0500 | [diff] [blame] | 1046 | u32 capture_size; |
| 1047 | vlib_node_t *n; |
| 1048 | i32 n_left; |
| 1049 | f64 time_now = vlib_time_now (vm); |
| 1050 | u32 *from; |
Dave Barach | 3ae2873 | 2018-11-16 17:19:00 -0500 | [diff] [blame] | 1051 | u8 *d; |
Dave Barach | 7fff3d2 | 2018-11-27 16:52:59 -0500 | [diff] [blame] | 1052 | u8 string_count; |
Dave Barach | 3ae2873 | 2018-11-16 17:19:00 -0500 | [diff] [blame] | 1053 | |
| 1054 | /* Input nodes don't have frames yet */ |
| 1055 | if (frame == 0 || frame->n_vectors == 0) |
| 1056 | return; |
| 1057 | |
| 1058 | from = vlib_frame_vector_args (frame); |
| 1059 | vlib_get_buffers (vm, from, bufs, frame->n_vectors); |
| 1060 | bufp = bufs; |
| 1061 | |
Dave Barach | 3ae2873 | 2018-11-16 17:19:00 -0500 | [diff] [blame] | 1062 | n = vlib_get_node (vm, node->node_index); |
Dave Barach | 3ae2873 | 2018-11-16 17:19:00 -0500 | [diff] [blame] | 1063 | |
| 1064 | for (i = 0; i < frame->n_vectors; i++) |
| 1065 | { |
| 1066 | if (PREDICT_TRUE (pm->n_packets_captured < pm->n_packets_to_capture)) |
| 1067 | { |
| 1068 | b = bufp[i]; |
| 1069 | |
Dave Barach | 7fff3d2 | 2018-11-27 16:52:59 -0500 | [diff] [blame] | 1070 | vec_reset_length (vm->pcap_buffer); |
| 1071 | string_count = 0; |
Dave Barach | 1201a80 | 2018-11-20 12:08:39 -0500 | [diff] [blame] | 1072 | |
Dave Barach | 7fff3d2 | 2018-11-27 16:52:59 -0500 | [diff] [blame] | 1073 | /* Version, flags */ |
| 1074 | A ((u8) VLIB_PCAP_MAJOR_VERSION); |
| 1075 | A ((u8) VLIB_PCAP_MINOR_VERSION); |
| 1076 | A (0 /* string_count */ ); |
| 1077 | A (n->protocol_hint); |
| 1078 | |
| 1079 | /* Buffer index (big endian) */ |
| 1080 | A ((from[i] >> 24) & 0xff); |
| 1081 | A ((from[i] >> 16) & 0xff); |
| 1082 | A ((from[i] >> 8) & 0xff); |
| 1083 | A ((from[i] >> 0) & 0xff); |
| 1084 | |
| 1085 | /* Node name, NULL-terminated ASCII */ |
| 1086 | vm->pcap_buffer = format (vm->pcap_buffer, "%v%c", n->name, 0); |
| 1087 | string_count++; |
| 1088 | |
| 1089 | vm->pcap_buffer = format (vm->pcap_buffer, "%U%c", |
| 1090 | format_buffer_metadata, b, 0); |
| 1091 | string_count++; |
| 1092 | vm->pcap_buffer = format (vm->pcap_buffer, "%U%c", |
| 1093 | format_vnet_buffer_opaque, b, 0); |
| 1094 | string_count++; |
| 1095 | vm->pcap_buffer = format (vm->pcap_buffer, "%U%c", |
| 1096 | format_vnet_buffer_opaque2, b, 0); |
| 1097 | string_count++; |
Dave Barach | 1201a80 | 2018-11-20 12:08:39 -0500 | [diff] [blame] | 1098 | |
| 1099 | /* Is this packet traced? */ |
| 1100 | if (PREDICT_FALSE (b->flags & VLIB_BUFFER_IS_TRACED)) |
| 1101 | { |
| 1102 | vlib_trace_header_t **h |
Dave Barach | a638c18 | 2019-06-21 18:24:07 -0400 | [diff] [blame] | 1103 | = pool_elt_at_index (tm->trace_buffer_pool, |
| 1104 | vlib_buffer_get_trace_index (b)); |
Dave Barach | 1201a80 | 2018-11-20 12:08:39 -0500 | [diff] [blame] | 1105 | |
Dave Barach | 7fff3d2 | 2018-11-27 16:52:59 -0500 | [diff] [blame] | 1106 | vm->pcap_buffer = format (vm->pcap_buffer, "%U%c", |
| 1107 | format_vlib_trace, vm, h[0], 0); |
| 1108 | string_count++; |
Dave Barach | 1201a80 | 2018-11-20 12:08:39 -0500 | [diff] [blame] | 1109 | } |
| 1110 | |
Dave Barach | 7fff3d2 | 2018-11-27 16:52:59 -0500 | [diff] [blame] | 1111 | /* Save the string count */ |
| 1112 | vm->pcap_buffer[2] = string_count; |
| 1113 | |
| 1114 | /* Figure out how many bytes in the pcap trace */ |
| 1115 | capture_size = vec_len (vm->pcap_buffer) + |
| 1116 | +vlib_buffer_length_in_chain (vm, b); |
Dave Barach | 3ae2873 | 2018-11-16 17:19:00 -0500 | [diff] [blame] | 1117 | |
| 1118 | clib_spinlock_lock_if_init (&pm->lock); |
Dave Barach | 1201a80 | 2018-11-20 12:08:39 -0500 | [diff] [blame] | 1119 | n_left = clib_min (capture_size, 16384); |
Dave Barach | 3ae2873 | 2018-11-16 17:19:00 -0500 | [diff] [blame] | 1120 | d = pcap_add_packet (pm, time_now, n_left, capture_size); |
| 1121 | |
Dave Barach | 7fff3d2 | 2018-11-27 16:52:59 -0500 | [diff] [blame] | 1122 | /* Copy the header */ |
| 1123 | clib_memcpy_fast (d, vm->pcap_buffer, vec_len (vm->pcap_buffer)); |
| 1124 | d += vec_len (vm->pcap_buffer); |
Dave Barach | 1201a80 | 2018-11-20 12:08:39 -0500 | [diff] [blame] | 1125 | |
| 1126 | n_left = clib_min |
| 1127 | (vlib_buffer_length_in_chain (vm, b), |
Dave Barach | 7fff3d2 | 2018-11-27 16:52:59 -0500 | [diff] [blame] | 1128 | (16384 - vec_len (vm->pcap_buffer))); |
Dave Barach | 3ae2873 | 2018-11-16 17:19:00 -0500 | [diff] [blame] | 1129 | /* Copy the packet data */ |
| 1130 | while (1) |
| 1131 | { |
| 1132 | u32 copy_length = clib_min ((u32) n_left, b->current_length); |
| 1133 | clib_memcpy_fast (d, b->data + b->current_data, copy_length); |
| 1134 | n_left -= b->current_length; |
| 1135 | if (n_left <= 0) |
| 1136 | break; |
| 1137 | d += b->current_length; |
| 1138 | ASSERT (b->flags & VLIB_BUFFER_NEXT_PRESENT); |
| 1139 | b = vlib_get_buffer (vm, b->next_buffer); |
| 1140 | } |
| 1141 | clib_spinlock_unlock_if_init (&pm->lock); |
| 1142 | } |
| 1143 | } |
| 1144 | } |
| 1145 | |
Damjan Marion | 9a332e1 | 2017-03-28 15:11:20 +0200 | [diff] [blame] | 1146 | static_always_inline u64 |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1147 | dispatch_node (vlib_main_t * vm, |
| 1148 | vlib_node_runtime_t * node, |
| 1149 | vlib_node_type_t type, |
| 1150 | vlib_node_state_t dispatch_state, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1151 | vlib_frame_t * frame, u64 last_time_stamp) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1152 | { |
| 1153 | uword n, v; |
| 1154 | u64 t; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1155 | vlib_node_main_t *nm = &vm->node_main; |
| 1156 | vlib_next_frame_t *nf; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1157 | |
| 1158 | if (CLIB_DEBUG > 0) |
| 1159 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1160 | vlib_node_t *n = vlib_get_node (vm, node->node_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1161 | ASSERT (n->type == type); |
| 1162 | } |
| 1163 | |
| 1164 | /* Only non-internal nodes may be disabled. */ |
| 1165 | if (type != VLIB_NODE_TYPE_INTERNAL && node->state != dispatch_state) |
| 1166 | { |
| 1167 | ASSERT (type != VLIB_NODE_TYPE_INTERNAL); |
| 1168 | return last_time_stamp; |
| 1169 | } |
| 1170 | |
| 1171 | if ((type == VLIB_NODE_TYPE_PRE_INPUT || type == VLIB_NODE_TYPE_INPUT) |
| 1172 | && dispatch_state != VLIB_NODE_STATE_INTERRUPT) |
| 1173 | { |
| 1174 | u32 c = node->input_main_loops_per_call; |
| 1175 | /* Only call node when count reaches zero. */ |
| 1176 | if (c) |
| 1177 | { |
| 1178 | node->input_main_loops_per_call = c - 1; |
| 1179 | return last_time_stamp; |
| 1180 | } |
| 1181 | } |
| 1182 | |
| 1183 | /* Speculatively prefetch next frames. */ |
| 1184 | if (node->n_next_nodes > 0) |
| 1185 | { |
| 1186 | nf = vec_elt_at_index (nm->next_frames, node->next_frame_index); |
| 1187 | CLIB_PREFETCH (nf, 4 * sizeof (nf[0]), WRITE); |
| 1188 | } |
| 1189 | |
| 1190 | vm->cpu_time_last_node_dispatch = last_time_stamp; |
| 1191 | |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1192 | vlib_elog_main_loop_event (vm, node->node_index, |
| 1193 | last_time_stamp, frame ? frame->n_vectors : 0, |
| 1194 | /* is_after */ 0); |
| 1195 | |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 1196 | vlib_node_runtime_perf_counter (vm, node, frame, 0, last_time_stamp, |
| 1197 | VLIB_NODE_RUNTIME_PERF_BEFORE); |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1198 | |
| 1199 | /* |
| 1200 | * Turn this on if you run into |
| 1201 | * "bad monkey" contexts, and you want to know exactly |
| 1202 | * which nodes they've visited... See ixge.c... |
| 1203 | */ |
| 1204 | if (VLIB_BUFFER_TRACE_TRAJECTORY && frame) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1205 | { |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1206 | int i; |
| 1207 | u32 *from; |
| 1208 | from = vlib_frame_vector_args (frame); |
| 1209 | for (i = 0; i < frame->n_vectors; i++) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1210 | { |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1211 | vlib_buffer_t *b = vlib_get_buffer (vm, from[i]); |
| 1212 | add_trajectory_trace (b, node->node_index); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1213 | } |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1214 | if (PREDICT_FALSE (vm->dispatch_pcap_enable)) |
| 1215 | dispatch_pcap_trace (vm, node, frame); |
Damjan Marion | 8b60fb0 | 2020-11-27 20:15:17 +0100 | [diff] [blame] | 1216 | |
| 1217 | if (PREDICT_TRUE (vm->dispatch_wrapper_fn == 0)) |
| 1218 | n = node->function (vm, node, frame); |
| 1219 | else |
| 1220 | n = vm->dispatch_wrapper_fn (vm, node, frame); |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1221 | } |
| 1222 | else |
| 1223 | { |
| 1224 | if (PREDICT_FALSE (vm->dispatch_pcap_enable)) |
| 1225 | dispatch_pcap_trace (vm, node, frame); |
Damjan Marion | 8b60fb0 | 2020-11-27 20:15:17 +0100 | [diff] [blame] | 1226 | |
| 1227 | if (PREDICT_TRUE (vm->dispatch_wrapper_fn == 0)) |
| 1228 | n = node->function (vm, node, frame); |
| 1229 | else |
| 1230 | n = vm->dispatch_wrapper_fn (vm, node, frame); |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1231 | } |
| 1232 | |
| 1233 | t = clib_cpu_time_now (); |
| 1234 | |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 1235 | vlib_node_runtime_perf_counter (vm, node, frame, n, t, |
| 1236 | VLIB_NODE_RUNTIME_PERF_AFTER); |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1237 | |
| 1238 | vlib_elog_main_loop_event (vm, node->node_index, t, n, 1 /* is_after */ ); |
| 1239 | |
| 1240 | vm->main_loop_vectors_processed += n; |
| 1241 | vm->main_loop_nodes_processed += n > 0; |
| 1242 | |
| 1243 | v = vlib_node_runtime_update_stats (vm, node, |
| 1244 | /* n_calls */ 1, |
| 1245 | /* n_vectors */ n, |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 1246 | /* n_clocks */ t - last_time_stamp); |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1247 | |
| 1248 | /* When in interrupt mode and vector rate crosses threshold switch to |
| 1249 | polling mode. */ |
| 1250 | if (PREDICT_FALSE ((dispatch_state == VLIB_NODE_STATE_INTERRUPT) |
| 1251 | || (dispatch_state == VLIB_NODE_STATE_POLLING |
| 1252 | && (node->flags |
| 1253 | & |
| 1254 | VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE)))) |
| 1255 | { |
| 1256 | /* *INDENT-OFF* */ |
| 1257 | ELOG_TYPE_DECLARE (e) = |
| 1258 | { |
| 1259 | .function = (char *) __FUNCTION__, |
| 1260 | .format = "%s vector length %d, switching to %s", |
| 1261 | .format_args = "T4i4t4", |
| 1262 | .n_enum_strings = 2, |
| 1263 | .enum_strings = { |
| 1264 | "interrupt", "polling", |
| 1265 | }, |
| 1266 | }; |
| 1267 | /* *INDENT-ON* */ |
| 1268 | struct |
| 1269 | { |
| 1270 | u32 node_name, vector_length, is_polling; |
| 1271 | } *ed; |
| 1272 | |
| 1273 | if ((dispatch_state == VLIB_NODE_STATE_INTERRUPT |
| 1274 | && v >= nm->polling_threshold_vector_length) && |
| 1275 | !(node->flags & |
| 1276 | VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE)) |
Dave Barach | 3ae2873 | 2018-11-16 17:19:00 -0500 | [diff] [blame] | 1277 | { |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1278 | vlib_node_t *n = vlib_get_node (vm, node->node_index); |
| 1279 | n->state = VLIB_NODE_STATE_POLLING; |
| 1280 | node->state = VLIB_NODE_STATE_POLLING; |
| 1281 | node->flags &= |
| 1282 | ~VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE; |
| 1283 | node->flags |= VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE; |
| 1284 | nm->input_node_counts_by_state[VLIB_NODE_STATE_INTERRUPT] -= 1; |
| 1285 | nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] += 1; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1286 | |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1287 | if (PREDICT_FALSE (vlib_global_main.elog_trace_graph_dispatch)) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1288 | { |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1289 | vlib_worker_thread_t *w = vlib_worker_threads |
| 1290 | + vm->thread_index; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1291 | |
Steven | 6aa75af | 2017-02-24 10:03:22 -0800 | [diff] [blame] | 1292 | ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e, |
| 1293 | w->elog_track); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1294 | ed->node_name = n->name_elog_string; |
| 1295 | ed->vector_length = v; |
| 1296 | ed->is_polling = 1; |
| 1297 | } |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1298 | } |
| 1299 | else if (dispatch_state == VLIB_NODE_STATE_POLLING |
| 1300 | && v <= nm->interrupt_threshold_vector_length) |
| 1301 | { |
| 1302 | vlib_node_t *n = vlib_get_node (vm, node->node_index); |
| 1303 | if (node->flags & |
| 1304 | VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1305 | { |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1306 | /* Switch to interrupt mode after dispatch in polling one more time. |
| 1307 | This allows driver to re-enable interrupts. */ |
| 1308 | n->state = VLIB_NODE_STATE_INTERRUPT; |
| 1309 | node->state = VLIB_NODE_STATE_INTERRUPT; |
| 1310 | node->flags &= |
| 1311 | ~VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE; |
| 1312 | nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] -= 1; |
| 1313 | nm->input_node_counts_by_state[VLIB_NODE_STATE_INTERRUPT] += 1; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1314 | |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1315 | } |
| 1316 | else |
| 1317 | { |
| 1318 | vlib_worker_thread_t *w = vlib_worker_threads |
| 1319 | + vm->thread_index; |
| 1320 | node->flags |= |
| 1321 | VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE; |
| 1322 | if (PREDICT_FALSE (vlib_global_main.elog_trace_graph_dispatch)) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1323 | { |
Steven | 6aa75af | 2017-02-24 10:03:22 -0800 | [diff] [blame] | 1324 | ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e, |
| 1325 | w->elog_track); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1326 | ed->node_name = n->name_elog_string; |
| 1327 | ed->vector_length = v; |
| 1328 | ed->is_polling = 0; |
| 1329 | } |
| 1330 | } |
| 1331 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1332 | } |
| 1333 | |
| 1334 | return t; |
| 1335 | } |
| 1336 | |
Damjan Marion | 9a332e1 | 2017-03-28 15:11:20 +0200 | [diff] [blame] | 1337 | static u64 |
Dave Barach | a626999 | 2017-06-07 08:18:49 -0400 | [diff] [blame] | 1338 | dispatch_pending_node (vlib_main_t * vm, uword pending_frame_index, |
| 1339 | u64 last_time_stamp) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1340 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1341 | vlib_node_main_t *nm = &vm->node_main; |
| 1342 | vlib_frame_t *f; |
Dave Barach | 11fb09e | 2020-08-06 12:10:09 -0400 | [diff] [blame] | 1343 | vlib_next_frame_t *nf, nf_placeholder; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1344 | vlib_node_runtime_t *n; |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 1345 | vlib_frame_t *restore_frame; |
Dave Barach | a626999 | 2017-06-07 08:18:49 -0400 | [diff] [blame] | 1346 | vlib_pending_frame_t *p; |
| 1347 | |
| 1348 | /* See comment below about dangling references to nm->pending_frames */ |
| 1349 | p = nm->pending_frames + pending_frame_index; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1350 | |
| 1351 | n = vec_elt_at_index (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL], |
| 1352 | p->node_runtime_index); |
| 1353 | |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 1354 | f = vlib_get_frame (vm, p->frame); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1355 | if (p->next_frame_index == VLIB_PENDING_FRAME_NO_NEXT_FRAME) |
| 1356 | { |
Dave Barach | 11fb09e | 2020-08-06 12:10:09 -0400 | [diff] [blame] | 1357 | /* No next frame: so use placeholder on stack. */ |
| 1358 | nf = &nf_placeholder; |
Damjan Marion | 633b6fd | 2018-09-14 14:38:53 +0200 | [diff] [blame] | 1359 | nf->flags = f->frame_flags & VLIB_NODE_FLAG_TRACE; |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 1360 | nf->frame = NULL; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1361 | } |
| 1362 | else |
| 1363 | nf = vec_elt_at_index (nm->next_frames, p->next_frame_index); |
| 1364 | |
Damjan Marion | 633b6fd | 2018-09-14 14:38:53 +0200 | [diff] [blame] | 1365 | ASSERT (f->frame_flags & VLIB_FRAME_IS_ALLOCATED); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1366 | |
| 1367 | /* Force allocation of new frame while current frame is being |
| 1368 | dispatched. */ |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 1369 | restore_frame = NULL; |
| 1370 | if (nf->frame == p->frame) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1371 | { |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 1372 | nf->frame = NULL; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1373 | nf->flags &= ~VLIB_FRAME_IS_ALLOCATED; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1374 | if (!(n->flags & VLIB_NODE_FLAG_FRAME_NO_FREE_AFTER_DISPATCH)) |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 1375 | restore_frame = p->frame; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1376 | } |
| 1377 | |
| 1378 | /* Frame must be pending. */ |
Damjan Marion | 633b6fd | 2018-09-14 14:38:53 +0200 | [diff] [blame] | 1379 | ASSERT (f->frame_flags & VLIB_FRAME_PENDING); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1380 | ASSERT (f->n_vectors > 0); |
| 1381 | |
| 1382 | /* Copy trace flag from next frame to node. |
| 1383 | Trace flag indicates that at least one vector in the dispatched |
| 1384 | frame is traced. */ |
| 1385 | n->flags &= ~VLIB_NODE_FLAG_TRACE; |
| 1386 | n->flags |= (nf->flags & VLIB_FRAME_TRACE) ? VLIB_NODE_FLAG_TRACE : 0; |
| 1387 | nf->flags &= ~VLIB_FRAME_TRACE; |
| 1388 | |
| 1389 | last_time_stamp = dispatch_node (vm, n, |
| 1390 | VLIB_NODE_TYPE_INTERNAL, |
| 1391 | VLIB_NODE_STATE_POLLING, |
| 1392 | f, last_time_stamp); |
Dave Barach | a8df85c | 2019-10-01 13:34:23 -0400 | [diff] [blame] | 1393 | /* Internal node vector-rate accounting, for summary stats */ |
| 1394 | vm->internal_node_vectors += f->n_vectors; |
| 1395 | vm->internal_node_calls++; |
| 1396 | vm->internal_node_last_vectors_per_main_loop = |
| 1397 | (f->n_vectors > vm->internal_node_last_vectors_per_main_loop) ? |
| 1398 | f->n_vectors : vm->internal_node_last_vectors_per_main_loop; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1399 | |
Damjan Marion | 296988d | 2019-02-21 20:24:54 +0100 | [diff] [blame] | 1400 | f->frame_flags &= ~(VLIB_FRAME_PENDING | VLIB_FRAME_NO_APPEND); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1401 | |
| 1402 | /* Frame is ready to be used again, so restore it. */ |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 1403 | if (restore_frame != NULL) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1404 | { |
Dave Barach | a626999 | 2017-06-07 08:18:49 -0400 | [diff] [blame] | 1405 | /* |
| 1406 | * We musn't restore a frame that is flagged to be freed. This |
| 1407 | * shouldn't happen since frames to be freed post dispatch are |
| 1408 | * those used when the to-node frame becomes full i.e. they form a |
| 1409 | * sort of queue of frames to a single node. If we get here then |
| 1410 | * the to-node frame and the pending frame *were* the same, and so |
| 1411 | * we removed the to-node frame. Therefore this frame is no |
| 1412 | * longer part of the queue for that node and hence it cannot be |
| 1413 | * it's overspill. |
Neale Ranns | 8817061 | 2016-11-22 08:29:51 +0000 | [diff] [blame] | 1414 | */ |
Damjan Marion | 633b6fd | 2018-09-14 14:38:53 +0200 | [diff] [blame] | 1415 | ASSERT (!(f->frame_flags & VLIB_FRAME_FREE_AFTER_DISPATCH)); |
Neale Ranns | 8817061 | 2016-11-22 08:29:51 +0000 | [diff] [blame] | 1416 | |
Dave Barach | a626999 | 2017-06-07 08:18:49 -0400 | [diff] [blame] | 1417 | /* |
| 1418 | * NB: dispatching node n can result in the creation and scheduling |
| 1419 | * of new frames, and hence in the reallocation of nm->pending_frames. |
| 1420 | * Recompute p, or no supper. This was broken for more than 10 years. |
| 1421 | */ |
| 1422 | p = nm->pending_frames + pending_frame_index; |
| 1423 | |
| 1424 | /* |
| 1425 | * p->next_frame_index can change during node dispatch if node |
| 1426 | * function decides to change graph hook up. |
| 1427 | */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1428 | nf = vec_elt_at_index (nm->next_frames, p->next_frame_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1429 | nf->flags |= VLIB_FRAME_IS_ALLOCATED; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1430 | |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 1431 | if (NULL == nf->frame) |
Neale Ranns | 8817061 | 2016-11-22 08:29:51 +0000 | [diff] [blame] | 1432 | { |
| 1433 | /* no new frame has been assigned to this node, use the saved one */ |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 1434 | nf->frame = restore_frame; |
Neale Ranns | 8817061 | 2016-11-22 08:29:51 +0000 | [diff] [blame] | 1435 | f->n_vectors = 0; |
| 1436 | } |
| 1437 | else |
| 1438 | { |
| 1439 | /* The node has gained a frame, implying packets from the current frame |
| 1440 | were re-queued to this same node. we don't need the saved one |
| 1441 | anymore */ |
| 1442 | vlib_frame_free (vm, n, f); |
| 1443 | } |
| 1444 | } |
| 1445 | else |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1446 | { |
Damjan Marion | 633b6fd | 2018-09-14 14:38:53 +0200 | [diff] [blame] | 1447 | if (f->frame_flags & VLIB_FRAME_FREE_AFTER_DISPATCH) |
Neale Ranns | 8817061 | 2016-11-22 08:29:51 +0000 | [diff] [blame] | 1448 | { |
| 1449 | ASSERT (!(n->flags & VLIB_NODE_FLAG_FRAME_NO_FREE_AFTER_DISPATCH)); |
| 1450 | vlib_frame_free (vm, n, f); |
| 1451 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1452 | } |
| 1453 | |
| 1454 | return last_time_stamp; |
| 1455 | } |
| 1456 | |
| 1457 | always_inline uword |
| 1458 | vlib_process_stack_is_valid (vlib_process_t * p) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1459 | { |
| 1460 | return p->stack[0] == VLIB_PROCESS_STACK_MAGIC; |
| 1461 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1462 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1463 | typedef struct |
| 1464 | { |
| 1465 | vlib_main_t *vm; |
| 1466 | vlib_process_t *process; |
| 1467 | vlib_frame_t *frame; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1468 | } vlib_process_bootstrap_args_t; |
| 1469 | |
| 1470 | /* Called in process stack. */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1471 | static uword |
| 1472 | vlib_process_bootstrap (uword _a) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1473 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1474 | vlib_process_bootstrap_args_t *a; |
| 1475 | vlib_main_t *vm; |
| 1476 | vlib_node_runtime_t *node; |
| 1477 | vlib_frame_t *f; |
| 1478 | vlib_process_t *p; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1479 | uword n; |
| 1480 | |
| 1481 | a = uword_to_pointer (_a, vlib_process_bootstrap_args_t *); |
| 1482 | |
| 1483 | vm = a->vm; |
| 1484 | p = a->process; |
Damjan Marion | cea4652 | 2020-05-21 16:47:05 +0200 | [diff] [blame] | 1485 | vlib_process_finish_switch_stack (vm); |
| 1486 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1487 | f = a->frame; |
| 1488 | node = &p->node_runtime; |
| 1489 | |
| 1490 | n = node->function (vm, node, f); |
| 1491 | |
| 1492 | ASSERT (vlib_process_stack_is_valid (p)); |
| 1493 | |
Damjan Marion | cea4652 | 2020-05-21 16:47:05 +0200 | [diff] [blame] | 1494 | vlib_process_start_switch_stack (vm, 0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1495 | clib_longjmp (&p->return_longjmp, n); |
| 1496 | |
| 1497 | return n; |
| 1498 | } |
| 1499 | |
| 1500 | /* Called in main stack. */ |
| 1501 | static_always_inline uword |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1502 | vlib_process_startup (vlib_main_t * vm, vlib_process_t * p, vlib_frame_t * f) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1503 | { |
| 1504 | vlib_process_bootstrap_args_t a; |
| 1505 | uword r; |
| 1506 | |
| 1507 | a.vm = vm; |
| 1508 | a.process = p; |
| 1509 | a.frame = f; |
| 1510 | |
| 1511 | r = clib_setjmp (&p->return_longjmp, VLIB_PROCESS_RETURN_LONGJMP_RETURN); |
| 1512 | if (r == VLIB_PROCESS_RETURN_LONGJMP_RETURN) |
Damjan Marion | cea4652 | 2020-05-21 16:47:05 +0200 | [diff] [blame] | 1513 | { |
| 1514 | vlib_process_start_switch_stack (vm, p); |
| 1515 | r = clib_calljmp (vlib_process_bootstrap, pointer_to_uword (&a), |
| 1516 | (void *) p->stack + (1 << p->log2_n_stack_bytes)); |
| 1517 | } |
| 1518 | else |
| 1519 | vlib_process_finish_switch_stack (vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1520 | |
| 1521 | return r; |
| 1522 | } |
| 1523 | |
| 1524 | static_always_inline uword |
Damjan Marion | cea4652 | 2020-05-21 16:47:05 +0200 | [diff] [blame] | 1525 | vlib_process_resume (vlib_main_t * vm, vlib_process_t * p) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1526 | { |
| 1527 | uword r; |
| 1528 | p->flags &= ~(VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK |
| 1529 | | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT |
| 1530 | | VLIB_PROCESS_RESUME_PENDING); |
| 1531 | r = clib_setjmp (&p->return_longjmp, VLIB_PROCESS_RETURN_LONGJMP_RETURN); |
| 1532 | if (r == VLIB_PROCESS_RETURN_LONGJMP_RETURN) |
Damjan Marion | cea4652 | 2020-05-21 16:47:05 +0200 | [diff] [blame] | 1533 | { |
| 1534 | vlib_process_start_switch_stack (vm, p); |
| 1535 | clib_longjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_RESUME); |
| 1536 | } |
| 1537 | else |
| 1538 | vlib_process_finish_switch_stack (vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1539 | return r; |
| 1540 | } |
| 1541 | |
| 1542 | static u64 |
| 1543 | dispatch_process (vlib_main_t * vm, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1544 | vlib_process_t * p, vlib_frame_t * f, u64 last_time_stamp) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1545 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1546 | vlib_node_main_t *nm = &vm->node_main; |
| 1547 | vlib_node_runtime_t *node_runtime = &p->node_runtime; |
| 1548 | vlib_node_t *node = vlib_get_node (vm, node_runtime->node_index); |
Florin Coras | fd542f1 | 2018-05-16 19:28:24 -0700 | [diff] [blame] | 1549 | u32 old_process_index; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1550 | u64 t; |
| 1551 | uword n_vectors, is_suspend; |
| 1552 | |
| 1553 | if (node->state != VLIB_NODE_STATE_POLLING |
| 1554 | || (p->flags & (VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK |
| 1555 | | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT))) |
| 1556 | return last_time_stamp; |
| 1557 | |
| 1558 | p->flags |= VLIB_PROCESS_IS_RUNNING; |
| 1559 | |
| 1560 | t = last_time_stamp; |
| 1561 | vlib_elog_main_loop_event (vm, node_runtime->node_index, t, |
| 1562 | f ? f->n_vectors : 0, /* is_after */ 0); |
| 1563 | |
| 1564 | /* Save away current process for suspend. */ |
Florin Coras | fd542f1 | 2018-05-16 19:28:24 -0700 | [diff] [blame] | 1565 | old_process_index = nm->current_process_index; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1566 | nm->current_process_index = node->runtime_index; |
| 1567 | |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 1568 | vlib_node_runtime_perf_counter (vm, node_runtime, f, 0, last_time_stamp, |
| 1569 | VLIB_NODE_RUNTIME_PERF_BEFORE); |
| 1570 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1571 | n_vectors = vlib_process_startup (vm, p, f); |
| 1572 | |
Florin Coras | fd542f1 | 2018-05-16 19:28:24 -0700 | [diff] [blame] | 1573 | nm->current_process_index = old_process_index; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1574 | |
| 1575 | ASSERT (n_vectors != VLIB_PROCESS_RETURN_LONGJMP_RETURN); |
| 1576 | is_suspend = n_vectors == VLIB_PROCESS_RETURN_LONGJMP_SUSPEND; |
| 1577 | if (is_suspend) |
| 1578 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1579 | vlib_pending_frame_t *pf; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1580 | |
| 1581 | n_vectors = 0; |
| 1582 | pool_get (nm->suspended_process_frames, pf); |
| 1583 | pf->node_runtime_index = node->runtime_index; |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 1584 | pf->frame = f; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1585 | pf->next_frame_index = ~0; |
| 1586 | |
| 1587 | p->n_suspends += 1; |
| 1588 | p->suspended_process_frame_index = pf - nm->suspended_process_frames; |
| 1589 | |
| 1590 | if (p->flags & VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK) |
Dave Barach | 5c20a01 | 2017-06-13 08:48:31 -0400 | [diff] [blame] | 1591 | { |
| 1592 | TWT (tw_timer_wheel) * tw = |
| 1593 | (TWT (tw_timer_wheel) *) nm->timing_wheel; |
| 1594 | p->stop_timer_handle = |
| 1595 | TW (tw_timer_start) (tw, |
| 1596 | vlib_timing_wheel_data_set_suspended_process |
| 1597 | (node->runtime_index) /* [sic] pool idex */ , |
| 1598 | 0 /* timer_id */ , |
| 1599 | p->resume_clock_interval); |
| 1600 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1601 | } |
| 1602 | else |
| 1603 | p->flags &= ~VLIB_PROCESS_IS_RUNNING; |
| 1604 | |
| 1605 | t = clib_cpu_time_now (); |
| 1606 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1607 | vlib_elog_main_loop_event (vm, node_runtime->node_index, t, is_suspend, |
| 1608 | /* is_after */ 1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1609 | |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 1610 | vlib_node_runtime_perf_counter (vm, node_runtime, f, n_vectors, t, |
| 1611 | VLIB_NODE_RUNTIME_PERF_AFTER); |
| 1612 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1613 | vlib_process_update_stats (vm, p, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1614 | /* n_calls */ !is_suspend, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1615 | /* n_vectors */ n_vectors, |
Dave Barach | ec595ef | 2019-01-24 10:34:24 -0500 | [diff] [blame] | 1616 | /* n_clocks */ t - last_time_stamp); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1617 | |
| 1618 | return t; |
| 1619 | } |
| 1620 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1621 | void |
| 1622 | vlib_start_process (vlib_main_t * vm, uword process_index) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1623 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1624 | vlib_node_main_t *nm = &vm->node_main; |
| 1625 | vlib_process_t *p = vec_elt (nm->processes, process_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1626 | dispatch_process (vm, p, /* frame */ 0, /* cpu_time_now */ 0); |
| 1627 | } |
| 1628 | |
| 1629 | static u64 |
| 1630 | dispatch_suspended_process (vlib_main_t * vm, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1631 | uword process_index, u64 last_time_stamp) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1632 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1633 | vlib_node_main_t *nm = &vm->node_main; |
| 1634 | vlib_node_runtime_t *node_runtime; |
| 1635 | vlib_node_t *node; |
| 1636 | vlib_frame_t *f; |
| 1637 | vlib_process_t *p; |
| 1638 | vlib_pending_frame_t *pf; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1639 | u64 t, n_vectors, is_suspend; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1640 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1641 | t = last_time_stamp; |
| 1642 | |
| 1643 | p = vec_elt (nm->processes, process_index); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1644 | if (PREDICT_FALSE (!(p->flags & VLIB_PROCESS_IS_RUNNING))) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1645 | return last_time_stamp; |
| 1646 | |
| 1647 | ASSERT (p->flags & (VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK |
| 1648 | | VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT)); |
| 1649 | |
Florin Coras | 221d6f1 | 2018-11-07 20:46:38 -0800 | [diff] [blame] | 1650 | pf = pool_elt_at_index (nm->suspended_process_frames, |
| 1651 | p->suspended_process_frame_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1652 | |
| 1653 | node_runtime = &p->node_runtime; |
| 1654 | node = vlib_get_node (vm, node_runtime->node_index); |
Andreas Schultz | 58b2eb1 | 2019-07-15 15:40:56 +0200 | [diff] [blame] | 1655 | f = pf->frame; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1656 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1657 | vlib_elog_main_loop_event (vm, node_runtime->node_index, t, |
| 1658 | f ? f->n_vectors : 0, /* is_after */ 0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1659 | |
| 1660 | /* Save away current process for suspend. */ |
| 1661 | nm->current_process_index = node->runtime_index; |
| 1662 | |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 1663 | vlib_node_runtime_perf_counter (vm, node_runtime, f, 0, last_time_stamp, |
| 1664 | VLIB_NODE_RUNTIME_PERF_BEFORE); |
| 1665 | |
Damjan Marion | cea4652 | 2020-05-21 16:47:05 +0200 | [diff] [blame] | 1666 | n_vectors = vlib_process_resume (vm, p); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1667 | t = clib_cpu_time_now (); |
| 1668 | |
| 1669 | nm->current_process_index = ~0; |
| 1670 | |
| 1671 | is_suspend = n_vectors == VLIB_PROCESS_RETURN_LONGJMP_SUSPEND; |
| 1672 | if (is_suspend) |
| 1673 | { |
| 1674 | /* Suspend it again. */ |
| 1675 | n_vectors = 0; |
| 1676 | p->n_suspends += 1; |
| 1677 | if (p->flags & VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK) |
Dave Barach | 5c20a01 | 2017-06-13 08:48:31 -0400 | [diff] [blame] | 1678 | { |
| 1679 | p->stop_timer_handle = |
| 1680 | TW (tw_timer_start) ((TWT (tw_timer_wheel) *) nm->timing_wheel, |
| 1681 | vlib_timing_wheel_data_set_suspended_process |
| 1682 | (node->runtime_index) /* [sic] pool idex */ , |
| 1683 | 0 /* timer_id */ , |
| 1684 | p->resume_clock_interval); |
| 1685 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1686 | } |
| 1687 | else |
| 1688 | { |
| 1689 | p->flags &= ~VLIB_PROCESS_IS_RUNNING; |
Florin Coras | 221d6f1 | 2018-11-07 20:46:38 -0800 | [diff] [blame] | 1690 | pool_put_index (nm->suspended_process_frames, |
| 1691 | p->suspended_process_frame_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1692 | p->suspended_process_frame_index = ~0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1693 | } |
| 1694 | |
| 1695 | t = clib_cpu_time_now (); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1696 | vlib_elog_main_loop_event (vm, node_runtime->node_index, t, !is_suspend, |
| 1697 | /* is_after */ 1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1698 | |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 1699 | vlib_node_runtime_perf_counter (vm, node_runtime, f, n_vectors, t, |
| 1700 | VLIB_NODE_RUNTIME_PERF_AFTER); |
| 1701 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1702 | vlib_process_update_stats (vm, p, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1703 | /* n_calls */ !is_suspend, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1704 | /* n_vectors */ n_vectors, |
Dave Barach | ec595ef | 2019-01-24 10:34:24 -0500 | [diff] [blame] | 1705 | /* n_clocks */ t - last_time_stamp); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1706 | |
| 1707 | return t; |
| 1708 | } |
| 1709 | |
Dave Barach | 2877eee | 2017-12-15 12:22:57 -0500 | [diff] [blame] | 1710 | void vl_api_send_pending_rpc_requests (vlib_main_t *) __attribute__ ((weak)); |
| 1711 | void |
| 1712 | vl_api_send_pending_rpc_requests (vlib_main_t * vm) |
| 1713 | { |
| 1714 | } |
| 1715 | |
Dave Barach | 27d978c | 2020-11-03 09:59:06 -0500 | [diff] [blame] | 1716 | static inline void |
| 1717 | pcap_postmortem_reset (vlib_main_t * vm) |
| 1718 | { |
| 1719 | pcap_main_t *pm = &vm->dispatch_pcap_main; |
| 1720 | |
| 1721 | /* Reset the trace buffer and capture count */ |
| 1722 | clib_spinlock_lock_if_init (&pm->lock); |
| 1723 | vec_reset_length (pm->pcap_data); |
| 1724 | pm->n_packets_captured = 0; |
| 1725 | clib_spinlock_unlock_if_init (&pm->lock); |
| 1726 | } |
| 1727 | |
| 1728 | |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1729 | static_always_inline void |
| 1730 | vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1731 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1732 | vlib_node_main_t *nm = &vm->node_main; |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1733 | vlib_thread_main_t *tm = vlib_get_thread_main (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1734 | uword i; |
| 1735 | u64 cpu_time_now; |
Dave Barach | 000a029 | 2020-02-17 17:07:12 -0500 | [diff] [blame] | 1736 | f64 now; |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1737 | vlib_frame_queue_main_t *fqm; |
Dave Barach | 80965f5 | 2019-03-11 09:57:38 -0400 | [diff] [blame] | 1738 | u32 frame_queue_check_counter = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1739 | |
| 1740 | /* Initialize pending node vector. */ |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1741 | if (is_main) |
| 1742 | { |
| 1743 | vec_resize (nm->pending_frames, 32); |
| 1744 | _vec_len (nm->pending_frames) = 0; |
| 1745 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1746 | |
| 1747 | /* Mark time of main loop start. */ |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1748 | if (is_main) |
| 1749 | { |
| 1750 | cpu_time_now = vm->clib_time.last_cpu_time; |
| 1751 | vm->cpu_time_main_loop_start = cpu_time_now; |
| 1752 | } |
| 1753 | else |
| 1754 | cpu_time_now = clib_cpu_time_now (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1755 | |
Damjan Marion | 2c2b640 | 2017-03-28 14:16:15 +0200 | [diff] [blame] | 1756 | /* Pre-allocate interupt runtime indices and lock. */ |
Damjan Marion | 9410053 | 2020-11-06 23:25:57 +0100 | [diff] [blame] | 1757 | vec_alloc_aligned (nm->pending_interrupts, 1, CLIB_CACHE_LINE_BYTES); |
Damjan Marion | 2c2b640 | 2017-03-28 14:16:15 +0200 | [diff] [blame] | 1758 | |
| 1759 | /* Pre-allocate expired nodes. */ |
Steven | 7312cc7 | 2017-03-15 21:18:55 -0700 | [diff] [blame] | 1760 | if (!nm->polling_threshold_vector_length) |
| 1761 | nm->polling_threshold_vector_length = 10; |
| 1762 | if (!nm->interrupt_threshold_vector_length) |
| 1763 | nm->interrupt_threshold_vector_length = 5; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1764 | |
Damjan Marion | 29c0b33 | 2019-01-28 13:41:27 +0100 | [diff] [blame] | 1765 | vm->cpu_id = clib_get_current_cpu_id (); |
| 1766 | vm->numa_node = clib_get_current_numa_node (); |
Florin Coras | 4c95995 | 2020-02-09 18:09:31 +0000 | [diff] [blame] | 1767 | os_set_numa_index (vm->numa_node); |
Damjan Marion | 29c0b33 | 2019-01-28 13:41:27 +0100 | [diff] [blame] | 1768 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1769 | /* Start all processes. */ |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1770 | if (is_main) |
| 1771 | { |
| 1772 | uword i; |
Dave Barach | c602b38 | 2019-06-03 19:48:22 -0400 | [diff] [blame] | 1773 | |
| 1774 | /* |
| 1775 | * Perform an initial barrier sync. Pays no attention to |
| 1776 | * the barrier sync hold-down timer scheme, which won't work |
| 1777 | * at this point in time. |
| 1778 | */ |
| 1779 | vlib_worker_thread_initial_barrier_sync_and_release (vm); |
| 1780 | |
Steven | f3b5364 | 2017-05-01 14:03:02 -0700 | [diff] [blame] | 1781 | nm->current_process_index = ~0; |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1782 | for (i = 0; i < vec_len (nm->processes); i++) |
| 1783 | cpu_time_now = dispatch_process (vm, nm->processes[i], /* frame */ 0, |
| 1784 | cpu_time_now); |
| 1785 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1786 | |
| 1787 | while (1) |
| 1788 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1789 | vlib_node_runtime_t *n; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1790 | |
Dave Barach | 2877eee | 2017-12-15 12:22:57 -0500 | [diff] [blame] | 1791 | if (PREDICT_FALSE (_vec_len (vm->pending_rpc_requests) > 0)) |
Dave Barach | f6c68d7 | 2018-11-01 08:12:52 -0400 | [diff] [blame] | 1792 | { |
| 1793 | if (!is_main) |
| 1794 | vl_api_send_pending_rpc_requests (vm); |
| 1795 | } |
Dave Barach | 2877eee | 2017-12-15 12:22:57 -0500 | [diff] [blame] | 1796 | |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1797 | if (!is_main) |
Damjan Marion | f6e6c78 | 2020-09-17 09:54:07 +0200 | [diff] [blame] | 1798 | vlib_worker_thread_barrier_check (); |
| 1799 | |
| 1800 | if (PREDICT_FALSE (vm->check_frame_queues + frame_queue_check_counter)) |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1801 | { |
Damjan Marion | f6e6c78 | 2020-09-17 09:54:07 +0200 | [diff] [blame] | 1802 | u32 processed = 0; |
| 1803 | |
| 1804 | if (vm->check_frame_queues) |
Dave Barach | 80965f5 | 2019-03-11 09:57:38 -0400 | [diff] [blame] | 1805 | { |
Damjan Marion | f6e6c78 | 2020-09-17 09:54:07 +0200 | [diff] [blame] | 1806 | frame_queue_check_counter = 100; |
| 1807 | vm->check_frame_queues = 0; |
Dave Barach | 80965f5 | 2019-03-11 09:57:38 -0400 | [diff] [blame] | 1808 | } |
Damjan Marion | f6e6c78 | 2020-09-17 09:54:07 +0200 | [diff] [blame] | 1809 | |
| 1810 | vec_foreach (fqm, tm->frame_queue_mains) |
| 1811 | processed += vlib_frame_queue_dequeue (vm, fqm); |
| 1812 | |
| 1813 | /* No handoff queue work found? */ |
| 1814 | if (processed) |
| 1815 | frame_queue_check_counter = 100; |
| 1816 | else |
| 1817 | frame_queue_check_counter--; |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1818 | } |
| 1819 | |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 1820 | if (PREDICT_FALSE (vec_len (vm->worker_thread_main_loop_callbacks))) |
| 1821 | clib_call_callbacks (vm->worker_thread_main_loop_callbacks, vm, |
| 1822 | cpu_time_now); |
| 1823 | |
Dave Barach | 27d978c | 2020-11-03 09:59:06 -0500 | [diff] [blame] | 1824 | /* |
| 1825 | * When trying to understand aggravating, hard-to-reproduce |
| 1826 | * bugs: reset / restart the pcap dispatch trace once per |
| 1827 | * main thread dispatch cycle. All threads share the same |
| 1828 | * (spinlock-protected) dispatch trace buffer. It might take |
| 1829 | * a few tries to capture a good post-mortem trace of |
| 1830 | * a multi-thread issue. Best we can do without a big refactor job. |
| 1831 | */ |
| 1832 | if (is_main && PREDICT_FALSE (vm->dispatch_pcap_postmortem != 0)) |
| 1833 | pcap_postmortem_reset (vm); |
| 1834 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1835 | /* Process pre-input nodes. */ |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 1836 | cpu_time_now = clib_cpu_time_now (); |
Damjan Marion | ceab788 | 2018-01-19 20:56:12 +0100 | [diff] [blame] | 1837 | vec_foreach (n, nm->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT]) |
| 1838 | cpu_time_now = dispatch_node (vm, n, |
| 1839 | VLIB_NODE_TYPE_PRE_INPUT, |
| 1840 | VLIB_NODE_STATE_POLLING, |
| 1841 | /* frame */ 0, |
| 1842 | cpu_time_now); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1843 | |
| 1844 | /* Next process input nodes. */ |
| 1845 | vec_foreach (n, nm->nodes_by_type[VLIB_NODE_TYPE_INPUT]) |
| 1846 | cpu_time_now = dispatch_node (vm, n, |
| 1847 | VLIB_NODE_TYPE_INPUT, |
| 1848 | VLIB_NODE_STATE_POLLING, |
| 1849 | /* frame */ 0, |
| 1850 | cpu_time_now); |
| 1851 | |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1852 | if (PREDICT_TRUE (is_main && vm->queue_signal_pending == 0)) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1853 | vm->queue_signal_callback (vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1854 | |
Damjan Marion | 9410053 | 2020-11-06 23:25:57 +0100 | [diff] [blame] | 1855 | if (__atomic_load_n (nm->pending_interrupts, __ATOMIC_ACQUIRE)) |
Damjan Marion | 0b31630 | 2020-09-09 18:55:16 +0200 | [diff] [blame] | 1856 | { |
Damjan Marion | 9410053 | 2020-11-06 23:25:57 +0100 | [diff] [blame] | 1857 | int int_num = -1; |
| 1858 | *nm->pending_interrupts = 0; |
Dave Barach | d47c509 | 2018-01-19 13:09:20 -0500 | [diff] [blame] | 1859 | |
Damjan Marion | 9410053 | 2020-11-06 23:25:57 +0100 | [diff] [blame] | 1860 | while ((int_num = |
| 1861 | clib_interrupt_get_next (nm->interrupts, int_num)) != -1) |
| 1862 | { |
| 1863 | vlib_node_runtime_t *n; |
| 1864 | clib_interrupt_clear (nm->interrupts, int_num); |
| 1865 | n = vec_elt_at_index (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT], |
| 1866 | int_num); |
| 1867 | cpu_time_now = dispatch_node (vm, n, VLIB_NODE_TYPE_INPUT, |
| 1868 | VLIB_NODE_STATE_INTERRUPT, |
| 1869 | /* frame */ 0, cpu_time_now); |
| 1870 | } |
Damjan Marion | 1033b49 | 2020-06-03 12:20:41 +0200 | [diff] [blame] | 1871 | } |
| 1872 | |
Dave Barach | e324898 | 2018-08-14 13:47:58 -0400 | [diff] [blame] | 1873 | /* Input nodes may have added work to the pending vector. |
| 1874 | Process pending vector until there is nothing left. |
| 1875 | All pending vectors will be processed from input -> output. */ |
| 1876 | for (i = 0; i < _vec_len (nm->pending_frames); i++) |
| 1877 | cpu_time_now = dispatch_pending_node (vm, i, cpu_time_now); |
| 1878 | /* Reset pending vector for next iteration. */ |
| 1879 | _vec_len (nm->pending_frames) = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1880 | |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1881 | if (is_main) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1882 | { |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1883 | /* *INDENT-OFF* */ |
| 1884 | ELOG_TYPE_DECLARE (es) = |
| 1885 | { |
| 1886 | .format = "process tw start", |
| 1887 | .format_args = "", |
| 1888 | }; |
| 1889 | ELOG_TYPE_DECLARE (ee) = |
| 1890 | { |
| 1891 | .format = "process tw end: %d", |
| 1892 | .format_args = "i4", |
| 1893 | }; |
| 1894 | /* *INDENT-ON* */ |
| 1895 | |
| 1896 | struct |
| 1897 | { |
| 1898 | int nready_procs; |
| 1899 | } *ed; |
| 1900 | |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1901 | /* Check if process nodes have expired from timing wheel. */ |
Dave Barach | 5c20a01 | 2017-06-13 08:48:31 -0400 | [diff] [blame] | 1902 | ASSERT (nm->data_from_advancing_timing_wheel != 0); |
| 1903 | |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1904 | if (PREDICT_FALSE (vm->elog_trace_graph_dispatch)) |
| 1905 | ed = ELOG_DATA (&vlib_global_main.elog_main, es); |
| 1906 | |
Dave Barach | 5c20a01 | 2017-06-13 08:48:31 -0400 | [diff] [blame] | 1907 | nm->data_from_advancing_timing_wheel = |
| 1908 | TW (tw_timer_expire_timers_vec) |
| 1909 | ((TWT (tw_timer_wheel) *) nm->timing_wheel, vlib_time_now (vm), |
| 1910 | nm->data_from_advancing_timing_wheel); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1911 | |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1912 | ASSERT (nm->data_from_advancing_timing_wheel != 0); |
Dave Barach | 5c20a01 | 2017-06-13 08:48:31 -0400 | [diff] [blame] | 1913 | |
Dave Barach | 900cbad | 2019-01-31 19:12:51 -0500 | [diff] [blame] | 1914 | if (PREDICT_FALSE (vm->elog_trace_graph_dispatch)) |
| 1915 | { |
| 1916 | ed = ELOG_DATA (&vlib_global_main.elog_main, ee); |
| 1917 | ed->nready_procs = |
| 1918 | _vec_len (nm->data_from_advancing_timing_wheel); |
| 1919 | } |
| 1920 | |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1921 | if (PREDICT_FALSE |
| 1922 | (_vec_len (nm->data_from_advancing_timing_wheel) > 0)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1923 | { |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1924 | uword i; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1925 | |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1926 | for (i = 0; i < _vec_len (nm->data_from_advancing_timing_wheel); |
| 1927 | i++) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1928 | { |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1929 | u32 d = nm->data_from_advancing_timing_wheel[i]; |
| 1930 | u32 di = vlib_timing_wheel_data_get_index (d); |
| 1931 | |
| 1932 | if (vlib_timing_wheel_data_is_timed_event (d)) |
| 1933 | { |
| 1934 | vlib_signal_timed_event_data_t *te = |
| 1935 | pool_elt_at_index (nm->signal_timed_event_data_pool, |
| 1936 | di); |
| 1937 | vlib_node_t *n = |
| 1938 | vlib_get_node (vm, te->process_node_index); |
| 1939 | vlib_process_t *p = |
| 1940 | vec_elt (nm->processes, n->runtime_index); |
| 1941 | void *data; |
| 1942 | data = |
| 1943 | vlib_process_signal_event_helper (nm, n, p, |
| 1944 | te->event_type_index, |
| 1945 | te->n_data_elts, |
| 1946 | te->n_data_elt_bytes); |
| 1947 | if (te->n_data_bytes < sizeof (te->inline_event_data)) |
Dave Barach | 178cf49 | 2018-11-13 16:34:13 -0500 | [diff] [blame] | 1948 | clib_memcpy_fast (data, te->inline_event_data, |
| 1949 | te->n_data_bytes); |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1950 | else |
| 1951 | { |
Dave Barach | 178cf49 | 2018-11-13 16:34:13 -0500 | [diff] [blame] | 1952 | clib_memcpy_fast (data, te->event_data_as_vector, |
| 1953 | te->n_data_bytes); |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1954 | vec_free (te->event_data_as_vector); |
| 1955 | } |
| 1956 | pool_put (nm->signal_timed_event_data_pool, te); |
| 1957 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1958 | else |
| 1959 | { |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1960 | cpu_time_now = clib_cpu_time_now (); |
| 1961 | cpu_time_now = |
| 1962 | dispatch_suspended_process (vm, di, cpu_time_now); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1963 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1964 | } |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1965 | _vec_len (nm->data_from_advancing_timing_wheel) = 0; |
| 1966 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1967 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1968 | vlib_increment_main_loop_counter (vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1969 | /* Record time stamp in case there are no enabled nodes and above |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1970 | calls do not update time stamp. */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1971 | cpu_time_now = clib_cpu_time_now (); |
Dave Barach | 000a029 | 2020-02-17 17:07:12 -0500 | [diff] [blame] | 1972 | vm->loops_this_reporting_interval++; |
| 1973 | now = clib_time_now_internal (&vm->clib_time, cpu_time_now); |
| 1974 | /* Time to update loops_per_second? */ |
| 1975 | if (PREDICT_FALSE (now >= vm->loop_interval_end)) |
| 1976 | { |
| 1977 | /* Next sample ends in 20ms */ |
| 1978 | if (vm->loop_interval_start) |
| 1979 | { |
| 1980 | f64 this_loops_per_second; |
| 1981 | |
| 1982 | this_loops_per_second = |
| 1983 | ((f64) vm->loops_this_reporting_interval) / (now - |
| 1984 | vm->loop_interval_start); |
| 1985 | |
| 1986 | vm->loops_per_second = |
| 1987 | vm->loops_per_second * vm->damping_constant + |
| 1988 | (1.0 - vm->damping_constant) * this_loops_per_second; |
| 1989 | if (vm->loops_per_second != 0.0) |
| 1990 | vm->seconds_per_loop = 1.0 / vm->loops_per_second; |
| 1991 | else |
| 1992 | vm->seconds_per_loop = 0.0; |
| 1993 | } |
| 1994 | /* New interval starts now, and ends in 20ms */ |
| 1995 | vm->loop_interval_start = now; |
| 1996 | vm->loop_interval_end = now + 2e-4; |
| 1997 | vm->loops_this_reporting_interval = 0; |
| 1998 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1999 | } |
| 2000 | } |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 2001 | |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 2002 | static void |
| 2003 | vlib_main_loop (vlib_main_t * vm) |
| 2004 | { |
| 2005 | vlib_main_or_worker_loop (vm, /* is_main */ 1); |
| 2006 | } |
| 2007 | |
| 2008 | void |
| 2009 | vlib_worker_loop (vlib_main_t * vm) |
| 2010 | { |
| 2011 | vlib_main_or_worker_loop (vm, /* is_main */ 0); |
| 2012 | } |
| 2013 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2014 | vlib_main_t vlib_global_main; |
| 2015 | |
| 2016 | static clib_error_t * |
| 2017 | vlib_main_configure (vlib_main_t * vm, unformat_input_t * input) |
| 2018 | { |
| 2019 | int turn_on_mem_trace = 0; |
| 2020 | |
| 2021 | while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) |
| 2022 | { |
| 2023 | if (unformat (input, "memory-trace")) |
| 2024 | turn_on_mem_trace = 1; |
| 2025 | |
| 2026 | else if (unformat (input, "elog-events %d", |
Dave Barach | bc867c3 | 2020-11-25 10:07:09 -0500 | [diff] [blame] | 2027 | &vm->configured_elog_ring_size)) |
| 2028 | vm->configured_elog_ring_size = |
| 2029 | 1 << max_log2 (vm->configured_elog_ring_size); |
Dave Barach | 8148131 | 2017-05-16 09:08:14 -0400 | [diff] [blame] | 2030 | else if (unformat (input, "elog-post-mortem-dump")) |
| 2031 | vm->elog_post_mortem_dump = 1; |
Dave Barach | c74b43c | 2020-04-09 17:24:07 -0400 | [diff] [blame] | 2032 | else if (unformat (input, "buffer-alloc-success-rate %f", |
| 2033 | &vm->buffer_alloc_success_rate)) |
| 2034 | { |
| 2035 | if (VLIB_BUFFER_ALLOC_FAULT_INJECTOR == 0) |
| 2036 | return clib_error_return |
| 2037 | (0, "Buffer fault injection not configured"); |
| 2038 | } |
| 2039 | else if (unformat (input, "buffer-alloc-success-seed %u", |
| 2040 | &vm->buffer_alloc_success_seed)) |
| 2041 | { |
| 2042 | if (VLIB_BUFFER_ALLOC_FAULT_INJECTOR == 0) |
| 2043 | return clib_error_return |
| 2044 | (0, "Buffer fault injection not configured"); |
| 2045 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2046 | else |
| 2047 | return unformat_parse_error (input); |
| 2048 | } |
| 2049 | |
| 2050 | unformat_free (input); |
| 2051 | |
| 2052 | /* Enable memory trace as early as possible. */ |
| 2053 | if (turn_on_mem_trace) |
| 2054 | clib_mem_trace (1); |
| 2055 | |
| 2056 | return 0; |
| 2057 | } |
| 2058 | |
| 2059 | VLIB_EARLY_CONFIG_FUNCTION (vlib_main_configure, "vlib"); |
| 2060 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 2061 | static void |
Dave Barach | 11fb09e | 2020-08-06 12:10:09 -0400 | [diff] [blame] | 2062 | placeholder_queue_signal_callback (vlib_main_t * vm) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 2063 | { |
| 2064 | } |
Dave Barach | 16c75df | 2016-05-31 14:05:46 -0400 | [diff] [blame] | 2065 | |
Dave Barach | 1f80658 | 2018-06-14 09:18:21 -0400 | [diff] [blame] | 2066 | #define foreach_weak_reference_stub \ |
| 2067 | _(vlib_map_stat_segment_init) \ |
| 2068 | _(vpe_api_init) \ |
| 2069 | _(vlibmemory_init) \ |
| 2070 | _(map_api_segment_init) |
| 2071 | |
| 2072 | #define _(name) \ |
| 2073 | clib_error_t *name (vlib_main_t *vm) __attribute__((weak)); \ |
| 2074 | clib_error_t *name (vlib_main_t *vm) { return 0; } |
| 2075 | foreach_weak_reference_stub; |
| 2076 | #undef _ |
| 2077 | |
Dave Barach | b09f4d0 | 2019-07-15 16:00:03 -0400 | [diff] [blame] | 2078 | void vl_api_set_elog_main (elog_main_t * m) __attribute__ ((weak)); |
| 2079 | void |
| 2080 | vl_api_set_elog_main (elog_main_t * m) |
| 2081 | { |
| 2082 | clib_warning ("STUB"); |
| 2083 | } |
| 2084 | |
| 2085 | int vl_api_set_elog_trace_api_messages (int enable) __attribute__ ((weak)); |
| 2086 | int |
| 2087 | vl_api_set_elog_trace_api_messages (int enable) |
| 2088 | { |
| 2089 | clib_warning ("STUB"); |
| 2090 | return 0; |
| 2091 | } |
| 2092 | |
| 2093 | int vl_api_get_elog_trace_api_messages (void) __attribute__ ((weak)); |
| 2094 | int |
| 2095 | vl_api_get_elog_trace_api_messages (void) |
| 2096 | { |
| 2097 | clib_warning ("STUB"); |
| 2098 | return 0; |
| 2099 | } |
| 2100 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2101 | /* Main function. */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 2102 | int |
Eyal Bari | d334a6b | 2016-09-19 10:23:39 +0300 | [diff] [blame] | 2103 | vlib_main (vlib_main_t * volatile vm, unformat_input_t * input) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2104 | { |
Eyal Bari | d334a6b | 2016-09-19 10:23:39 +0300 | [diff] [blame] | 2105 | clib_error_t *volatile error; |
Dave Barach | 5c20a01 | 2017-06-13 08:48:31 -0400 | [diff] [blame] | 2106 | vlib_node_main_t *nm = &vm->node_main; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2107 | |
Dave Barach | 11fb09e | 2020-08-06 12:10:09 -0400 | [diff] [blame] | 2108 | vm->queue_signal_callback = placeholder_queue_signal_callback; |
Dave Barach | 16c75df | 2016-05-31 14:05:46 -0400 | [diff] [blame] | 2109 | |
Dave Barach | bc867c3 | 2020-11-25 10:07:09 -0500 | [diff] [blame] | 2110 | /* Reconfigure event log which is enabled very early */ |
| 2111 | if (vm->configured_elog_ring_size && |
| 2112 | vm->configured_elog_ring_size != vm->elog_main.event_ring_size) |
| 2113 | elog_resize (&vm->elog_main, vm->configured_elog_ring_size); |
Dave Barach | b09f4d0 | 2019-07-15 16:00:03 -0400 | [diff] [blame] | 2114 | vl_api_set_elog_main (&vm->elog_main); |
| 2115 | (void) vl_api_set_elog_trace_api_messages (1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2116 | |
| 2117 | /* Default name. */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 2118 | if (!vm->name) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2119 | vm->name = "VLIB"; |
| 2120 | |
Damjan Marion | 68b4da6 | 2018-09-30 18:26:20 +0200 | [diff] [blame] | 2121 | if ((error = vlib_physmem_init (vm))) |
Damjan Marion | 04a7f05 | 2017-07-10 15:06:17 +0200 | [diff] [blame] | 2122 | { |
Damjan Marion | 49d66f1 | 2017-07-20 18:10:35 +0200 | [diff] [blame] | 2123 | clib_error_report (error); |
| 2124 | goto done; |
Damjan Marion | 04a7f05 | 2017-07-10 15:06:17 +0200 | [diff] [blame] | 2125 | } |
Damjan Marion | 49d66f1 | 2017-07-20 18:10:35 +0200 | [diff] [blame] | 2126 | |
Filip Tehlar | d2bbdef | 2019-02-22 05:05:53 -0800 | [diff] [blame] | 2127 | if ((error = vlib_map_stat_segment_init (vm))) |
| 2128 | { |
| 2129 | clib_error_report (error); |
| 2130 | goto done; |
| 2131 | } |
| 2132 | |
Damjan Marion | 49d66f1 | 2017-07-20 18:10:35 +0200 | [diff] [blame] | 2133 | if ((error = vlib_buffer_main_init (vm))) |
Damjan Marion | 04a7f05 | 2017-07-10 15:06:17 +0200 | [diff] [blame] | 2134 | { |
Damjan Marion | 49d66f1 | 2017-07-20 18:10:35 +0200 | [diff] [blame] | 2135 | clib_error_report (error); |
| 2136 | goto done; |
Damjan Marion | 04a7f05 | 2017-07-10 15:06:17 +0200 | [diff] [blame] | 2137 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2138 | |
| 2139 | if ((error = vlib_thread_init (vm))) |
| 2140 | { |
| 2141 | clib_error_report (error); |
| 2142 | goto done; |
| 2143 | } |
| 2144 | |
| 2145 | /* Register static nodes so that init functions may use them. */ |
| 2146 | vlib_register_all_static_nodes (vm); |
| 2147 | |
| 2148 | /* Set seed for random number generator. |
| 2149 | Allow user to specify seed to make random sequence deterministic. */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 2150 | if (!unformat (input, "seed %wd", &vm->random_seed)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2151 | vm->random_seed = clib_cpu_time_now (); |
| 2152 | clib_random_buffer_init (&vm->random_buffer, vm->random_seed); |
| 2153 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2154 | /* Initialize node graph. */ |
| 2155 | if ((error = vlib_node_main_init (vm))) |
| 2156 | { |
| 2157 | /* Arrange for graph hook up error to not be fatal when debugging. */ |
| 2158 | if (CLIB_DEBUG > 0) |
| 2159 | clib_error_report (error); |
| 2160 | else |
| 2161 | goto done; |
| 2162 | } |
| 2163 | |
Dave Barach | 1f80658 | 2018-06-14 09:18:21 -0400 | [diff] [blame] | 2164 | /* Direct call / weak reference, for vlib standalone use-cases */ |
| 2165 | if ((error = vpe_api_init (vm))) |
Dave Barach | 048a4e5 | 2018-06-01 18:52:25 -0400 | [diff] [blame] | 2166 | { |
| 2167 | clib_error_report (error); |
| 2168 | goto done; |
| 2169 | } |
| 2170 | |
Dave Barach | 1f80658 | 2018-06-14 09:18:21 -0400 | [diff] [blame] | 2171 | if ((error = vlibmemory_init (vm))) |
Dave Barach | 048a4e5 | 2018-06-01 18:52:25 -0400 | [diff] [blame] | 2172 | { |
| 2173 | clib_error_report (error); |
| 2174 | goto done; |
| 2175 | } |
| 2176 | |
Dave Barach | 1f80658 | 2018-06-14 09:18:21 -0400 | [diff] [blame] | 2177 | if ((error = map_api_segment_init (vm))) |
Dave Barach | 048a4e5 | 2018-06-01 18:52:25 -0400 | [diff] [blame] | 2178 | { |
| 2179 | clib_error_report (error); |
| 2180 | goto done; |
| 2181 | } |
| 2182 | |
Ole Troan | 964f93e | 2016-06-10 13:22:36 +0200 | [diff] [blame] | 2183 | /* See unix/main.c; most likely already set up */ |
| 2184 | if (vm->init_functions_called == 0) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 2185 | vm->init_functions_called = hash_create (0, /* value bytes */ 0); |
Ole Troan | 964f93e | 2016-06-10 13:22:36 +0200 | [diff] [blame] | 2186 | if ((error = vlib_call_all_init_functions (vm))) |
| 2187 | goto done; |
| 2188 | |
Dave Barach | 5c20a01 | 2017-06-13 08:48:31 -0400 | [diff] [blame] | 2189 | nm->timing_wheel = clib_mem_alloc_aligned (sizeof (TWT (tw_timer_wheel)), |
| 2190 | CLIB_CACHE_LINE_BYTES); |
| 2191 | |
| 2192 | vec_validate (nm->data_from_advancing_timing_wheel, 10); |
| 2193 | _vec_len (nm->data_from_advancing_timing_wheel) = 0; |
| 2194 | |
| 2195 | /* Create the process timing wheel */ |
| 2196 | TW (tw_timer_wheel_init) ((TWT (tw_timer_wheel) *) nm->timing_wheel, |
| 2197 | 0 /* no callback */ , |
| 2198 | 10e-6 /* timer period 10us */ , |
| 2199 | ~0 /* max expirations per call */ ); |
| 2200 | |
Dave Barach | 2877eee | 2017-12-15 12:22:57 -0500 | [diff] [blame] | 2201 | vec_validate (vm->pending_rpc_requests, 0); |
| 2202 | _vec_len (vm->pending_rpc_requests) = 0; |
Dave Barach | f6c68d7 | 2018-11-01 08:12:52 -0400 | [diff] [blame] | 2203 | vec_validate (vm->processing_rpc_requests, 0); |
| 2204 | _vec_len (vm->processing_rpc_requests) = 0; |
Dave Barach | 2877eee | 2017-12-15 12:22:57 -0500 | [diff] [blame] | 2205 | |
Dave Barach | c74b43c | 2020-04-09 17:24:07 -0400 | [diff] [blame] | 2206 | /* Default params for the buffer allocator fault injector, if configured */ |
| 2207 | if (VLIB_BUFFER_ALLOC_FAULT_INJECTOR > 0) |
| 2208 | { |
| 2209 | vm->buffer_alloc_success_seed = 0xdeaddabe; |
| 2210 | vm->buffer_alloc_success_rate = 0.80; |
| 2211 | } |
| 2212 | |
Dave Barach | d1e17d0 | 2019-03-21 18:01:48 -0400 | [diff] [blame] | 2213 | if ((error = vlib_call_all_config_functions (vm, input, 0 /* is_early */ ))) |
| 2214 | goto done; |
| 2215 | |
Dave Barach | 000a029 | 2020-02-17 17:07:12 -0500 | [diff] [blame] | 2216 | /* |
| 2217 | * Use exponential smoothing, with a half-life of 1 second |
| 2218 | * reported_rate(t) = reported_rate(t-1) * K + rate(t)*(1-K) |
| 2219 | * |
| 2220 | * Sample every 20ms, aka 50 samples per second |
| 2221 | * K = exp (-1.0/20.0); |
| 2222 | * K = 0.95 |
| 2223 | */ |
| 2224 | vm->damping_constant = exp (-1.0 / 20.0); |
| 2225 | |
Dave Barach | c602b38 | 2019-06-03 19:48:22 -0400 | [diff] [blame] | 2226 | /* Sort per-thread init functions before we start threads */ |
| 2227 | vlib_sort_init_exit_functions (&vm->worker_init_function_registrations); |
| 2228 | |
Dave Barach | d1e17d0 | 2019-03-21 18:01:48 -0400 | [diff] [blame] | 2229 | /* Call all main loop enter functions. */ |
| 2230 | { |
| 2231 | clib_error_t *sub_error; |
| 2232 | sub_error = vlib_call_all_main_loop_enter_functions (vm); |
| 2233 | if (sub_error) |
| 2234 | clib_error_report (sub_error); |
| 2235 | } |
| 2236 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2237 | switch (clib_setjmp (&vm->main_loop_exit, VLIB_MAIN_LOOP_EXIT_NONE)) |
| 2238 | { |
| 2239 | case VLIB_MAIN_LOOP_EXIT_NONE: |
| 2240 | vm->main_loop_exit_set = 1; |
| 2241 | break; |
| 2242 | |
| 2243 | case VLIB_MAIN_LOOP_EXIT_CLI: |
| 2244 | goto done; |
| 2245 | |
| 2246 | default: |
| 2247 | error = vm->main_loop_error; |
| 2248 | goto done; |
| 2249 | } |
| 2250 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2251 | vlib_main_loop (vm); |
| 2252 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 2253 | done: |
Kommula Shiva Shankar | ced43e2 | 2021-01-28 13:05:59 +0530 | [diff] [blame] | 2254 | vlib_worker_thread_barrier_sync (vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2255 | /* Call all exit functions. */ |
| 2256 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 2257 | clib_error_t *sub_error; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2258 | sub_error = vlib_call_all_main_loop_exit_functions (vm); |
| 2259 | if (sub_error) |
| 2260 | clib_error_report (sub_error); |
| 2261 | } |
Kommula Shiva Shankar | ced43e2 | 2021-01-28 13:05:59 +0530 | [diff] [blame] | 2262 | vlib_worker_thread_barrier_release (vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2263 | |
| 2264 | if (error) |
| 2265 | clib_error_report (error); |
| 2266 | |
| 2267 | return 0; |
| 2268 | } |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 2269 | |
Dave Barach | e5948fb | 2019-08-29 18:01:30 -0400 | [diff] [blame] | 2270 | int |
| 2271 | vlib_pcap_dispatch_trace_configure (vlib_pcap_dispatch_trace_args_t * a) |
Dave Barach | 3ae2873 | 2018-11-16 17:19:00 -0500 | [diff] [blame] | 2272 | { |
Dave Barach | e5948fb | 2019-08-29 18:01:30 -0400 | [diff] [blame] | 2273 | vlib_main_t *vm = vlib_get_main (); |
Dave Barach | 3ae2873 | 2018-11-16 17:19:00 -0500 | [diff] [blame] | 2274 | pcap_main_t *pm = &vm->dispatch_pcap_main; |
Dave Barach | 1201a80 | 2018-11-20 12:08:39 -0500 | [diff] [blame] | 2275 | vlib_trace_main_t *tm; |
| 2276 | vlib_trace_node_t *tn; |
Dave Barach | 3ae2873 | 2018-11-16 17:19:00 -0500 | [diff] [blame] | 2277 | |
Dave Barach | e5948fb | 2019-08-29 18:01:30 -0400 | [diff] [blame] | 2278 | if (a->status) |
| 2279 | { |
| 2280 | if (vm->dispatch_pcap_enable) |
| 2281 | { |
| 2282 | int i; |
| 2283 | vlib_cli_output |
| 2284 | (vm, "pcap dispatch capture enabled: %d of %d pkts...", |
| 2285 | pm->n_packets_captured, pm->n_packets_to_capture); |
| 2286 | vlib_cli_output (vm, "capture to file %s", pm->file_name); |
| 2287 | |
| 2288 | for (i = 0; i < vec_len (vm->dispatch_buffer_trace_nodes); i++) |
| 2289 | { |
| 2290 | vlib_cli_output (vm, |
| 2291 | "Buffer trace of %d pkts from %U enabled...", |
| 2292 | a->buffer_traces_to_capture, |
| 2293 | format_vlib_node_name, vm, |
| 2294 | vm->dispatch_buffer_trace_nodes[i]); |
| 2295 | } |
| 2296 | } |
| 2297 | else |
| 2298 | vlib_cli_output (vm, "pcap dispatch capture disabled"); |
| 2299 | return 0; |
| 2300 | } |
| 2301 | |
| 2302 | /* Consistency checks */ |
| 2303 | |
| 2304 | /* Enable w/ capture already enabled not allowed */ |
| 2305 | if (vm->dispatch_pcap_enable && a->enable) |
| 2306 | return -7; /* VNET_API_ERROR_INVALID_VALUE */ |
| 2307 | |
| 2308 | /* Disable capture with capture already disabled, not interesting */ |
| 2309 | if (vm->dispatch_pcap_enable == 0 && a->enable == 0) |
| 2310 | return -81; /* VNET_API_ERROR_VALUE_EXIST */ |
| 2311 | |
| 2312 | /* Change number of packets to capture while capturing */ |
Dave Barach | b97641c | 2019-09-09 16:38:17 -0400 | [diff] [blame] | 2313 | if (vm->dispatch_pcap_enable && a->enable |
Dave Barach | e5948fb | 2019-08-29 18:01:30 -0400 | [diff] [blame] | 2314 | && (pm->n_packets_to_capture != a->packets_to_capture)) |
| 2315 | return -8; /* VNET_API_ERROR_INVALID_VALUE_2 */ |
| 2316 | |
| 2317 | /* Independent of enable/disable, to allow buffer trace multi nodes */ |
| 2318 | if (a->buffer_trace_node_index != ~0) |
| 2319 | { |
| 2320 | /* *INDENT-OFF* */ |
| 2321 | foreach_vlib_main (( |
| 2322 | { |
| 2323 | tm = &this_vlib_main->trace_main; |
| 2324 | tm->verbose = 0; /* not sure this ever did anything... */ |
| 2325 | vec_validate (tm->nodes, a->buffer_trace_node_index); |
| 2326 | tn = tm->nodes + a->buffer_trace_node_index; |
| 2327 | tn->limit += a->buffer_traces_to_capture; |
Dave Barach | 27d978c | 2020-11-03 09:59:06 -0500 | [diff] [blame] | 2328 | if (a->post_mortem) |
| 2329 | { |
| 2330 | tm->filter_flag = FILTER_FLAG_POST_MORTEM; |
| 2331 | tm->filter_count = ~0; |
| 2332 | } |
Dave Barach | e5948fb | 2019-08-29 18:01:30 -0400 | [diff] [blame] | 2333 | tm->trace_enable = 1; |
| 2334 | })); |
| 2335 | /* *INDENT-ON* */ |
| 2336 | vec_add1 (vm->dispatch_buffer_trace_nodes, a->buffer_trace_node_index); |
| 2337 | } |
| 2338 | |
| 2339 | if (a->enable) |
| 2340 | { |
| 2341 | /* Clean up from previous run, if any */ |
| 2342 | vec_free (pm->file_name); |
| 2343 | vec_free (pm->pcap_data); |
| 2344 | memset (pm, 0, sizeof (*pm)); |
| 2345 | |
Dave Barach | 11fb09e | 2020-08-06 12:10:09 -0400 | [diff] [blame] | 2346 | vec_validate_aligned (vnet_trace_placeholder, 2048, |
| 2347 | CLIB_CACHE_LINE_BYTES); |
Dave Barach | e5948fb | 2019-08-29 18:01:30 -0400 | [diff] [blame] | 2348 | if (pm->lock == 0) |
| 2349 | clib_spinlock_init (&(pm->lock)); |
| 2350 | |
| 2351 | if (a->filename == 0) |
| 2352 | a->filename = format (0, "/tmp/dispatch.pcap%c", 0); |
| 2353 | |
| 2354 | pm->file_name = (char *) a->filename; |
| 2355 | pm->n_packets_captured = 0; |
| 2356 | pm->packet_type = PCAP_PACKET_TYPE_vpp; |
Dave Barach | e5948fb | 2019-08-29 18:01:30 -0400 | [diff] [blame] | 2357 | pm->n_packets_to_capture = a->packets_to_capture; |
Dave Barach | 27d978c | 2020-11-03 09:59:06 -0500 | [diff] [blame] | 2358 | vm->dispatch_pcap_postmortem = a->post_mortem; |
Dave Barach | 349cd1a | 2019-10-18 14:44:05 -0400 | [diff] [blame] | 2359 | /* *INDENT-OFF* */ |
Dave Barach | 27d978c | 2020-11-03 09:59:06 -0500 | [diff] [blame] | 2360 | foreach_vlib_main (({ this_vlib_main->dispatch_pcap_enable = 1;})); |
Dave Barach | 349cd1a | 2019-10-18 14:44:05 -0400 | [diff] [blame] | 2361 | /* *INDENT-ON* */ |
Dave Barach | e5948fb | 2019-08-29 18:01:30 -0400 | [diff] [blame] | 2362 | } |
| 2363 | else |
| 2364 | { |
Dave Barach | 349cd1a | 2019-10-18 14:44:05 -0400 | [diff] [blame] | 2365 | /* *INDENT-OFF* */ |
Dave Barach | 27d978c | 2020-11-03 09:59:06 -0500 | [diff] [blame] | 2366 | foreach_vlib_main (( |
| 2367 | { |
| 2368 | this_vlib_main->dispatch_pcap_enable = 0; |
| 2369 | this_vlib_main->dispatch_pcap_postmortem = 0; |
| 2370 | tm = &this_vlib_main->trace_main; |
| 2371 | tm->filter_flag = 0; |
| 2372 | tm->filter_count = 0; |
| 2373 | tm->trace_enable = 0; |
| 2374 | })); |
Dave Barach | 349cd1a | 2019-10-18 14:44:05 -0400 | [diff] [blame] | 2375 | /* *INDENT-ON* */ |
Dave Barach | e5948fb | 2019-08-29 18:01:30 -0400 | [diff] [blame] | 2376 | vec_reset_length (vm->dispatch_buffer_trace_nodes); |
| 2377 | if (pm->n_packets_captured) |
| 2378 | { |
| 2379 | clib_error_t *error; |
| 2380 | pm->n_packets_to_capture = pm->n_packets_captured; |
| 2381 | vlib_cli_output (vm, "Write %d packets to %s, and stop capture...", |
| 2382 | pm->n_packets_captured, pm->file_name); |
| 2383 | error = pcap_write (pm); |
Andrew Yourtchenko | 4da1506 | 2019-09-11 14:14:43 +0000 | [diff] [blame] | 2384 | if (pm->flags & PCAP_MAIN_INIT_DONE) |
Dave Barach | e5948fb | 2019-08-29 18:01:30 -0400 | [diff] [blame] | 2385 | pcap_close (pm); |
| 2386 | /* Report I/O errors... */ |
| 2387 | if (error) |
| 2388 | { |
| 2389 | clib_error_report (error); |
| 2390 | return -11; /* VNET_API_ERROR_SYSCALL_ERROR_1 */ |
| 2391 | } |
| 2392 | return 0; |
| 2393 | } |
| 2394 | else |
| 2395 | return -6; /* VNET_API_ERROR_NO_SUCH_ENTRY */ |
| 2396 | } |
| 2397 | |
| 2398 | return 0; |
| 2399 | } |
| 2400 | |
| 2401 | static clib_error_t * |
| 2402 | dispatch_trace_command_fn (vlib_main_t * vm, |
| 2403 | unformat_input_t * input, vlib_cli_command_t * cmd) |
| 2404 | { |
| 2405 | unformat_input_t _line_input, *line_input = &_line_input; |
| 2406 | vlib_pcap_dispatch_trace_args_t _a, *a = &_a; |
| 2407 | u8 *filename = 0; |
| 2408 | u32 max = 1000; |
| 2409 | int rv; |
| 2410 | int enable = 0; |
| 2411 | int status = 0; |
Dave Barach | 27d978c | 2020-11-03 09:59:06 -0500 | [diff] [blame] | 2412 | int post_mortem = 0; |
Dave Barach | e5948fb | 2019-08-29 18:01:30 -0400 | [diff] [blame] | 2413 | u32 node_index = ~0, buffer_traces_to_capture = 100; |
| 2414 | |
Dave Barach | 3ae2873 | 2018-11-16 17:19:00 -0500 | [diff] [blame] | 2415 | /* Get a line of input. */ |
| 2416 | if (!unformat_user (input, unformat_line_input, line_input)) |
| 2417 | return 0; |
| 2418 | |
| 2419 | while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) |
| 2420 | { |
Dave Barach | e5948fb | 2019-08-29 18:01:30 -0400 | [diff] [blame] | 2421 | if (unformat (line_input, "on %=", &enable, 1)) |
| 2422 | ; |
| 2423 | else if (unformat (line_input, "enable %=", &enable, 1)) |
| 2424 | ; |
| 2425 | else if (unformat (line_input, "off %=", &enable, 0)) |
| 2426 | ; |
| 2427 | else if (unformat (line_input, "disable %=", &enable, 0)) |
| 2428 | ; |
Dave Barach | 3ae2873 | 2018-11-16 17:19:00 -0500 | [diff] [blame] | 2429 | else if (unformat (line_input, "max %d", &max)) |
Dave Barach | e5948fb | 2019-08-29 18:01:30 -0400 | [diff] [blame] | 2430 | ; |
| 2431 | else if (unformat (line_input, "packets-to-capture %d", &max)) |
| 2432 | ; |
| 2433 | else if (unformat (line_input, "file %U", unformat_vlib_tmpfile, |
| 2434 | &filename)) |
| 2435 | ; |
| 2436 | else if (unformat (line_input, "status %=", &status, 1)) |
| 2437 | ; |
Dave Barach | 1201a80 | 2018-11-20 12:08:39 -0500 | [diff] [blame] | 2438 | else if (unformat (line_input, "buffer-trace %U %d", |
Dave Barach | e5948fb | 2019-08-29 18:01:30 -0400 | [diff] [blame] | 2439 | unformat_vlib_node, vm, &node_index, |
| 2440 | &buffer_traces_to_capture)) |
| 2441 | ; |
Dave Barach | 27d978c | 2020-11-03 09:59:06 -0500 | [diff] [blame] | 2442 | else if (unformat (line_input, "post-mortem %=", &post_mortem, 1)) |
| 2443 | ; |
Dave Barach | 3ae2873 | 2018-11-16 17:19:00 -0500 | [diff] [blame] | 2444 | else |
| 2445 | { |
Dave Barach | e5948fb | 2019-08-29 18:01:30 -0400 | [diff] [blame] | 2446 | return clib_error_return (0, "unknown input `%U'", |
| 2447 | format_unformat_error, line_input); |
Dave Barach | 3ae2873 | 2018-11-16 17:19:00 -0500 | [diff] [blame] | 2448 | } |
| 2449 | } |
Dave Barach | e5948fb | 2019-08-29 18:01:30 -0400 | [diff] [blame] | 2450 | |
Dave Barach | 3ae2873 | 2018-11-16 17:19:00 -0500 | [diff] [blame] | 2451 | unformat_free (line_input); |
| 2452 | |
Dave Barach | e5948fb | 2019-08-29 18:01:30 -0400 | [diff] [blame] | 2453 | /* no need for memset (a, 0, sizeof (*a)), set all fields here. */ |
| 2454 | a->filename = filename; |
| 2455 | a->enable = enable; |
| 2456 | a->status = status; |
| 2457 | a->packets_to_capture = max; |
| 2458 | a->buffer_trace_node_index = node_index; |
| 2459 | a->buffer_traces_to_capture = buffer_traces_to_capture; |
Dave Barach | 27d978c | 2020-11-03 09:59:06 -0500 | [diff] [blame] | 2460 | a->post_mortem = post_mortem; |
Dave Barach | e5948fb | 2019-08-29 18:01:30 -0400 | [diff] [blame] | 2461 | |
| 2462 | rv = vlib_pcap_dispatch_trace_configure (a); |
| 2463 | |
| 2464 | switch (rv) |
Dave Barach | 3ae2873 | 2018-11-16 17:19:00 -0500 | [diff] [blame] | 2465 | { |
Dave Barach | e5948fb | 2019-08-29 18:01:30 -0400 | [diff] [blame] | 2466 | case 0: |
| 2467 | break; |
Dave Barach | 3ae2873 | 2018-11-16 17:19:00 -0500 | [diff] [blame] | 2468 | |
Dave Barach | e5948fb | 2019-08-29 18:01:30 -0400 | [diff] [blame] | 2469 | case -7: |
| 2470 | return clib_error_return (0, "dispatch trace already enabled..."); |
Dave Barach | 3ae2873 | 2018-11-16 17:19:00 -0500 | [diff] [blame] | 2471 | |
Dave Barach | e5948fb | 2019-08-29 18:01:30 -0400 | [diff] [blame] | 2472 | case -81: |
| 2473 | return clib_error_return (0, "dispatch trace already disabled..."); |
Dave Barach | 3ae2873 | 2018-11-16 17:19:00 -0500 | [diff] [blame] | 2474 | |
Dave Barach | e5948fb | 2019-08-29 18:01:30 -0400 | [diff] [blame] | 2475 | case -8: |
| 2476 | return clib_error_return |
| 2477 | (0, "can't change number of records to capture while tracing..."); |
| 2478 | |
| 2479 | case -11: |
| 2480 | return clib_error_return (0, "I/O writing trace capture..."); |
| 2481 | |
| 2482 | case -6: |
| 2483 | return clib_error_return (0, "No packets captured..."); |
| 2484 | |
| 2485 | default: |
| 2486 | vlib_cli_output (vm, "WARNING: trace configure returned %d", rv); |
| 2487 | break; |
Dave Barach | 3ae2873 | 2018-11-16 17:19:00 -0500 | [diff] [blame] | 2488 | } |
Dave Barach | e5948fb | 2019-08-29 18:01:30 -0400 | [diff] [blame] | 2489 | return 0; |
Dave Barach | 3ae2873 | 2018-11-16 17:19:00 -0500 | [diff] [blame] | 2490 | } |
| 2491 | |
| 2492 | /*? |
| 2493 | * This command is used to start or stop pcap dispatch trace capture, or show |
| 2494 | * the capture status. |
| 2495 | * |
| 2496 | * This command has the following optional parameters: |
| 2497 | * |
| 2498 | * - <b>on|off</b> - Used to start or stop capture. |
| 2499 | * |
| 2500 | * - <b>max <nn></b> - Depth of local buffer. Once '<em>nn</em>' number |
| 2501 | * of packets have been received, buffer is flushed to file. Once another |
| 2502 | * '<em>nn</em>' number of packets have been received, buffer is flushed |
| 2503 | * to file, overwriting previous write. If not entered, value defaults |
| 2504 | * to 100. Can only be updated if packet capture is off. |
| 2505 | * |
| 2506 | * - <b>file <name></b> - Used to specify the output filename. The file will |
| 2507 | * be placed in the '<em>/tmp</em>' directory, so only the filename is |
| 2508 | * supported. Directory should not be entered. If file already exists, file |
| 2509 | * will be overwritten. If no filename is provided, '<em>/tmp/vpe.pcap</em>' |
| 2510 | * will be used. Can only be updated if packet capture is off. |
| 2511 | * |
| 2512 | * - <b>status</b> - Displays the current status and configured attributes |
| 2513 | * associated with a packet capture. If packet capture is in progress, |
| 2514 | * '<em>status</em>' also will return the number of packets currently in |
| 2515 | * the local buffer. All additional attributes entered on command line |
| 2516 | * with '<em>status</em>' will be ignored and not applied. |
| 2517 | * |
| 2518 | * @cliexpar |
| 2519 | * Example of how to display the status of capture when off: |
| 2520 | * @cliexstart{pcap dispatch trace status} |
| 2521 | * max is 100, for any interface to file /tmp/vpe.pcap |
| 2522 | * pcap dispatch capture is off... |
| 2523 | * @cliexend |
| 2524 | * Example of how to start a dispatch trace capture: |
| 2525 | * @cliexstart{pcap dispatch trace on max 35 file dispatchTrace.pcap} |
Dave Barach | 1201a80 | 2018-11-20 12:08:39 -0500 | [diff] [blame] | 2526 | * pcap dispatch capture on... |
| 2527 | * @cliexend |
| 2528 | * Example of how to start a dispatch trace capture with buffer tracing |
| 2529 | * @cliexstart{pcap dispatch trace on max 10000 file dispatchTrace.pcap buffer-trace dpdk-input 1000} |
| 2530 | * pcap dispatch capture on... |
Dave Barach | 3ae2873 | 2018-11-16 17:19:00 -0500 | [diff] [blame] | 2531 | * @cliexend |
| 2532 | * Example of how to display the status of a tx packet capture in progress: |
| 2533 | * @cliexstart{pcap tx trace status} |
| 2534 | * max is 35, dispatch trace to file /tmp/vppTest.pcap |
| 2535 | * pcap tx capture is on: 20 of 35 pkts... |
| 2536 | * @cliexend |
| 2537 | * Example of how to stop a tx packet capture: |
| 2538 | * @cliexstart{vppctl pcap dispatch trace off} |
| 2539 | * captured 21 pkts... |
| 2540 | * saved to /tmp/dispatchTrace.pcap... |
Dave Barach | 27d978c | 2020-11-03 09:59:06 -0500 | [diff] [blame] | 2541 | * Example of how to start a post-mortem dispatch trace: |
| 2542 | * pcap dispatch trace on max 20000 buffer-trace |
| 2543 | * dpdk-input 3000000000 post-mortem |
Dave Barach | 3ae2873 | 2018-11-16 17:19:00 -0500 | [diff] [blame] | 2544 | * @cliexend |
| 2545 | ?*/ |
| 2546 | /* *INDENT-OFF* */ |
| 2547 | VLIB_CLI_COMMAND (pcap_dispatch_trace_command, static) = { |
| 2548 | .path = "pcap dispatch trace", |
| 2549 | .short_help = |
Dave Barach | 1201a80 | 2018-11-20 12:08:39 -0500 | [diff] [blame] | 2550 | "pcap dispatch trace [on|off] [max <nn>] [file <name>] [status]\n" |
Dave Barach | 27d978c | 2020-11-03 09:59:06 -0500 | [diff] [blame] | 2551 | " [buffer-trace <input-node-name> <nn>][post-mortem]", |
Dave Barach | e5948fb | 2019-08-29 18:01:30 -0400 | [diff] [blame] | 2552 | .function = dispatch_trace_command_fn, |
Dave Barach | 3ae2873 | 2018-11-16 17:19:00 -0500 | [diff] [blame] | 2553 | }; |
| 2554 | /* *INDENT-ON* */ |
| 2555 | |
Dave Barach | ab1a50c | 2020-10-06 14:08:16 -0400 | [diff] [blame] | 2556 | vlib_main_t * |
| 2557 | vlib_get_main_not_inline (void) |
| 2558 | { |
| 2559 | return vlib_get_main (); |
| 2560 | } |
| 2561 | |
| 2562 | elog_main_t * |
| 2563 | vlib_get_elog_main_not_inline () |
| 2564 | { |
| 2565 | return &vlib_global_main.elog_main; |
| 2566 | } |
| 2567 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 2568 | /* |
| 2569 | * fd.io coding-style-patch-verification: ON |
| 2570 | * |
| 2571 | * Local Variables: |
| 2572 | * eval: (c-set-style "gnu") |
| 2573 | * End: |
| 2574 | */ |