Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2015 Cisco and/or its affiliates. |
| 3 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | * you may not use this file except in compliance with the License. |
| 5 | * You may obtain a copy of the License at: |
| 6 | * |
| 7 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | * |
| 9 | * Unless required by applicable law or agreed to in writing, software |
| 10 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | * See the License for the specific language governing permissions and |
| 13 | * limitations under the License. |
| 14 | */ |
| 15 | /* |
| 16 | * buffer.c: allocate/free network buffers. |
| 17 | * |
| 18 | * Copyright (c) 2008 Eliot Dresselhaus |
| 19 | * |
| 20 | * Permission is hereby granted, free of charge, to any person obtaining |
| 21 | * a copy of this software and associated documentation files (the |
| 22 | * "Software"), to deal in the Software without restriction, including |
| 23 | * without limitation the rights to use, copy, modify, merge, publish, |
| 24 | * distribute, sublicense, and/or sell copies of the Software, and to |
| 25 | * permit persons to whom the Software is furnished to do so, subject to |
| 26 | * the following conditions: |
| 27 | * |
| 28 | * The above copyright notice and this permission notice shall be |
| 29 | * included in all copies or substantial portions of the Software. |
| 30 | * |
| 31 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 32 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 33 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 34 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE |
| 35 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION |
| 36 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION |
| 37 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
| 38 | */ |
| 39 | |
Chris Luke | d4024f5 | 2016-09-06 09:32:36 -0400 | [diff] [blame] | 40 | /** |
Chris Luke | d4024f5 | 2016-09-06 09:32:36 -0400 | [diff] [blame] | 41 | * @file |
| 42 | * |
| 43 | * Allocate/free network buffers. |
| 44 | */ |
| 45 | |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 46 | #include <vppinfra/linux/sysfs.h> |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 47 | #include <vlib/vlib.h> |
Damjan Marion | 374e2c5 | 2017-03-09 20:38:15 +0100 | [diff] [blame] | 48 | #include <vlib/unix/unix.h> |
Damjan Marion | 8973b07 | 2022-03-01 15:51:18 +0100 | [diff] [blame^] | 49 | #include <vlib/stats/stats.h> |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 50 | |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 51 | #define VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA 16384 |
| 52 | #define VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA_UNPRIV 8192 |
Damjan Marion | 901d16c | 2018-10-23 19:50:20 +0200 | [diff] [blame] | 53 | |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 54 | #ifdef CLIB_HAVE_VEC128 |
| 55 | /* Assumptions by vlib_buffer_free_inline: */ |
| 56 | STATIC_ASSERT_FITS_IN (vlib_buffer_t, flags, 16); |
| 57 | STATIC_ASSERT_FITS_IN (vlib_buffer_t, ref_count, 16); |
| 58 | STATIC_ASSERT_FITS_IN (vlib_buffer_t, buffer_pool_index, 16); |
| 59 | #endif |
Damjan Marion | 04a7f05 | 2017-07-10 15:06:17 +0200 | [diff] [blame] | 60 | |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 61 | /* Make sure that buffer template size is not accidentally changed */ |
| 62 | STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, template_end, 64); |
| 63 | |
| 64 | u16 __vlib_buffer_external_hdr_size = 0; |
Damjan Marion | 567e61d | 2018-10-24 17:08:26 +0200 | [diff] [blame] | 65 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 66 | uword |
| 67 | vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm, |
| 68 | vlib_buffer_t * b_first) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 69 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 70 | vlib_buffer_t *b = b_first; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 71 | uword l_first = b_first->current_length; |
| 72 | uword l = 0; |
| 73 | while (b->flags & VLIB_BUFFER_NEXT_PRESENT) |
| 74 | { |
| 75 | b = vlib_get_buffer (vm, b->next_buffer); |
| 76 | l += b->current_length; |
| 77 | } |
| 78 | b_first->total_length_not_including_first_buffer = l; |
| 79 | b_first->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; |
| 80 | return l + l_first; |
| 81 | } |
| 82 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 83 | u8 * |
Benoît Ganne | 4354317 | 2019-10-21 15:13:54 +0200 | [diff] [blame] | 84 | format_vlib_buffer_no_chain (u8 * s, va_list * args) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 85 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 86 | vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *); |
Christophe Fontaine | d3c008d | 2017-10-02 18:10:54 +0200 | [diff] [blame] | 87 | u32 indent = format_get_indent (s); |
Damjan Marion | dac0352 | 2018-02-01 15:30:13 +0100 | [diff] [blame] | 88 | u8 *a = 0; |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 89 | |
Damjan Marion | dac0352 | 2018-02-01 15:30:13 +0100 | [diff] [blame] | 90 | #define _(bit, name, v) \ |
| 91 | if (v && (b->flags & VLIB_BUFFER_##name)) \ |
| 92 | a = format (a, "%s ", v); |
| 93 | foreach_vlib_buffer_flag |
| 94 | #undef _ |
Damjan Marion | 36eb7c2 | 2019-01-18 20:45:30 +0100 | [diff] [blame] | 95 | s = format (s, "current data %d, length %d, buffer-pool %d, " |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 96 | "ref-count %u", b->current_data, b->current_length, |
| 97 | b->buffer_pool_index, b->ref_count); |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 98 | |
Mohsin Kazmi | 30d46e7 | 2022-01-11 11:34:56 +0000 | [diff] [blame] | 99 | if (b->flags & VLIB_BUFFER_NEXT_PRESENT) |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 100 | s = format (s, ", totlen-nifb %d", |
| 101 | b->total_length_not_including_first_buffer); |
| 102 | |
| 103 | if (b->flags & VLIB_BUFFER_IS_TRACED) |
Dave Barach | a638c18 | 2019-06-21 18:24:07 -0400 | [diff] [blame] | 104 | s = format (s, ", trace handle 0x%x", b->trace_handle); |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 105 | |
Damjan Marion | dac0352 | 2018-02-01 15:30:13 +0100 | [diff] [blame] | 106 | if (a) |
| 107 | s = format (s, "\n%U%v", format_white_space, indent, a); |
| 108 | vec_free (a); |
| 109 | |
Benoît Ganne | 4354317 | 2019-10-21 15:13:54 +0200 | [diff] [blame] | 110 | return s; |
| 111 | } |
| 112 | |
| 113 | u8 * |
| 114 | format_vlib_buffer (u8 * s, va_list * args) |
| 115 | { |
| 116 | vlib_main_t *vm = vlib_get_main (); |
| 117 | vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *); |
| 118 | u32 indent = format_get_indent (s); |
| 119 | |
| 120 | s = format (s, "%U", format_vlib_buffer_no_chain, b); |
| 121 | |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 122 | while (b->flags & VLIB_BUFFER_NEXT_PRESENT) |
| 123 | { |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 124 | u32 next_buffer = b->next_buffer; |
| 125 | b = vlib_get_buffer (vm, next_buffer); |
| 126 | |
Damjan Marion | c47ed03 | 2017-01-25 14:18:03 +0100 | [diff] [blame] | 127 | s = |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 128 | format (s, "\n%Unext-buffer 0x%x, segment length %d, ref-count %u", |
Damjan Marion | c47ed03 | 2017-01-25 14:18:03 +0100 | [diff] [blame] | 129 | format_white_space, indent, next_buffer, b->current_length, |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 130 | b->ref_count); |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 131 | } |
| 132 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 133 | return s; |
| 134 | } |
| 135 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 136 | u8 * |
| 137 | format_vlib_buffer_and_data (u8 * s, va_list * args) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 138 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 139 | vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *); |
| 140 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 141 | s = format (s, "%U, %U", |
| 142 | format_vlib_buffer, b, |
| 143 | format_hex_bytes, vlib_buffer_get_current (b), 64); |
| 144 | |
| 145 | return s; |
| 146 | } |
| 147 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 148 | static u8 * |
| 149 | format_vlib_buffer_known_state (u8 * s, va_list * args) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 150 | { |
| 151 | vlib_buffer_known_state_t state = va_arg (*args, vlib_buffer_known_state_t); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 152 | char *t; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 153 | |
| 154 | switch (state) |
| 155 | { |
| 156 | case VLIB_BUFFER_UNKNOWN: |
| 157 | t = "unknown"; |
| 158 | break; |
| 159 | |
| 160 | case VLIB_BUFFER_KNOWN_ALLOCATED: |
| 161 | t = "known-allocated"; |
| 162 | break; |
| 163 | |
| 164 | case VLIB_BUFFER_KNOWN_FREE: |
| 165 | t = "known-free"; |
| 166 | break; |
| 167 | |
| 168 | default: |
| 169 | t = "invalid"; |
| 170 | break; |
| 171 | } |
| 172 | |
| 173 | return format (s, "%s", t); |
| 174 | } |
| 175 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 176 | u8 * |
| 177 | format_vlib_buffer_contents (u8 * s, va_list * va) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 178 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 179 | vlib_main_t *vm = va_arg (*va, vlib_main_t *); |
| 180 | vlib_buffer_t *b = va_arg (*va, vlib_buffer_t *); |
| 181 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 182 | while (1) |
| 183 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 184 | vec_add (s, vlib_buffer_get_current (b), b->current_length); |
| 185 | if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 186 | break; |
| 187 | b = vlib_get_buffer (vm, b->next_buffer); |
| 188 | } |
| 189 | |
| 190 | return s; |
| 191 | } |
| 192 | |
| 193 | static u8 * |
| 194 | vlib_validate_buffer_helper (vlib_main_t * vm, |
| 195 | u32 bi, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 196 | uword follow_buffer_next, uword ** unique_hash) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 197 | { |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 198 | vlib_buffer_main_t *bm = vm->buffer_main; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 199 | vlib_buffer_t *b = vlib_get_buffer (vm, bi); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 200 | |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 201 | if (vec_len (bm->buffer_pools) <= b->buffer_pool_index) |
| 202 | return format (0, "unknown buffer pool 0x%x", b->buffer_pool_index); |
| 203 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 204 | if ((signed) b->current_data < (signed) -VLIB_BUFFER_PRE_DATA_SIZE) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 205 | return format (0, "current data %d before pre-data", b->current_data); |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 206 | |
Damjan Marion | 8934a04 | 2019-02-09 23:29:26 +0100 | [diff] [blame] | 207 | if (b->current_data + b->current_length > |
| 208 | vlib_buffer_get_default_data_size (vm)) |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 209 | return format (0, "%d-%d beyond end of buffer %d", b->current_data, |
Damjan Marion | 8934a04 | 2019-02-09 23:29:26 +0100 | [diff] [blame] | 210 | b->current_length, vlib_buffer_get_default_data_size (vm)); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 211 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 212 | if (follow_buffer_next && (b->flags & VLIB_BUFFER_NEXT_PRESENT)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 213 | { |
| 214 | vlib_buffer_known_state_t k; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 215 | u8 *msg, *result; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 216 | |
Damjan Marion | d50e347 | 2019-01-20 00:03:56 +0100 | [diff] [blame] | 217 | k = vlib_buffer_is_known (vm, b->next_buffer); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 218 | if (k != VLIB_BUFFER_KNOWN_ALLOCATED) |
| 219 | return format (0, "next 0x%x: %U", |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 220 | b->next_buffer, format_vlib_buffer_known_state, k); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 221 | |
| 222 | if (unique_hash) |
| 223 | { |
| 224 | if (hash_get (*unique_hash, b->next_buffer)) |
| 225 | return format (0, "duplicate buffer 0x%x", b->next_buffer); |
| 226 | |
| 227 | hash_set1 (*unique_hash, b->next_buffer); |
| 228 | } |
| 229 | |
| 230 | msg = vlib_validate_buffer (vm, b->next_buffer, follow_buffer_next); |
| 231 | if (msg) |
| 232 | { |
| 233 | result = format (0, "next 0x%x: %v", b->next_buffer, msg); |
| 234 | vec_free (msg); |
| 235 | return result; |
| 236 | } |
| 237 | } |
| 238 | |
| 239 | return 0; |
| 240 | } |
| 241 | |
| 242 | u8 * |
| 243 | vlib_validate_buffer (vlib_main_t * vm, u32 bi, uword follow_buffer_next) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 244 | { |
| 245 | return vlib_validate_buffer_helper (vm, bi, follow_buffer_next, |
| 246 | /* unique_hash */ 0); |
| 247 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 248 | |
| 249 | u8 * |
| 250 | vlib_validate_buffers (vlib_main_t * vm, |
| 251 | u32 * buffers, |
| 252 | uword next_buffer_stride, |
| 253 | uword n_buffers, |
| 254 | vlib_buffer_known_state_t known_state, |
| 255 | uword follow_buffer_next) |
| 256 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 257 | uword i, *hash; |
| 258 | u32 bi, *b = buffers; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 259 | vlib_buffer_known_state_t k; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 260 | u8 *msg = 0, *result = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 261 | |
| 262 | hash = hash_create (0, 0); |
| 263 | for (i = 0; i < n_buffers; i++) |
| 264 | { |
| 265 | bi = b[0]; |
| 266 | b += next_buffer_stride; |
| 267 | |
| 268 | /* Buffer is not unique. */ |
| 269 | if (hash_get (hash, bi)) |
| 270 | { |
| 271 | msg = format (0, "not unique"); |
| 272 | goto done; |
| 273 | } |
| 274 | |
Damjan Marion | d50e347 | 2019-01-20 00:03:56 +0100 | [diff] [blame] | 275 | k = vlib_buffer_is_known (vm, bi); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 276 | if (k != known_state) |
| 277 | { |
| 278 | msg = format (0, "is %U; expected %U", |
| 279 | format_vlib_buffer_known_state, k, |
| 280 | format_vlib_buffer_known_state, known_state); |
| 281 | goto done; |
| 282 | } |
| 283 | |
| 284 | msg = vlib_validate_buffer_helper (vm, bi, follow_buffer_next, &hash); |
| 285 | if (msg) |
| 286 | goto done; |
| 287 | |
| 288 | hash_set1 (hash, bi); |
| 289 | } |
| 290 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 291 | done: |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 292 | if (msg) |
| 293 | { |
| 294 | result = format (0, "0x%x: %v", bi, msg); |
| 295 | vec_free (msg); |
| 296 | } |
| 297 | hash_free (hash); |
| 298 | return result; |
| 299 | } |
| 300 | |
Paul Vinciguerra | 8feeaff | 2019-03-27 11:25:48 -0700 | [diff] [blame] | 301 | /* When debugging validate that given buffers are either known allocated |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 302 | or known free. */ |
Damjan Marion | c8a26c6 | 2017-11-24 20:15:23 +0100 | [diff] [blame] | 303 | void |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 304 | vlib_buffer_validate_alloc_free (vlib_main_t * vm, |
| 305 | u32 * buffers, |
| 306 | uword n_buffers, |
| 307 | vlib_buffer_known_state_t expected_state) |
| 308 | { |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 309 | vlib_buffer_main_t *bm = vm->buffer_main; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 310 | u32 *b; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 311 | uword i, bi, is_free; |
| 312 | |
| 313 | if (CLIB_DEBUG == 0) |
| 314 | return; |
| 315 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 316 | is_free = expected_state == VLIB_BUFFER_KNOWN_ALLOCATED; |
| 317 | b = buffers; |
| 318 | for (i = 0; i < n_buffers; i++) |
| 319 | { |
| 320 | vlib_buffer_known_state_t known; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 321 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 322 | bi = b[0]; |
| 323 | b += 1; |
Damjan Marion | d50e347 | 2019-01-20 00:03:56 +0100 | [diff] [blame] | 324 | known = vlib_buffer_is_known (vm, bi); |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 325 | |
| 326 | if (known == VLIB_BUFFER_UNKNOWN && |
| 327 | expected_state == VLIB_BUFFER_KNOWN_FREE) |
| 328 | known = VLIB_BUFFER_KNOWN_FREE; |
| 329 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 330 | if (known != expected_state) |
| 331 | { |
Klement Sekera | e82c73a | 2019-01-30 18:29:28 +0100 | [diff] [blame] | 332 | clib_panic ("%s %U buffer 0x%x", is_free ? "freeing" : "allocating", |
| 333 | format_vlib_buffer_known_state, known, bi); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 334 | } |
| 335 | |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 336 | clib_spinlock_lock (&bm->buffer_known_hash_lockp); |
| 337 | hash_set (bm->buffer_known_hash, bi, is_free ? VLIB_BUFFER_KNOWN_FREE : |
| 338 | VLIB_BUFFER_KNOWN_ALLOCATED); |
| 339 | clib_spinlock_unlock (&bm->buffer_known_hash_lockp); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 340 | } |
| 341 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 342 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 343 | void |
| 344 | vlib_packet_template_init (vlib_main_t * vm, |
| 345 | vlib_packet_template_t * t, |
| 346 | void *packet_data, |
| 347 | uword n_packet_data_bytes, |
Damjan Marion | d1274cb | 2018-03-13 21:32:17 +0100 | [diff] [blame] | 348 | uword min_n_buffers_each_alloc, char *fmt, ...) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 349 | { |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 350 | va_list va; |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 351 | |
| 352 | va_start (va, fmt); |
Damjan Marion | 671e60e | 2018-12-30 18:09:59 +0100 | [diff] [blame] | 353 | t->name = va_format (0, fmt, &va); |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 354 | va_end (va); |
| 355 | |
| 356 | vlib_worker_thread_barrier_sync (vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 357 | |
Dave Barach | b7b9299 | 2018-10-17 10:38:51 -0400 | [diff] [blame] | 358 | clib_memset (t, 0, sizeof (t[0])); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 359 | |
| 360 | vec_add (t->packet_data, packet_data, n_packet_data_bytes); |
Damjan Marion | d1274cb | 2018-03-13 21:32:17 +0100 | [diff] [blame] | 361 | t->min_n_buffers_each_alloc = min_n_buffers_each_alloc; |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 362 | vlib_worker_thread_barrier_release (vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 363 | } |
| 364 | |
| 365 | void * |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 366 | vlib_packet_template_get_packet (vlib_main_t * vm, |
| 367 | vlib_packet_template_t * t, u32 * bi_result) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 368 | { |
| 369 | u32 bi; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 370 | vlib_buffer_t *b; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 371 | |
| 372 | if (vlib_buffer_alloc (vm, &bi, 1) != 1) |
| 373 | return 0; |
| 374 | |
| 375 | *bi_result = bi; |
| 376 | |
| 377 | b = vlib_get_buffer (vm, bi); |
Dave Barach | 178cf49 | 2018-11-13 16:34:13 -0500 | [diff] [blame] | 378 | clib_memcpy_fast (vlib_buffer_get_current (b), |
| 379 | t->packet_data, vec_len (t->packet_data)); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 380 | b->current_length = vec_len (t->packet_data); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 381 | |
| 382 | return b->data; |
| 383 | } |
| 384 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 385 | /* Append given data to end of buffer, possibly allocating new buffers. */ |
Dave Barach | 3a63fc5 | 2019-01-07 09:15:47 -0500 | [diff] [blame] | 386 | int |
Damjan Marion | ab9b7ec | 2019-01-18 20:24:44 +0100 | [diff] [blame] | 387 | vlib_buffer_add_data (vlib_main_t * vm, u32 * buffer_index, void *data, |
| 388 | u32 n_data_bytes) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 389 | { |
| 390 | u32 n_buffer_bytes, n_left, n_left_this_buffer, bi; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 391 | vlib_buffer_t *b; |
| 392 | void *d; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 393 | |
Dave Barach | 3a63fc5 | 2019-01-07 09:15:47 -0500 | [diff] [blame] | 394 | bi = *buffer_index; |
Damjan Marion | ab9b7ec | 2019-01-18 20:24:44 +0100 | [diff] [blame] | 395 | if (bi == ~0 && 1 != vlib_buffer_alloc (vm, &bi, 1)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 396 | goto out_of_buffers; |
| 397 | |
| 398 | d = data; |
| 399 | n_left = n_data_bytes; |
Damjan Marion | 8934a04 | 2019-02-09 23:29:26 +0100 | [diff] [blame] | 400 | n_buffer_bytes = vlib_buffer_get_default_data_size (vm); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 401 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 402 | b = vlib_get_buffer (vm, bi); |
| 403 | b->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID; |
| 404 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 405 | /* Get to the end of the chain before we try to append data... */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 406 | while (b->flags & VLIB_BUFFER_NEXT_PRESENT) |
| 407 | b = vlib_get_buffer (vm, b->next_buffer); |
| 408 | |
| 409 | while (1) |
| 410 | { |
| 411 | u32 n; |
| 412 | |
| 413 | ASSERT (n_buffer_bytes >= b->current_length); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 414 | n_left_this_buffer = |
| 415 | n_buffer_bytes - (b->current_data + b->current_length); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 416 | n = clib_min (n_left_this_buffer, n_left); |
Dave Barach | 178cf49 | 2018-11-13 16:34:13 -0500 | [diff] [blame] | 417 | clib_memcpy_fast (vlib_buffer_get_current (b) + b->current_length, d, |
| 418 | n); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 419 | b->current_length += n; |
| 420 | n_left -= n; |
| 421 | if (n_left == 0) |
| 422 | break; |
| 423 | |
| 424 | d += n; |
Damjan Marion | ab9b7ec | 2019-01-18 20:24:44 +0100 | [diff] [blame] | 425 | if (1 != vlib_buffer_alloc (vm, &b->next_buffer, 1)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 426 | goto out_of_buffers; |
| 427 | |
| 428 | b->flags |= VLIB_BUFFER_NEXT_PRESENT; |
| 429 | |
| 430 | b = vlib_get_buffer (vm, b->next_buffer); |
| 431 | } |
| 432 | |
Dave Barach | 3a63fc5 | 2019-01-07 09:15:47 -0500 | [diff] [blame] | 433 | *buffer_index = bi; |
| 434 | return 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 435 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 436 | out_of_buffers: |
Dave Barach | 3a63fc5 | 2019-01-07 09:15:47 -0500 | [diff] [blame] | 437 | clib_warning ("out of buffers"); |
| 438 | return 1; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 439 | } |
| 440 | |
Pierre Pfister | 328e99b | 2016-02-12 13:18:42 +0000 | [diff] [blame] | 441 | u16 |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 442 | vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 443 | vlib_buffer_t * first, |
Damjan Marion | dac0352 | 2018-02-01 15:30:13 +0100 | [diff] [blame] | 444 | vlib_buffer_t ** last, void *data, |
| 445 | u16 data_len) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 446 | { |
Pierre Pfister | 328e99b | 2016-02-12 13:18:42 +0000 | [diff] [blame] | 447 | vlib_buffer_t *l = *last; |
Damjan Marion | 8934a04 | 2019-02-09 23:29:26 +0100 | [diff] [blame] | 448 | u32 n_buffer_bytes = vlib_buffer_get_default_data_size (vm); |
Pierre Pfister | 328e99b | 2016-02-12 13:18:42 +0000 | [diff] [blame] | 449 | u16 copied = 0; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 450 | ASSERT (n_buffer_bytes >= l->current_length + l->current_data); |
| 451 | while (data_len) |
| 452 | { |
| 453 | u16 max = n_buffer_bytes - l->current_length - l->current_data; |
| 454 | if (max == 0) |
| 455 | { |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 456 | if (1 != vlib_buffer_alloc_from_pool (vm, &l->next_buffer, 1, |
| 457 | first->buffer_pool_index)) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 458 | return copied; |
Eyal Bari | b688fb1 | 2018-11-12 16:13:49 +0200 | [diff] [blame] | 459 | *last = l = vlib_buffer_chain_buffer (vm, l, l->next_buffer); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 460 | max = n_buffer_bytes - l->current_length - l->current_data; |
| 461 | } |
Pierre Pfister | 328e99b | 2016-02-12 13:18:42 +0000 | [diff] [blame] | 462 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 463 | u16 len = (data_len > max) ? max : data_len; |
Dave Barach | 178cf49 | 2018-11-13 16:34:13 -0500 | [diff] [blame] | 464 | clib_memcpy_fast (vlib_buffer_get_current (l) + l->current_length, |
| 465 | data + copied, len); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 466 | vlib_buffer_chain_increase_length (first, l, len); |
| 467 | data_len -= len; |
| 468 | copied += len; |
| 469 | } |
Pierre Pfister | 328e99b | 2016-02-12 13:18:42 +0000 | [diff] [blame] | 470 | return copied; |
| 471 | } |
| 472 | |
Benoît Ganne | c16fe46 | 2020-06-15 12:24:36 +0200 | [diff] [blame] | 473 | static uword |
| 474 | vlib_buffer_alloc_size (uword ext_hdr_size, uword data_size) |
| 475 | { |
Damjan Marion | 0be1b76 | 2020-09-04 12:34:58 +0200 | [diff] [blame] | 476 | uword alloc_size = ext_hdr_size + sizeof (vlib_buffer_t) + data_size; |
| 477 | alloc_size = CLIB_CACHE_LINE_ROUND (alloc_size); |
| 478 | |
| 479 | /* in case when we have even number of cachelines, we add one more for |
| 480 | * better cache occupancy */ |
| 481 | alloc_size |= CLIB_CACHE_LINE_BYTES; |
| 482 | |
| 483 | return alloc_size; |
Benoît Ganne | c16fe46 | 2020-06-15 12:24:36 +0200 | [diff] [blame] | 484 | } |
| 485 | |
Damjan Marion | b592d1b | 2019-02-28 23:16:11 +0100 | [diff] [blame] | 486 | u8 |
| 487 | vlib_buffer_pool_create (vlib_main_t * vm, char *name, u32 data_size, |
| 488 | u32 physmem_map_index) |
Damjan Marion | 04a7f05 | 2017-07-10 15:06:17 +0200 | [diff] [blame] | 489 | { |
Damjan Marion | d50e347 | 2019-01-20 00:03:56 +0100 | [diff] [blame] | 490 | vlib_buffer_main_t *bm = vm->buffer_main; |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 491 | vlib_buffer_pool_t *bp; |
Damjan Marion | 68b4da6 | 2018-09-30 18:26:20 +0200 | [diff] [blame] | 492 | vlib_physmem_map_t *m = vlib_physmem_get_map (vm, physmem_map_index); |
| 493 | uword start = pointer_to_uword (m->base); |
Damjan Marion | 8e8d3c8 | 2018-10-23 22:54:40 +0200 | [diff] [blame] | 494 | uword size = (uword) m->n_pages << m->log2_page_size; |
Damjan Marion | 672ab69 | 2019-02-06 17:56:57 +0100 | [diff] [blame] | 495 | uword i, j; |
Damjan Marion | b592d1b | 2019-02-28 23:16:11 +0100 | [diff] [blame] | 496 | u32 alloc_size, n_alloc_per_page; |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 497 | |
Damjan Marion | b592d1b | 2019-02-28 23:16:11 +0100 | [diff] [blame] | 498 | if (vec_len (bm->buffer_pools) >= 255) |
| 499 | return ~0; |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 500 | |
Tianyu Li | 70b1cbd | 2021-06-23 07:35:03 +0000 | [diff] [blame] | 501 | vec_add2_aligned (bm->buffer_pools, bp, 1, CLIB_CACHE_LINE_BYTES); |
Damjan Marion | 04a7f05 | 2017-07-10 15:06:17 +0200 | [diff] [blame] | 502 | |
| 503 | if (bm->buffer_mem_size == 0) |
| 504 | { |
| 505 | bm->buffer_mem_start = start; |
| 506 | bm->buffer_mem_size = size; |
| 507 | } |
| 508 | else if (start < bm->buffer_mem_start) |
| 509 | { |
| 510 | bm->buffer_mem_size += bm->buffer_mem_start - start; |
| 511 | bm->buffer_mem_start = start; |
| 512 | if (size > bm->buffer_mem_size) |
| 513 | bm->buffer_mem_size = size; |
| 514 | } |
| 515 | else if (start > bm->buffer_mem_start) |
| 516 | { |
| 517 | uword new_size = start - bm->buffer_mem_start + size; |
| 518 | if (new_size > bm->buffer_mem_size) |
| 519 | bm->buffer_mem_size = new_size; |
| 520 | } |
| 521 | |
| 522 | if ((u64) bm->buffer_mem_size > |
| 523 | ((u64) 1 << (32 + CLIB_LOG2_CACHE_LINE_BYTES))) |
| 524 | { |
| 525 | clib_panic ("buffer memory size out of range!"); |
| 526 | } |
Damjan Marion | 149ba77 | 2017-10-12 13:09:26 +0200 | [diff] [blame] | 527 | |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 528 | bp->start = start; |
| 529 | bp->size = size; |
| 530 | bp->index = bp - bm->buffer_pools; |
| 531 | bp->buffer_template.buffer_pool_index = bp->index; |
| 532 | bp->buffer_template.ref_count = 1; |
| 533 | bp->physmem_map_index = physmem_map_index; |
| 534 | bp->name = format (0, "%s%c", name, 0); |
| 535 | bp->data_size = data_size; |
| 536 | bp->numa_node = m->numa_node; |
Damjan Marion | d1274cb | 2018-03-13 21:32:17 +0100 | [diff] [blame] | 537 | |
Damjan Marion | 6ffb7c6 | 2021-03-26 13:06:13 +0100 | [diff] [blame] | 538 | vec_validate_aligned (bp->threads, vlib_get_n_threads () - 1, |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 539 | CLIB_CACHE_LINE_BYTES); |
| 540 | |
Benoît Ganne | c16fe46 | 2020-06-15 12:24:36 +0200 | [diff] [blame] | 541 | alloc_size = vlib_buffer_alloc_size (bm->ext_hdr_size, data_size); |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 542 | n_alloc_per_page = (1ULL << m->log2_page_size) / alloc_size; |
| 543 | |
| 544 | /* preallocate buffer indices memory */ |
Damjan Marion | b6e8b1a | 2019-03-12 18:14:15 +0100 | [diff] [blame] | 545 | bp->n_buffers = m->n_pages * n_alloc_per_page; |
| 546 | bp->buffers = clib_mem_alloc_aligned (bp->n_buffers * sizeof (u32), |
| 547 | CLIB_CACHE_LINE_BYTES); |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 548 | |
| 549 | clib_spinlock_init (&bp->lock); |
| 550 | |
Damjan Marion | 672ab69 | 2019-02-06 17:56:57 +0100 | [diff] [blame] | 551 | for (j = 0; j < m->n_pages; j++) |
| 552 | for (i = 0; i < n_alloc_per_page; i++) |
| 553 | { |
| 554 | u8 *p; |
| 555 | u32 bi; |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 556 | |
Damjan Marion | 672ab69 | 2019-02-06 17:56:57 +0100 | [diff] [blame] | 557 | p = m->base + (j << m->log2_page_size) + i * alloc_size; |
| 558 | p += bm->ext_hdr_size; |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 559 | |
Dave Barach | c74b43c | 2020-04-09 17:24:07 -0400 | [diff] [blame] | 560 | /* |
| 561 | * Waste 1 buffer (maximum) so that 0 is never a valid buffer index. |
| 562 | * Allows various places to ASSERT (bi != 0). Much easier |
| 563 | * than debugging downstream crashes in successor nodes. |
| 564 | */ |
| 565 | if (p == m->base) |
| 566 | continue; |
| 567 | |
Damjan Marion | 672ab69 | 2019-02-06 17:56:57 +0100 | [diff] [blame] | 568 | vlib_buffer_copy_template ((vlib_buffer_t *) p, &bp->buffer_template); |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 569 | |
Damjan Marion | 672ab69 | 2019-02-06 17:56:57 +0100 | [diff] [blame] | 570 | bi = vlib_get_buffer_index (vm, (vlib_buffer_t *) p); |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 571 | |
Damjan Marion | b6e8b1a | 2019-03-12 18:14:15 +0100 | [diff] [blame] | 572 | bp->buffers[bp->n_avail++] = bi; |
| 573 | |
Damjan Marion | 672ab69 | 2019-02-06 17:56:57 +0100 | [diff] [blame] | 574 | vlib_get_buffer (vm, bi); |
| 575 | } |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 576 | |
Damjan Marion | b592d1b | 2019-02-28 23:16:11 +0100 | [diff] [blame] | 577 | return bp->index; |
Damjan Marion | 04a7f05 | 2017-07-10 15:06:17 +0200 | [diff] [blame] | 578 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 579 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 580 | static u8 * |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 581 | format_vlib_buffer_pool (u8 * s, va_list * va) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 582 | { |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 583 | vlib_main_t *vm = va_arg (*va, vlib_main_t *); |
| 584 | vlib_buffer_pool_t *bp = va_arg (*va, vlib_buffer_pool_t *); |
| 585 | vlib_buffer_pool_thread_t *bpt; |
| 586 | u32 cached = 0; |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 587 | |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 588 | if (!bp) |
| 589 | return format (s, "%-20s%=6s%=6s%=6s%=11s%=6s%=8s%=8s%=8s", |
| 590 | "Pool Name", "Index", "NUMA", "Size", "Data Size", |
| 591 | "Total", "Avail", "Cached", "Used"); |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 592 | |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 593 | /* *INDENT-OFF* */ |
| 594 | vec_foreach (bpt, bp->threads) |
Damjan Marion | b6e8b1a | 2019-03-12 18:14:15 +0100 | [diff] [blame] | 595 | cached += bpt->n_cached; |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 596 | /* *INDENT-ON* */ |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 597 | |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 598 | s = format (s, "%-20s%=6d%=6d%=6u%=11u%=6u%=8u%=8u%=8u", |
| 599 | bp->name, bp->index, bp->numa_node, bp->data_size + |
| 600 | sizeof (vlib_buffer_t) + vm->buffer_main->ext_hdr_size, |
Damjan Marion | b6e8b1a | 2019-03-12 18:14:15 +0100 | [diff] [blame] | 601 | bp->data_size, bp->n_buffers, bp->n_avail, cached, |
| 602 | bp->n_buffers - bp->n_avail - cached); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 603 | |
| 604 | return s; |
| 605 | } |
| 606 | |
Benoît Ganne | e09a233 | 2021-03-09 15:37:49 +0100 | [diff] [blame] | 607 | u8 * |
| 608 | format_vlib_buffer_pool_all (u8 *s, va_list *va) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 609 | { |
Benoît Ganne | e09a233 | 2021-03-09 15:37:49 +0100 | [diff] [blame] | 610 | vlib_main_t *vm = va_arg (*va, vlib_main_t *); |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 611 | vlib_buffer_main_t *bm = vm->buffer_main; |
| 612 | vlib_buffer_pool_t *bp; |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 613 | |
Benoît Ganne | e09a233 | 2021-03-09 15:37:49 +0100 | [diff] [blame] | 614 | s = format (s, "%U", format_vlib_buffer_pool, vm, 0); |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 615 | |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 616 | vec_foreach (bp, bm->buffer_pools) |
Benoît Ganne | e09a233 | 2021-03-09 15:37:49 +0100 | [diff] [blame] | 617 | s = format (s, "\n%U", format_vlib_buffer_pool, vm, bp); |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 618 | |
Benoît Ganne | e09a233 | 2021-03-09 15:37:49 +0100 | [diff] [blame] | 619 | return s; |
| 620 | } |
| 621 | |
| 622 | static clib_error_t * |
| 623 | show_buffers (vlib_main_t *vm, unformat_input_t *input, |
| 624 | vlib_cli_command_t *cmd) |
| 625 | { |
| 626 | vlib_cli_output (vm, "%U", format_vlib_buffer_pool_all, vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 627 | return 0; |
| 628 | } |
| 629 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 630 | /* *INDENT-OFF* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 631 | VLIB_CLI_COMMAND (show_buffers_command, static) = { |
| 632 | .path = "show buffers", |
| 633 | .short_help = "Show packet buffer allocation", |
| 634 | .function = show_buffers, |
| 635 | }; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 636 | /* *INDENT-ON* */ |
| 637 | |
Damjan Marion | 49d66f1 | 2017-07-20 18:10:35 +0200 | [diff] [blame] | 638 | clib_error_t * |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 639 | vlib_buffer_worker_init (vlib_main_t * vm) |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 640 | { |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 641 | vlib_buffer_main_t *bm = vm->buffer_main; |
| 642 | vlib_buffer_pool_t *bp; |
| 643 | |
| 644 | /* *INDENT-OFF* */ |
| 645 | vec_foreach (bp, bm->buffer_pools) |
| 646 | { |
| 647 | clib_spinlock_lock (&bp->lock); |
Damjan Marion | 6ffb7c6 | 2021-03-26 13:06:13 +0100 | [diff] [blame] | 648 | vec_validate_aligned (bp->threads, vlib_get_n_threads () - 1, |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 649 | CLIB_CACHE_LINE_BYTES); |
| 650 | clib_spinlock_unlock (&bp->lock); |
| 651 | } |
| 652 | /* *INDENT-ON* */ |
| 653 | |
| 654 | return 0; |
| 655 | } |
| 656 | |
| 657 | VLIB_WORKER_INIT_FUNCTION (vlib_buffer_worker_init); |
| 658 | |
| 659 | static clib_error_t * |
Nathan Skrzypczak | 6155902 | 2020-11-23 16:25:21 +0100 | [diff] [blame] | 660 | vlib_buffer_main_init_numa_alloc (struct vlib_main_t *vm, u32 numa_node, |
| 661 | u32 * physmem_map_index, |
| 662 | clib_mem_page_sz_t log2_page_size, |
| 663 | u8 unpriv) |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 664 | { |
| 665 | vlib_buffer_main_t *bm = vm->buffer_main; |
Nathan Skrzypczak | 6155902 | 2020-11-23 16:25:21 +0100 | [diff] [blame] | 666 | u32 buffers_per_numa = bm->buffers_per_numa; |
Damjan Marion | 49d66f1 | 2017-07-20 18:10:35 +0200 | [diff] [blame] | 667 | clib_error_t *error; |
Nathan Skrzypczak | 6155902 | 2020-11-23 16:25:21 +0100 | [diff] [blame] | 668 | u32 buffer_size; |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 669 | uword n_pages, pagesize; |
Nathan Skrzypczak | 6155902 | 2020-11-23 16:25:21 +0100 | [diff] [blame] | 670 | u8 *name = 0; |
Damjan Marion | 567e61d | 2018-10-24 17:08:26 +0200 | [diff] [blame] | 671 | |
Nathan Skrzypczak | 6155902 | 2020-11-23 16:25:21 +0100 | [diff] [blame] | 672 | ASSERT (log2_page_size != CLIB_MEM_PAGE_SZ_UNKNOWN); |
Damjan Marion | 49d66f1 | 2017-07-20 18:10:35 +0200 | [diff] [blame] | 673 | |
Nathan Skrzypczak | 6155902 | 2020-11-23 16:25:21 +0100 | [diff] [blame] | 674 | pagesize = clib_mem_page_bytes (log2_page_size); |
| 675 | buffer_size = vlib_buffer_alloc_size (bm->ext_hdr_size, |
| 676 | vlib_buffer_get_default_data_size |
| 677 | (vm)); |
Benoît Ganne | c16fe46 | 2020-06-15 12:24:36 +0200 | [diff] [blame] | 678 | if (buffer_size > pagesize) |
Nathan Skrzypczak | 6155902 | 2020-11-23 16:25:21 +0100 | [diff] [blame] | 679 | return clib_error_return (0, "buffer size (%llu) is greater than page " |
| 680 | "size (%llu)", buffer_size, pagesize); |
Benoît Ganne | c16fe46 | 2020-06-15 12:24:36 +0200 | [diff] [blame] | 681 | |
Nathan Skrzypczak | 6155902 | 2020-11-23 16:25:21 +0100 | [diff] [blame] | 682 | if (buffers_per_numa == 0) |
| 683 | buffers_per_numa = unpriv ? VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA_UNPRIV : |
| 684 | VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA; |
| 685 | |
| 686 | name = format (0, "buffers-numa-%d%c", numa_node, 0); |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 687 | n_pages = (buffers_per_numa - 1) / (pagesize / buffer_size) + 1; |
| 688 | error = vlib_physmem_shared_map_create (vm, (char *) name, |
| 689 | n_pages * pagesize, |
| 690 | min_log2 (pagesize), numa_node, |
Nathan Skrzypczak | 6155902 | 2020-11-23 16:25:21 +0100 | [diff] [blame] | 691 | physmem_map_index); |
| 692 | vec_free (name); |
| 693 | return error; |
| 694 | } |
Damjan Marion | 567e61d | 2018-10-24 17:08:26 +0200 | [diff] [blame] | 695 | |
Nathan Skrzypczak | 6155902 | 2020-11-23 16:25:21 +0100 | [diff] [blame] | 696 | static clib_error_t * |
| 697 | vlib_buffer_main_init_numa_node (struct vlib_main_t *vm, u32 numa_node, |
| 698 | u8 * index) |
| 699 | { |
| 700 | vlib_buffer_main_t *bm = vm->buffer_main; |
| 701 | u32 physmem_map_index; |
| 702 | clib_error_t *error; |
| 703 | u8 *name = 0; |
| 704 | |
| 705 | if (bm->log2_page_size == CLIB_MEM_PAGE_SZ_UNKNOWN) |
Damjan Marion | 567e61d | 2018-10-24 17:08:26 +0200 | [diff] [blame] | 706 | { |
Nathan Skrzypczak | 6155902 | 2020-11-23 16:25:21 +0100 | [diff] [blame] | 707 | error = vlib_buffer_main_init_numa_alloc (vm, numa_node, |
| 708 | &physmem_map_index, |
| 709 | CLIB_MEM_PAGE_SZ_DEFAULT_HUGE, |
| 710 | 0 /* unpriv */ ); |
| 711 | if (!error) |
| 712 | goto buffer_pool_create; |
| 713 | |
| 714 | /* If alloc failed, retry without hugepages */ |
| 715 | vlib_log_warn (bm->log_default, |
| 716 | "numa[%u] falling back to non-hugepage backed " |
| 717 | "buffer pool (%U)", numa_node, format_clib_error, error); |
Damjan Marion | 567e61d | 2018-10-24 17:08:26 +0200 | [diff] [blame] | 718 | clib_error_free (error); |
Nathan Skrzypczak | 6155902 | 2020-11-23 16:25:21 +0100 | [diff] [blame] | 719 | |
| 720 | error = vlib_buffer_main_init_numa_alloc (vm, numa_node, |
| 721 | &physmem_map_index, |
| 722 | CLIB_MEM_PAGE_SZ_DEFAULT, |
| 723 | 1 /* unpriv */ ); |
Damjan Marion | 567e61d | 2018-10-24 17:08:26 +0200 | [diff] [blame] | 724 | } |
Nathan Skrzypczak | 6155902 | 2020-11-23 16:25:21 +0100 | [diff] [blame] | 725 | else |
| 726 | error = vlib_buffer_main_init_numa_alloc (vm, numa_node, |
| 727 | &physmem_map_index, |
| 728 | bm->log2_page_size, |
| 729 | 0 /* unpriv */ ); |
Damjan Marion | 567e61d | 2018-10-24 17:08:26 +0200 | [diff] [blame] | 730 | if (error) |
Nathan Skrzypczak | 6155902 | 2020-11-23 16:25:21 +0100 | [diff] [blame] | 731 | return error; |
Damjan Marion | 49d66f1 | 2017-07-20 18:10:35 +0200 | [diff] [blame] | 732 | |
Nathan Skrzypczak | 6155902 | 2020-11-23 16:25:21 +0100 | [diff] [blame] | 733 | buffer_pool_create: |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 734 | name = format (name, "default-numa-%d%c", numa_node, 0); |
Damjan Marion | b592d1b | 2019-02-28 23:16:11 +0100 | [diff] [blame] | 735 | *index = vlib_buffer_pool_create (vm, (char *) name, |
| 736 | vlib_buffer_get_default_data_size (vm), |
| 737 | physmem_map_index); |
| 738 | |
| 739 | if (*index == (u8) ~ 0) |
Nathan Skrzypczak | 6155902 | 2020-11-23 16:25:21 +0100 | [diff] [blame] | 740 | error = clib_error_return (0, "maximum number of buffer pools reached"); |
Guanghua Zhang | 785daf4 | 2019-08-18 08:35:04 +0800 | [diff] [blame] | 741 | vec_free (name); |
Nathan Skrzypczak | 6155902 | 2020-11-23 16:25:21 +0100 | [diff] [blame] | 742 | |
| 743 | |
Guanghua Zhang | 785daf4 | 2019-08-18 08:35:04 +0800 | [diff] [blame] | 744 | return error; |
Damjan Marion | 5de3fec | 2019-02-06 14:22:32 +0100 | [diff] [blame] | 745 | } |
| 746 | |
| 747 | void |
| 748 | vlib_buffer_main_alloc (vlib_main_t * vm) |
| 749 | { |
| 750 | vlib_buffer_main_t *bm; |
| 751 | |
| 752 | if (vm->buffer_main) |
| 753 | return; |
| 754 | |
| 755 | vm->buffer_main = bm = clib_mem_alloc (sizeof (bm[0])); |
| 756 | clib_memset (vm->buffer_main, 0, sizeof (bm[0])); |
| 757 | bm->default_data_size = VLIB_BUFFER_DEFAULT_DATA_SIZE; |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 758 | } |
| 759 | |
Filip Tehlar | d2bbdef | 2019-02-22 05:05:53 -0800 | [diff] [blame] | 760 | static u32 |
| 761 | buffer_get_cached (vlib_buffer_pool_t * bp) |
| 762 | { |
| 763 | u32 cached = 0; |
| 764 | vlib_buffer_pool_thread_t *bpt; |
| 765 | |
Filip Tehlar | b806d0e | 2019-02-28 04:36:10 -0800 | [diff] [blame] | 766 | clib_spinlock_lock (&bp->lock); |
| 767 | |
Filip Tehlar | d2bbdef | 2019-02-22 05:05:53 -0800 | [diff] [blame] | 768 | /* *INDENT-OFF* */ |
| 769 | vec_foreach (bpt, bp->threads) |
Damjan Marion | b6e8b1a | 2019-03-12 18:14:15 +0100 | [diff] [blame] | 770 | cached += bpt->n_cached; |
Filip Tehlar | d2bbdef | 2019-02-22 05:05:53 -0800 | [diff] [blame] | 771 | /* *INDENT-ON* */ |
| 772 | |
Filip Tehlar | b806d0e | 2019-02-28 04:36:10 -0800 | [diff] [blame] | 773 | clib_spinlock_unlock (&bp->lock); |
| 774 | |
Filip Tehlar | d2bbdef | 2019-02-22 05:05:53 -0800 | [diff] [blame] | 775 | return cached; |
| 776 | } |
| 777 | |
| 778 | static vlib_buffer_pool_t * |
Ole Troan | 849cbe2 | 2019-02-26 13:10:56 +0100 | [diff] [blame] | 779 | buffer_get_by_index (vlib_buffer_main_t * bm, u32 index) |
Filip Tehlar | d2bbdef | 2019-02-22 05:05:53 -0800 | [diff] [blame] | 780 | { |
| 781 | vlib_buffer_pool_t *bp; |
Ole Troan | 849cbe2 | 2019-02-26 13:10:56 +0100 | [diff] [blame] | 782 | if (!bm->buffer_pools || vec_len (bm->buffer_pools) < index) |
| 783 | return 0; |
| 784 | bp = vec_elt_at_index (bm->buffer_pools, index); |
Steven Luong | 9515510 | 2019-03-05 15:29:22 -0800 | [diff] [blame] | 785 | |
Ole Troan | 849cbe2 | 2019-02-26 13:10:56 +0100 | [diff] [blame] | 786 | return bp; |
Filip Tehlar | d2bbdef | 2019-02-22 05:05:53 -0800 | [diff] [blame] | 787 | } |
| 788 | |
| 789 | static void |
Damjan Marion | 8973b07 | 2022-03-01 15:51:18 +0100 | [diff] [blame^] | 790 | buffer_gauges_collect_used_fn (vlib_stats_collector_data_t *d) |
Filip Tehlar | d2bbdef | 2019-02-22 05:05:53 -0800 | [diff] [blame] | 791 | { |
| 792 | vlib_main_t *vm = vlib_get_main (); |
Damjan Marion | 8973b07 | 2022-03-01 15:51:18 +0100 | [diff] [blame^] | 793 | vlib_buffer_pool_t *bp = |
| 794 | buffer_get_by_index (vm->buffer_main, d->private_data); |
Filip Tehlar | d2bbdef | 2019-02-22 05:05:53 -0800 | [diff] [blame] | 795 | if (!bp) |
| 796 | return; |
| 797 | |
Damjan Marion | 8973b07 | 2022-03-01 15:51:18 +0100 | [diff] [blame^] | 798 | d->entry->value = bp->n_buffers - bp->n_avail - buffer_get_cached (bp); |
Filip Tehlar | d2bbdef | 2019-02-22 05:05:53 -0800 | [diff] [blame] | 799 | } |
| 800 | |
| 801 | static void |
Damjan Marion | 8973b07 | 2022-03-01 15:51:18 +0100 | [diff] [blame^] | 802 | buffer_gauges_collect_available_fn (vlib_stats_collector_data_t *d) |
Filip Tehlar | d2bbdef | 2019-02-22 05:05:53 -0800 | [diff] [blame] | 803 | { |
| 804 | vlib_main_t *vm = vlib_get_main (); |
Damjan Marion | 8973b07 | 2022-03-01 15:51:18 +0100 | [diff] [blame^] | 805 | vlib_buffer_pool_t *bp = |
| 806 | buffer_get_by_index (vm->buffer_main, d->private_data); |
Filip Tehlar | d2bbdef | 2019-02-22 05:05:53 -0800 | [diff] [blame] | 807 | if (!bp) |
| 808 | return; |
| 809 | |
Damjan Marion | 8973b07 | 2022-03-01 15:51:18 +0100 | [diff] [blame^] | 810 | d->entry->value = bp->n_avail; |
Filip Tehlar | d2bbdef | 2019-02-22 05:05:53 -0800 | [diff] [blame] | 811 | } |
| 812 | |
| 813 | static void |
Damjan Marion | 8973b07 | 2022-03-01 15:51:18 +0100 | [diff] [blame^] | 814 | buffer_gauges_collect_cached_fn (vlib_stats_collector_data_t *d) |
Filip Tehlar | d2bbdef | 2019-02-22 05:05:53 -0800 | [diff] [blame] | 815 | { |
| 816 | vlib_main_t *vm = vlib_get_main (); |
Damjan Marion | 8973b07 | 2022-03-01 15:51:18 +0100 | [diff] [blame^] | 817 | vlib_buffer_pool_t *bp = |
| 818 | buffer_get_by_index (vm->buffer_main, d->private_data); |
Filip Tehlar | d2bbdef | 2019-02-22 05:05:53 -0800 | [diff] [blame] | 819 | if (!bp) |
| 820 | return; |
| 821 | |
Damjan Marion | 8973b07 | 2022-03-01 15:51:18 +0100 | [diff] [blame^] | 822 | d->entry->value = buffer_get_cached (bp); |
Filip Tehlar | d2bbdef | 2019-02-22 05:05:53 -0800 | [diff] [blame] | 823 | } |
| 824 | |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 825 | clib_error_t * |
Filip Tehlar | d2bbdef | 2019-02-22 05:05:53 -0800 | [diff] [blame] | 826 | vlib_buffer_main_init (struct vlib_main_t * vm) |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 827 | { |
| 828 | vlib_buffer_main_t *bm; |
| 829 | clib_error_t *err; |
Damjan Marion | b592d1b | 2019-02-28 23:16:11 +0100 | [diff] [blame] | 830 | clib_bitmap_t *bmp = 0, *bmp_has_memory = 0; |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 831 | u32 numa_node; |
Filip Tehlar | d2bbdef | 2019-02-22 05:05:53 -0800 | [diff] [blame] | 832 | vlib_buffer_pool_t *bp; |
Damjan Marion | b592d1b | 2019-02-28 23:16:11 +0100 | [diff] [blame] | 833 | u8 *name = 0, first_valid_buffer_pool_index = ~0; |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 834 | |
Damjan Marion | 5de3fec | 2019-02-06 14:22:32 +0100 | [diff] [blame] | 835 | vlib_buffer_main_alloc (vm); |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 836 | |
| 837 | bm = vm->buffer_main; |
| 838 | bm->log_default = vlib_log_register_class ("buffer", 0); |
| 839 | bm->ext_hdr_size = __vlib_buffer_external_hdr_size; |
| 840 | |
| 841 | clib_spinlock_init (&bm->buffer_known_hash_lockp); |
| 842 | |
Damjan Marion | b592d1b | 2019-02-28 23:16:11 +0100 | [diff] [blame] | 843 | if ((err = clib_sysfs_read ("/sys/devices/system/node/online", "%U", |
| 844 | unformat_bitmap_list, &bmp))) |
| 845 | clib_error_free (err); |
| 846 | |
| 847 | if ((err = clib_sysfs_read ("/sys/devices/system/node/has_memory", "%U", |
| 848 | unformat_bitmap_list, &bmp_has_memory))) |
| 849 | clib_error_free (err); |
| 850 | |
| 851 | if (bmp && bmp_has_memory) |
| 852 | bmp = clib_bitmap_and (bmp, bmp_has_memory); |
| 853 | |
| 854 | /* no info from sysfs, assuming that only numa 0 exists */ |
| 855 | if (bmp == 0) |
| 856 | bmp = clib_bitmap_set (bmp, 0, 1); |
| 857 | |
| 858 | if (clib_bitmap_last_set (bmp) >= VLIB_BUFFER_MAX_NUMA_NODES) |
| 859 | clib_panic ("system have more than %u NUMA nodes", |
| 860 | VLIB_BUFFER_MAX_NUMA_NODES); |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 861 | |
| 862 | /* *INDENT-OFF* */ |
Damjan Marion | f0ca1e8 | 2020-12-13 23:26:56 +0100 | [diff] [blame] | 863 | clib_bitmap_foreach (numa_node, bmp) |
Damjan Marion | b592d1b | 2019-02-28 23:16:11 +0100 | [diff] [blame] | 864 | { |
| 865 | u8 *index = bm->default_buffer_pool_index_for_numa + numa_node; |
| 866 | index[0] = ~0; |
| 867 | if ((err = vlib_buffer_main_init_numa_node (vm, numa_node, index))) |
| 868 | { |
| 869 | clib_error_report (err); |
| 870 | clib_error_free (err); |
| 871 | continue; |
| 872 | } |
| 873 | |
| 874 | if (first_valid_buffer_pool_index == 0xff) |
| 875 | first_valid_buffer_pool_index = index[0]; |
Damjan Marion | f0ca1e8 | 2020-12-13 23:26:56 +0100 | [diff] [blame] | 876 | } |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 877 | /* *INDENT-ON* */ |
| 878 | |
Damjan Marion | b592d1b | 2019-02-28 23:16:11 +0100 | [diff] [blame] | 879 | if (first_valid_buffer_pool_index == (u8) ~ 0) |
| 880 | { |
| 881 | err = clib_error_return (0, "failed to allocate buffer pool(s)"); |
| 882 | goto done; |
| 883 | } |
| 884 | |
| 885 | /* *INDENT-OFF* */ |
Damjan Marion | f0ca1e8 | 2020-12-13 23:26:56 +0100 | [diff] [blame] | 886 | clib_bitmap_foreach (numa_node, bmp) |
Damjan Marion | b592d1b | 2019-02-28 23:16:11 +0100 | [diff] [blame] | 887 | { |
| 888 | if (bm->default_buffer_pool_index_for_numa[numa_node] == (u8) ~0) |
| 889 | bm->default_buffer_pool_index_for_numa[numa_node] = |
| 890 | first_valid_buffer_pool_index; |
Damjan Marion | f0ca1e8 | 2020-12-13 23:26:56 +0100 | [diff] [blame] | 891 | } |
Damjan Marion | b592d1b | 2019-02-28 23:16:11 +0100 | [diff] [blame] | 892 | /* *INDENT-ON* */ |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 893 | |
Filip Tehlar | d2bbdef | 2019-02-22 05:05:53 -0800 | [diff] [blame] | 894 | vec_foreach (bp, bm->buffer_pools) |
| 895 | { |
Damjan Marion | 8973b07 | 2022-03-01 15:51:18 +0100 | [diff] [blame^] | 896 | vlib_stats_collector_reg_t reg = { .private_data = bp - bm->buffer_pools }; |
Damjan Marion | b592d1b | 2019-02-28 23:16:11 +0100 | [diff] [blame] | 897 | if (bp->n_buffers == 0) |
| 898 | continue; |
| 899 | |
Damjan Marion | 8973b07 | 2022-03-01 15:51:18 +0100 | [diff] [blame^] | 900 | reg.entry_index = |
| 901 | vlib_stats_add_gauge ("/buffer-pools/%s/cached", bp->name); |
| 902 | reg.collect_fn = buffer_gauges_collect_cached_fn; |
| 903 | vlib_stats_register_collector_fn (®); |
Damjan Marion | b592d1b | 2019-02-28 23:16:11 +0100 | [diff] [blame] | 904 | |
Damjan Marion | 8973b07 | 2022-03-01 15:51:18 +0100 | [diff] [blame^] | 905 | reg.entry_index = vlib_stats_add_gauge ("/buffer-pools/%s/used", bp->name); |
| 906 | reg.collect_fn = buffer_gauges_collect_used_fn; |
| 907 | vlib_stats_register_collector_fn (®); |
Damjan Marion | b592d1b | 2019-02-28 23:16:11 +0100 | [diff] [blame] | 908 | |
Damjan Marion | 8973b07 | 2022-03-01 15:51:18 +0100 | [diff] [blame^] | 909 | reg.entry_index = |
| 910 | vlib_stats_add_gauge ("/buffer-pools/%s/available", bp->name); |
| 911 | reg.collect_fn = buffer_gauges_collect_available_fn; |
| 912 | vlib_stats_register_collector_fn (®); |
Filip Tehlar | d2bbdef | 2019-02-22 05:05:53 -0800 | [diff] [blame] | 913 | } |
| 914 | |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 915 | done: |
| 916 | vec_free (bmp); |
Damjan Marion | b592d1b | 2019-02-28 23:16:11 +0100 | [diff] [blame] | 917 | vec_free (bmp_has_memory); |
| 918 | vec_free (name); |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 919 | return err; |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 920 | } |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 921 | |
Damjan Marion | 49d66f1 | 2017-07-20 18:10:35 +0200 | [diff] [blame] | 922 | static clib_error_t * |
| 923 | vlib_buffers_configure (vlib_main_t * vm, unformat_input_t * input) |
| 924 | { |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 925 | vlib_buffer_main_t *bm; |
| 926 | |
Damjan Marion | 5de3fec | 2019-02-06 14:22:32 +0100 | [diff] [blame] | 927 | vlib_buffer_main_alloc (vm); |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 928 | |
| 929 | bm = vm->buffer_main; |
Nathan Skrzypczak | 6155902 | 2020-11-23 16:25:21 +0100 | [diff] [blame] | 930 | bm->log2_page_size = CLIB_MEM_PAGE_SZ_UNKNOWN; |
Damjan Marion | 49d66f1 | 2017-07-20 18:10:35 +0200 | [diff] [blame] | 931 | |
| 932 | while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) |
| 933 | { |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 934 | if (unformat (input, "buffers-per-numa %u", &bm->buffers_per_numa)) |
| 935 | ; |
Nathan Skrzypczak | 6155902 | 2020-11-23 16:25:21 +0100 | [diff] [blame] | 936 | else if (unformat (input, "page-size %U", unformat_log2_page_size, |
| 937 | &bm->log2_page_size)) |
| 938 | ; |
Damjan Marion | 5de3fec | 2019-02-06 14:22:32 +0100 | [diff] [blame] | 939 | else if (unformat (input, "default data-size %u", |
| 940 | &bm->default_data_size)) |
| 941 | ; |
Damjan Marion | 49d66f1 | 2017-07-20 18:10:35 +0200 | [diff] [blame] | 942 | else |
| 943 | return unformat_parse_error (input); |
| 944 | } |
| 945 | |
| 946 | unformat_free (input); |
| 947 | return 0; |
| 948 | } |
| 949 | |
| 950 | VLIB_EARLY_CONFIG_FUNCTION (vlib_buffers_configure, "buffers"); |
| 951 | |
Dave Barach | c74b43c | 2020-04-09 17:24:07 -0400 | [diff] [blame] | 952 | #if VLIB_BUFFER_ALLOC_FAULT_INJECTOR > 0 |
| 953 | u32 |
| 954 | vlib_buffer_alloc_may_fail (vlib_main_t * vm, u32 n_buffers) |
| 955 | { |
| 956 | f64 r; |
| 957 | |
| 958 | r = random_f64 (&vm->buffer_alloc_success_seed); |
| 959 | |
| 960 | /* Fail this request? */ |
| 961 | if (r > vm->buffer_alloc_success_rate) |
| 962 | n_buffers--; |
| 963 | /* 5% chance of returning nothing at all */ |
| 964 | if (r > vm->buffer_alloc_success_rate && r > 0.95) |
| 965 | n_buffers = 0; |
| 966 | |
| 967 | return n_buffers; |
| 968 | } |
| 969 | #endif |
Damjan Marion | 49d66f1 | 2017-07-20 18:10:35 +0200 | [diff] [blame] | 970 | |
Benoît Ganne | e09a233 | 2021-03-09 15:37:49 +0100 | [diff] [blame] | 971 | __clib_export int |
| 972 | vlib_buffer_set_alloc_free_callback ( |
| 973 | vlib_main_t *vm, vlib_buffer_alloc_free_callback_t *alloc_callback_fn, |
| 974 | vlib_buffer_alloc_free_callback_t *free_callback_fn) |
| 975 | { |
| 976 | vlib_buffer_main_t *bm = vm->buffer_main; |
| 977 | if ((alloc_callback_fn && bm->alloc_callback_fn) || |
| 978 | (free_callback_fn && bm->free_callback_fn)) |
| 979 | return 1; |
| 980 | bm->alloc_callback_fn = alloc_callback_fn; |
| 981 | bm->free_callback_fn = free_callback_fn; |
| 982 | return 0; |
| 983 | } |
| 984 | |
Chris Luke | d4024f5 | 2016-09-06 09:32:36 -0400 | [diff] [blame] | 985 | /** @endcond */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 986 | /* |
| 987 | * fd.io coding-style-patch-verification: ON |
| 988 | * |
| 989 | * Local Variables: |
| 990 | * eval: (c-set-style "gnu") |
| 991 | * End: |
| 992 | */ |