Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2015 Cisco and/or its affiliates. |
| 3 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | * you may not use this file except in compliance with the License. |
| 5 | * You may obtain a copy of the License at: |
| 6 | * |
| 7 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | * |
| 9 | * Unless required by applicable law or agreed to in writing, software |
| 10 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | * See the License for the specific language governing permissions and |
| 13 | * limitations under the License. |
| 14 | */ |
| 15 | /* |
| 16 | * buffer.c: allocate/free network buffers. |
| 17 | * |
| 18 | * Copyright (c) 2008 Eliot Dresselhaus |
| 19 | * |
| 20 | * Permission is hereby granted, free of charge, to any person obtaining |
| 21 | * a copy of this software and associated documentation files (the |
| 22 | * "Software"), to deal in the Software without restriction, including |
| 23 | * without limitation the rights to use, copy, modify, merge, publish, |
| 24 | * distribute, sublicense, and/or sell copies of the Software, and to |
| 25 | * permit persons to whom the Software is furnished to do so, subject to |
| 26 | * the following conditions: |
| 27 | * |
| 28 | * The above copyright notice and this permission notice shall be |
| 29 | * included in all copies or substantial portions of the Software. |
| 30 | * |
| 31 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 32 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 33 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 34 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE |
| 35 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION |
| 36 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION |
| 37 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
| 38 | */ |
| 39 | |
Chris Luke | d4024f5 | 2016-09-06 09:32:36 -0400 | [diff] [blame] | 40 | /** |
Chris Luke | d4024f5 | 2016-09-06 09:32:36 -0400 | [diff] [blame] | 41 | * @file |
| 42 | * |
| 43 | * Allocate/free network buffers. |
| 44 | */ |
| 45 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 46 | #include <vlib/vlib.h> |
Damjan Marion | 374e2c5 | 2017-03-09 20:38:15 +0100 | [diff] [blame] | 47 | #include <vlib/unix/unix.h> |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 48 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 49 | uword |
| 50 | vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm, |
| 51 | vlib_buffer_t * b_first) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 52 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 53 | vlib_buffer_t *b = b_first; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 54 | uword l_first = b_first->current_length; |
| 55 | uword l = 0; |
| 56 | while (b->flags & VLIB_BUFFER_NEXT_PRESENT) |
| 57 | { |
| 58 | b = vlib_get_buffer (vm, b->next_buffer); |
| 59 | l += b->current_length; |
| 60 | } |
| 61 | b_first->total_length_not_including_first_buffer = l; |
| 62 | b_first->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; |
| 63 | return l + l_first; |
| 64 | } |
| 65 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 66 | u8 * |
| 67 | format_vlib_buffer (u8 * s, va_list * args) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 68 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 69 | vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *); |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 70 | uword indent = format_get_indent (s); |
| 71 | |
Damjan Marion | c47ed03 | 2017-01-25 14:18:03 +0100 | [diff] [blame] | 72 | s = format (s, "current data %d, length %d, free-list %d, clone-count %u", |
| 73 | b->current_data, b->current_length, b->free_list_index, |
| 74 | b->n_add_refs); |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 75 | |
| 76 | if (b->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID) |
| 77 | s = format (s, ", totlen-nifb %d", |
| 78 | b->total_length_not_including_first_buffer); |
| 79 | |
| 80 | if (b->flags & VLIB_BUFFER_IS_TRACED) |
| 81 | s = format (s, ", trace 0x%x", b->trace_index); |
| 82 | |
| 83 | while (b->flags & VLIB_BUFFER_NEXT_PRESENT) |
| 84 | { |
| 85 | vlib_main_t *vm = vlib_get_main (); |
| 86 | u32 next_buffer = b->next_buffer; |
| 87 | b = vlib_get_buffer (vm, next_buffer); |
| 88 | |
Damjan Marion | c47ed03 | 2017-01-25 14:18:03 +0100 | [diff] [blame] | 89 | s = |
| 90 | format (s, "\n%Unext-buffer 0x%x, segment length %d, clone-count %u", |
| 91 | format_white_space, indent, next_buffer, b->current_length, |
| 92 | b->n_add_refs); |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 93 | } |
| 94 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 95 | return s; |
| 96 | } |
| 97 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 98 | u8 * |
| 99 | format_vlib_buffer_and_data (u8 * s, va_list * args) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 100 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 101 | vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *); |
| 102 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 103 | s = format (s, "%U, %U", |
| 104 | format_vlib_buffer, b, |
| 105 | format_hex_bytes, vlib_buffer_get_current (b), 64); |
| 106 | |
| 107 | return s; |
| 108 | } |
| 109 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 110 | static u8 * |
| 111 | format_vlib_buffer_known_state (u8 * s, va_list * args) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 112 | { |
| 113 | vlib_buffer_known_state_t state = va_arg (*args, vlib_buffer_known_state_t); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 114 | char *t; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 115 | |
| 116 | switch (state) |
| 117 | { |
| 118 | case VLIB_BUFFER_UNKNOWN: |
| 119 | t = "unknown"; |
| 120 | break; |
| 121 | |
| 122 | case VLIB_BUFFER_KNOWN_ALLOCATED: |
| 123 | t = "known-allocated"; |
| 124 | break; |
| 125 | |
| 126 | case VLIB_BUFFER_KNOWN_FREE: |
| 127 | t = "known-free"; |
| 128 | break; |
| 129 | |
| 130 | default: |
| 131 | t = "invalid"; |
| 132 | break; |
| 133 | } |
| 134 | |
| 135 | return format (s, "%s", t); |
| 136 | } |
| 137 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 138 | u8 * |
| 139 | format_vlib_buffer_contents (u8 * s, va_list * va) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 140 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 141 | vlib_main_t *vm = va_arg (*va, vlib_main_t *); |
| 142 | vlib_buffer_t *b = va_arg (*va, vlib_buffer_t *); |
| 143 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 144 | while (1) |
| 145 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 146 | vec_add (s, vlib_buffer_get_current (b), b->current_length); |
| 147 | if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 148 | break; |
| 149 | b = vlib_get_buffer (vm, b->next_buffer); |
| 150 | } |
| 151 | |
| 152 | return s; |
| 153 | } |
| 154 | |
| 155 | static u8 * |
| 156 | vlib_validate_buffer_helper (vlib_main_t * vm, |
| 157 | u32 bi, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 158 | uword follow_buffer_next, uword ** unique_hash) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 159 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 160 | vlib_buffer_t *b = vlib_get_buffer (vm, bi); |
| 161 | vlib_buffer_main_t *bm = vm->buffer_main; |
| 162 | vlib_buffer_free_list_t *fl; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 163 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 164 | if (pool_is_free_index (bm->buffer_free_list_pool, b->free_list_index)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 165 | return format (0, "unknown free list 0x%x", b->free_list_index); |
| 166 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 167 | fl = pool_elt_at_index (bm->buffer_free_list_pool, b->free_list_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 168 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 169 | if ((signed) b->current_data < (signed) -VLIB_BUFFER_PRE_DATA_SIZE) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 170 | return format (0, "current data %d before pre-data", b->current_data); |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 171 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 172 | if (b->current_data + b->current_length > fl->n_data_bytes) |
| 173 | return format (0, "%d-%d beyond end of buffer %d", |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 174 | b->current_data, b->current_length, fl->n_data_bytes); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 175 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 176 | if (follow_buffer_next && (b->flags & VLIB_BUFFER_NEXT_PRESENT)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 177 | { |
| 178 | vlib_buffer_known_state_t k; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 179 | u8 *msg, *result; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 180 | |
| 181 | k = vlib_buffer_is_known (vm, b->next_buffer); |
| 182 | if (k != VLIB_BUFFER_KNOWN_ALLOCATED) |
| 183 | return format (0, "next 0x%x: %U", |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 184 | b->next_buffer, format_vlib_buffer_known_state, k); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 185 | |
| 186 | if (unique_hash) |
| 187 | { |
| 188 | if (hash_get (*unique_hash, b->next_buffer)) |
| 189 | return format (0, "duplicate buffer 0x%x", b->next_buffer); |
| 190 | |
| 191 | hash_set1 (*unique_hash, b->next_buffer); |
| 192 | } |
| 193 | |
| 194 | msg = vlib_validate_buffer (vm, b->next_buffer, follow_buffer_next); |
| 195 | if (msg) |
| 196 | { |
| 197 | result = format (0, "next 0x%x: %v", b->next_buffer, msg); |
| 198 | vec_free (msg); |
| 199 | return result; |
| 200 | } |
| 201 | } |
| 202 | |
| 203 | return 0; |
| 204 | } |
| 205 | |
| 206 | u8 * |
| 207 | vlib_validate_buffer (vlib_main_t * vm, u32 bi, uword follow_buffer_next) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 208 | { |
| 209 | return vlib_validate_buffer_helper (vm, bi, follow_buffer_next, |
| 210 | /* unique_hash */ 0); |
| 211 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 212 | |
| 213 | u8 * |
| 214 | vlib_validate_buffers (vlib_main_t * vm, |
| 215 | u32 * buffers, |
| 216 | uword next_buffer_stride, |
| 217 | uword n_buffers, |
| 218 | vlib_buffer_known_state_t known_state, |
| 219 | uword follow_buffer_next) |
| 220 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 221 | uword i, *hash; |
| 222 | u32 bi, *b = buffers; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 223 | vlib_buffer_known_state_t k; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 224 | u8 *msg = 0, *result = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 225 | |
| 226 | hash = hash_create (0, 0); |
| 227 | for (i = 0; i < n_buffers; i++) |
| 228 | { |
| 229 | bi = b[0]; |
| 230 | b += next_buffer_stride; |
| 231 | |
| 232 | /* Buffer is not unique. */ |
| 233 | if (hash_get (hash, bi)) |
| 234 | { |
| 235 | msg = format (0, "not unique"); |
| 236 | goto done; |
| 237 | } |
| 238 | |
| 239 | k = vlib_buffer_is_known (vm, bi); |
| 240 | if (k != known_state) |
| 241 | { |
| 242 | msg = format (0, "is %U; expected %U", |
| 243 | format_vlib_buffer_known_state, k, |
| 244 | format_vlib_buffer_known_state, known_state); |
| 245 | goto done; |
| 246 | } |
| 247 | |
| 248 | msg = vlib_validate_buffer_helper (vm, bi, follow_buffer_next, &hash); |
| 249 | if (msg) |
| 250 | goto done; |
| 251 | |
| 252 | hash_set1 (hash, bi); |
| 253 | } |
| 254 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 255 | done: |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 256 | if (msg) |
| 257 | { |
| 258 | result = format (0, "0x%x: %v", bi, msg); |
| 259 | vec_free (msg); |
| 260 | } |
| 261 | hash_free (hash); |
| 262 | return result; |
| 263 | } |
| 264 | |
Dave Barach | 80f54e2 | 2017-03-08 19:08:56 -0500 | [diff] [blame] | 265 | /* |
| 266 | * Hand-craft a static vector w/ length 1, so vec_len(vlib_mains) =1 |
| 267 | * and vlib_mains[0] = &vlib_global_main from the beginning of time. |
| 268 | * |
| 269 | * The only place which should ever expand vlib_mains is start_workers() |
| 270 | * in threads.c. It knows about the bootstrap vector. |
| 271 | */ |
| 272 | /* *INDENT-OFF* */ |
| 273 | static struct |
| 274 | { |
| 275 | vec_header_t h; |
| 276 | vlib_main_t *vm; |
| 277 | } __attribute__ ((packed)) __bootstrap_vlib_main_vector |
| 278 | __attribute__ ((aligned (CLIB_CACHE_LINE_BYTES))) = |
| 279 | { |
| 280 | .h.len = 1, |
| 281 | .vm = &vlib_global_main, |
| 282 | }; |
| 283 | /* *INDENT-ON* */ |
| 284 | |
| 285 | vlib_main_t **vlib_mains = &__bootstrap_vlib_main_vector.vm; |
| 286 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 287 | |
| 288 | /* When dubugging validate that given buffers are either known allocated |
| 289 | or known free. */ |
Damjan Marion | c47ed03 | 2017-01-25 14:18:03 +0100 | [diff] [blame] | 290 | static void |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 291 | vlib_buffer_validate_alloc_free (vlib_main_t * vm, |
| 292 | u32 * buffers, |
| 293 | uword n_buffers, |
| 294 | vlib_buffer_known_state_t expected_state) |
| 295 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 296 | u32 *b; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 297 | uword i, bi, is_free; |
| 298 | |
| 299 | if (CLIB_DEBUG == 0) |
| 300 | return; |
| 301 | |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 302 | ASSERT (vlib_get_thread_index () == 0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 303 | |
| 304 | /* smp disaster check */ |
Dave Barach | 80f54e2 | 2017-03-08 19:08:56 -0500 | [diff] [blame] | 305 | if (vec_len (vlib_mains) > 1) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 306 | ASSERT (vm == vlib_mains[0]); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 307 | |
| 308 | is_free = expected_state == VLIB_BUFFER_KNOWN_ALLOCATED; |
| 309 | b = buffers; |
| 310 | for (i = 0; i < n_buffers; i++) |
| 311 | { |
| 312 | vlib_buffer_known_state_t known; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 313 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 314 | bi = b[0]; |
| 315 | b += 1; |
| 316 | known = vlib_buffer_is_known (vm, bi); |
| 317 | if (known != expected_state) |
| 318 | { |
| 319 | ASSERT (0); |
| 320 | vlib_panic_with_msg |
| 321 | (vm, "%s %U buffer 0x%x", |
| 322 | is_free ? "freeing" : "allocating", |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 323 | format_vlib_buffer_known_state, known, bi); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 324 | } |
| 325 | |
| 326 | vlib_buffer_set_known_state |
| 327 | (vm, bi, |
| 328 | is_free ? VLIB_BUFFER_KNOWN_FREE : VLIB_BUFFER_KNOWN_ALLOCATED); |
| 329 | } |
| 330 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 331 | |
Damjan Marion | 8a6a3b2 | 2017-01-17 14:12:42 +0100 | [diff] [blame] | 332 | void |
| 333 | vlib_buffer_merge_free_lists (vlib_buffer_free_list_t * dst, |
| 334 | vlib_buffer_free_list_t * src) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 335 | { |
| 336 | uword l; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 337 | u32 *d; |
| 338 | |
Damjan Marion | bd69a5f | 2017-02-05 23:44:42 +0100 | [diff] [blame] | 339 | l = vec_len (src->buffers); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 340 | if (l > 0) |
| 341 | { |
Damjan Marion | bd69a5f | 2017-02-05 23:44:42 +0100 | [diff] [blame] | 342 | vec_add2_aligned (dst->buffers, d, l, CLIB_CACHE_LINE_BYTES); |
| 343 | clib_memcpy (d, src->buffers, l * sizeof (d[0])); |
| 344 | vec_free (src->buffers); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 345 | } |
| 346 | } |
| 347 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 348 | /* Add buffer free list. */ |
| 349 | static u32 |
| 350 | vlib_buffer_create_free_list_helper (vlib_main_t * vm, |
| 351 | u32 n_data_bytes, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 352 | u32 is_public, u32 is_default, u8 * name) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 353 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 354 | vlib_buffer_main_t *bm = vm->buffer_main; |
| 355 | vlib_buffer_free_list_t *f; |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 356 | int i; |
| 357 | |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 358 | ASSERT (vlib_get_thread_index () == 0); |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 359 | |
| 360 | if (!is_default && pool_elts (bm->buffer_free_list_pool) == 0) |
| 361 | { |
| 362 | u32 default_free_free_list_index; |
| 363 | |
| 364 | /* *INDENT-OFF* */ |
| 365 | default_free_free_list_index = |
| 366 | vlib_buffer_create_free_list_helper |
| 367 | (vm, |
| 368 | /* default buffer size */ VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES, |
| 369 | /* is_public */ 1, |
| 370 | /* is_default */ 1, |
| 371 | (u8 *) "default"); |
| 372 | /* *INDENT-ON* */ |
| 373 | ASSERT (default_free_free_list_index == |
| 374 | VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); |
| 375 | |
| 376 | if (n_data_bytes == VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES && is_public) |
| 377 | return default_free_free_list_index; |
| 378 | } |
| 379 | |
| 380 | pool_get_aligned (bm->buffer_free_list_pool, f, CLIB_CACHE_LINE_BYTES); |
| 381 | |
| 382 | memset (f, 0, sizeof (f[0])); |
| 383 | f->index = f - bm->buffer_free_list_pool; |
| 384 | f->n_data_bytes = vlib_buffer_round_size (n_data_bytes); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 385 | f->min_n_buffers_each_physmem_alloc = VLIB_FRAME_SIZE; |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 386 | f->name = clib_mem_is_heap_object (name) ? name : format (0, "%s", name); |
| 387 | |
| 388 | /* Setup free buffer template. */ |
| 389 | f->buffer_init_template.free_list_index = f->index; |
Damjan Marion | c47ed03 | 2017-01-25 14:18:03 +0100 | [diff] [blame] | 390 | f->buffer_init_template.n_add_refs = 0; |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 391 | |
| 392 | if (is_public) |
| 393 | { |
| 394 | uword *p = hash_get (bm->free_list_by_size, f->n_data_bytes); |
| 395 | if (!p) |
| 396 | hash_set (bm->free_list_by_size, f->n_data_bytes, f->index); |
| 397 | } |
| 398 | |
| 399 | for (i = 1; i < vec_len (vlib_mains); i++) |
| 400 | { |
| 401 | vlib_buffer_main_t *wbm = vlib_mains[i]->buffer_main; |
| 402 | vlib_buffer_free_list_t *wf; |
| 403 | pool_get_aligned (wbm->buffer_free_list_pool, |
| 404 | wf, CLIB_CACHE_LINE_BYTES); |
| 405 | ASSERT (f - bm->buffer_free_list_pool == |
| 406 | wf - wbm->buffer_free_list_pool); |
| 407 | wf[0] = f[0]; |
Damjan Marion | bd69a5f | 2017-02-05 23:44:42 +0100 | [diff] [blame] | 408 | wf->buffers = 0; |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 409 | wf->n_alloc = 0; |
| 410 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 411 | |
| 412 | return f->index; |
| 413 | } |
| 414 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 415 | u32 |
| 416 | vlib_buffer_create_free_list (vlib_main_t * vm, u32 n_data_bytes, |
| 417 | char *fmt, ...) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 418 | { |
| 419 | va_list va; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 420 | u8 *name; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 421 | |
| 422 | va_start (va, fmt); |
| 423 | name = va_format (0, fmt, &va); |
| 424 | va_end (va); |
| 425 | |
| 426 | return vlib_buffer_create_free_list_helper (vm, n_data_bytes, |
| 427 | /* is_public */ 0, |
| 428 | /* is_default */ 0, |
| 429 | name); |
| 430 | } |
| 431 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 432 | u32 |
| 433 | vlib_buffer_get_or_create_free_list (vlib_main_t * vm, u32 n_data_bytes, |
| 434 | char *fmt, ...) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 435 | { |
| 436 | u32 i = vlib_buffer_get_free_list_with_size (vm, n_data_bytes); |
| 437 | |
| 438 | if (i == ~0) |
| 439 | { |
| 440 | va_list va; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 441 | u8 *name; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 442 | |
| 443 | va_start (va, fmt); |
| 444 | name = va_format (0, fmt, &va); |
| 445 | va_end (va); |
| 446 | |
| 447 | i = vlib_buffer_create_free_list_helper (vm, n_data_bytes, |
| 448 | /* is_public */ 1, |
| 449 | /* is_default */ 0, |
| 450 | name); |
| 451 | } |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 452 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 453 | return i; |
| 454 | } |
| 455 | |
| 456 | static void |
| 457 | del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f) |
| 458 | { |
| 459 | u32 i; |
| 460 | |
| 461 | for (i = 0; i < vec_len (f->buffer_memory_allocated); i++) |
| 462 | vm->os_physmem_free (f->buffer_memory_allocated[i]); |
| 463 | vec_free (f->name); |
| 464 | vec_free (f->buffer_memory_allocated); |
Damjan Marion | bd69a5f | 2017-02-05 23:44:42 +0100 | [diff] [blame] | 465 | vec_free (f->buffers); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 466 | } |
| 467 | |
| 468 | /* Add buffer free list. */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 469 | void |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 470 | vlib_buffer_delete_free_list_internal (vlib_main_t * vm, u32 free_list_index) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 471 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 472 | vlib_buffer_main_t *bm = vm->buffer_main; |
| 473 | vlib_buffer_free_list_t *f; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 474 | u32 merge_index; |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 475 | int i; |
| 476 | |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 477 | ASSERT (vlib_get_thread_index () == 0); |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 478 | |
| 479 | f = vlib_buffer_get_free_list (vm, free_list_index); |
| 480 | |
Damjan Marion | bd69a5f | 2017-02-05 23:44:42 +0100 | [diff] [blame] | 481 | ASSERT (vec_len (f->buffers) == f->n_alloc); |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 482 | merge_index = vlib_buffer_get_free_list_with_size (vm, f->n_data_bytes); |
| 483 | if (merge_index != ~0 && merge_index != free_list_index) |
| 484 | { |
Damjan Marion | 8a6a3b2 | 2017-01-17 14:12:42 +0100 | [diff] [blame] | 485 | vlib_buffer_merge_free_lists (pool_elt_at_index |
| 486 | (bm->buffer_free_list_pool, merge_index), |
| 487 | f); |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 488 | } |
| 489 | |
| 490 | del_free_list (vm, f); |
| 491 | |
| 492 | /* Poison it. */ |
| 493 | memset (f, 0xab, sizeof (f[0])); |
| 494 | |
| 495 | pool_put (bm->buffer_free_list_pool, f); |
| 496 | |
| 497 | for (i = 1; i < vec_len (vlib_mains); i++) |
| 498 | { |
| 499 | bm = vlib_mains[i]->buffer_main; |
| 500 | f = vlib_buffer_get_free_list (vlib_mains[i], free_list_index);; |
| 501 | memset (f, 0xab, sizeof (f[0])); |
| 502 | pool_put (bm->buffer_free_list_pool, f); |
| 503 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 504 | } |
| 505 | |
| 506 | /* Make sure free list has at least given number of free buffers. */ |
| 507 | static uword |
| 508 | fill_free_list (vlib_main_t * vm, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 509 | vlib_buffer_free_list_t * fl, uword min_free_buffers) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 510 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 511 | vlib_buffer_t *buffers, *b; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 512 | int n, n_bytes, i; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 513 | u32 *bi; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 514 | u32 n_remaining, n_alloc, n_this_chunk; |
| 515 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 516 | /* Already have enough free buffers on free list? */ |
Damjan Marion | bd69a5f | 2017-02-05 23:44:42 +0100 | [diff] [blame] | 517 | n = min_free_buffers - vec_len (fl->buffers); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 518 | if (n <= 0) |
| 519 | return min_free_buffers; |
| 520 | |
| 521 | /* Always allocate round number of buffers. */ |
Damjan Marion | bd69a5f | 2017-02-05 23:44:42 +0100 | [diff] [blame] | 522 | n = round_pow2 (n, CLIB_CACHE_LINE_BYTES / sizeof (u32)); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 523 | |
| 524 | /* Always allocate new buffers in reasonably large sized chunks. */ |
| 525 | n = clib_max (n, fl->min_n_buffers_each_physmem_alloc); |
| 526 | |
| 527 | n_remaining = n; |
| 528 | n_alloc = 0; |
| 529 | while (n_remaining > 0) |
| 530 | { |
| 531 | n_this_chunk = clib_min (n_remaining, 16); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 532 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 533 | n_bytes = n_this_chunk * (sizeof (b[0]) + fl->n_data_bytes); |
| 534 | |
| 535 | /* drb: removed power-of-2 ASSERT */ |
| 536 | buffers = vm->os_physmem_alloc_aligned (&vm->physmem_main, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 537 | n_bytes, |
| 538 | sizeof (vlib_buffer_t)); |
| 539 | if (!buffers) |
| 540 | return n_alloc; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 541 | |
| 542 | /* Record chunk as being allocated so we can free it later. */ |
| 543 | vec_add1 (fl->buffer_memory_allocated, buffers); |
| 544 | |
| 545 | fl->n_alloc += n_this_chunk; |
| 546 | n_alloc += n_this_chunk; |
| 547 | n_remaining -= n_this_chunk; |
| 548 | |
| 549 | b = buffers; |
Damjan Marion | bd69a5f | 2017-02-05 23:44:42 +0100 | [diff] [blame] | 550 | vec_add2_aligned (fl->buffers, bi, n_this_chunk, CLIB_CACHE_LINE_BYTES); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 551 | for (i = 0; i < n_this_chunk; i++) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 552 | { |
| 553 | bi[i] = vlib_get_buffer_index (vm, b); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 554 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 555 | if (CLIB_DEBUG > 0) |
| 556 | vlib_buffer_set_known_state (vm, bi[i], VLIB_BUFFER_KNOWN_FREE); |
| 557 | b = vlib_buffer_next_contiguous (b, fl->n_data_bytes); |
| 558 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 559 | |
| 560 | memset (buffers, 0, n_bytes); |
| 561 | |
| 562 | /* Initialize all new buffers. */ |
| 563 | b = buffers; |
| 564 | for (i = 0; i < n_this_chunk; i++) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 565 | { |
| 566 | vlib_buffer_init_for_free_list (b, fl); |
| 567 | b = vlib_buffer_next_contiguous (b, fl->n_data_bytes); |
| 568 | } |
| 569 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 570 | if (fl->buffer_init_function) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 571 | fl->buffer_init_function (vm, fl, bi, n_this_chunk); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 572 | } |
| 573 | return n_alloc; |
| 574 | } |
| 575 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 576 | static u32 |
| 577 | alloc_from_free_list (vlib_main_t * vm, |
| 578 | vlib_buffer_free_list_t * free_list, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 579 | u32 * alloc_buffers, u32 n_alloc_buffers) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 580 | { |
Damjan Marion | bd69a5f | 2017-02-05 23:44:42 +0100 | [diff] [blame] | 581 | u32 *dst, *src; |
| 582 | uword len; |
| 583 | uword n_filled; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 584 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 585 | dst = alloc_buffers; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 586 | |
Damjan Marion | 374e2c5 | 2017-03-09 20:38:15 +0100 | [diff] [blame] | 587 | /* wait with buffer memory allocation as long as possible |
| 588 | in case external buffer manager takes over */ |
| 589 | if (PREDICT_FALSE (vm->os_physmem_alloc_aligned == 0)) |
| 590 | unix_physmem_init (vm, 0 /* fail_if_physical_memory_not_present */ ); |
| 591 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 592 | n_filled = fill_free_list (vm, free_list, n_alloc_buffers); |
| 593 | if (n_filled == 0) |
| 594 | return 0; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 595 | |
Damjan Marion | bd69a5f | 2017-02-05 23:44:42 +0100 | [diff] [blame] | 596 | len = vec_len (free_list->buffers); |
| 597 | ASSERT (len >= n_alloc_buffers); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 598 | |
Damjan Marion | bd69a5f | 2017-02-05 23:44:42 +0100 | [diff] [blame] | 599 | src = free_list->buffers + len - n_alloc_buffers; |
| 600 | clib_memcpy (dst, src, n_alloc_buffers * sizeof (u32)); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 601 | |
Damjan Marion | bd69a5f | 2017-02-05 23:44:42 +0100 | [diff] [blame] | 602 | _vec_len (free_list->buffers) -= n_alloc_buffers; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 603 | |
| 604 | /* Verify that buffers are known free. */ |
| 605 | vlib_buffer_validate_alloc_free (vm, alloc_buffers, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 606 | n_alloc_buffers, VLIB_BUFFER_KNOWN_FREE); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 607 | |
| 608 | return n_alloc_buffers; |
| 609 | } |
| 610 | |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 611 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 612 | /* Allocate a given number of buffers into given array. |
| 613 | Returns number actually allocated which will be either zero or |
| 614 | number requested. */ |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 615 | static u32 |
| 616 | vlib_buffer_alloc_internal (vlib_main_t * vm, u32 * buffers, u32 n_buffers) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 617 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 618 | vlib_buffer_main_t *bm = vm->buffer_main; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 619 | |
| 620 | return alloc_from_free_list |
| 621 | (vm, |
| 622 | pool_elt_at_index (bm->buffer_free_list_pool, |
| 623 | VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX), |
| 624 | buffers, n_buffers); |
| 625 | } |
| 626 | |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 627 | static u32 |
| 628 | vlib_buffer_alloc_from_free_list_internal (vlib_main_t * vm, |
| 629 | u32 * buffers, |
| 630 | u32 n_buffers, u32 free_list_index) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 631 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 632 | vlib_buffer_main_t *bm = vm->buffer_main; |
| 633 | vlib_buffer_free_list_t *f; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 634 | f = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index); |
| 635 | return alloc_from_free_list (vm, f, buffers, n_buffers); |
| 636 | } |
| 637 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 638 | void * |
| 639 | vlib_set_buffer_free_callback (vlib_main_t * vm, void *fp) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 640 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 641 | vlib_buffer_main_t *bm = vm->buffer_main; |
| 642 | void *rv = bm->buffer_free_callback; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 643 | |
| 644 | bm->buffer_free_callback = fp; |
| 645 | return rv; |
| 646 | } |
| 647 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 648 | static_always_inline void |
| 649 | vlib_buffer_free_inline (vlib_main_t * vm, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 650 | u32 * buffers, u32 n_buffers, u32 follow_buffer_next) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 651 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 652 | vlib_buffer_main_t *bm = vm->buffer_main; |
| 653 | vlib_buffer_free_list_t *fl; |
Damjan Marion | c47ed03 | 2017-01-25 14:18:03 +0100 | [diff] [blame] | 654 | u32 fi; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 655 | int i; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 656 | u32 (*cb) (vlib_main_t * vm, u32 * buffers, u32 n_buffers, |
| 657 | u32 follow_buffer_next); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 658 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 659 | cb = bm->buffer_free_callback; |
| 660 | |
| 661 | if (PREDICT_FALSE (cb != 0)) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 662 | n_buffers = (*cb) (vm, buffers, n_buffers, follow_buffer_next); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 663 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 664 | if (!n_buffers) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 665 | return; |
| 666 | |
Damjan Marion | c47ed03 | 2017-01-25 14:18:03 +0100 | [diff] [blame] | 667 | for (i = 0; i < n_buffers; i++) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 668 | { |
Damjan Marion | c47ed03 | 2017-01-25 14:18:03 +0100 | [diff] [blame] | 669 | vlib_buffer_t *b; |
| 670 | u32 bi = buffers[i]; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 671 | |
Damjan Marion | c47ed03 | 2017-01-25 14:18:03 +0100 | [diff] [blame] | 672 | b = vlib_get_buffer (vm, bi); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 673 | |
Damjan Marion | c47ed03 | 2017-01-25 14:18:03 +0100 | [diff] [blame] | 674 | fl = vlib_buffer_get_buffer_free_list (vm, b, &fi); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 675 | |
Damjan Marion | c47ed03 | 2017-01-25 14:18:03 +0100 | [diff] [blame] | 676 | /* The only current use of this callback: multicast recycle */ |
| 677 | if (PREDICT_FALSE (fl->buffers_added_to_freelist_function != 0)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 678 | { |
Damjan Marion | c47ed03 | 2017-01-25 14:18:03 +0100 | [diff] [blame] | 679 | int j; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 680 | |
Damjan Marion | c47ed03 | 2017-01-25 14:18:03 +0100 | [diff] [blame] | 681 | vlib_buffer_add_to_free_list |
| 682 | (vm, fl, buffers[i], (b->flags & VLIB_BUFFER_RECYCLE) == 0); |
| 683 | |
| 684 | for (j = 0; j < vec_len (bm->announce_list); j++) |
| 685 | { |
| 686 | if (fl == bm->announce_list[j]) |
| 687 | goto already_announced; |
| 688 | } |
| 689 | vec_add1 (bm->announce_list, fl); |
| 690 | already_announced: |
| 691 | ; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 692 | } |
| 693 | else |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 694 | { |
Damjan Marion | c47ed03 | 2017-01-25 14:18:03 +0100 | [diff] [blame] | 695 | if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_RECYCLE) == 0)) |
| 696 | { |
| 697 | u32 flags, next; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 698 | |
Damjan Marion | c47ed03 | 2017-01-25 14:18:03 +0100 | [diff] [blame] | 699 | do |
| 700 | { |
| 701 | vlib_buffer_t *nb = vlib_get_buffer (vm, bi); |
| 702 | flags = nb->flags; |
| 703 | next = nb->next_buffer; |
| 704 | if (nb->n_add_refs) |
| 705 | nb->n_add_refs--; |
| 706 | else |
| 707 | { |
| 708 | vlib_buffer_validate_alloc_free (vm, &bi, 1, |
| 709 | VLIB_BUFFER_KNOWN_ALLOCATED); |
| 710 | vlib_buffer_add_to_free_list (vm, fl, bi, 1); |
| 711 | } |
| 712 | bi = next; |
| 713 | } |
| 714 | while (follow_buffer_next |
| 715 | && (flags & VLIB_BUFFER_NEXT_PRESENT)); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 716 | |
Damjan Marion | c47ed03 | 2017-01-25 14:18:03 +0100 | [diff] [blame] | 717 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 718 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 719 | } |
Damjan Marion | c47ed03 | 2017-01-25 14:18:03 +0100 | [diff] [blame] | 720 | if (vec_len (bm->announce_list)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 721 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 722 | vlib_buffer_free_list_t *fl; |
Damjan Marion | c47ed03 | 2017-01-25 14:18:03 +0100 | [diff] [blame] | 723 | for (i = 0; i < vec_len (bm->announce_list); i++) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 724 | { |
Damjan Marion | c47ed03 | 2017-01-25 14:18:03 +0100 | [diff] [blame] | 725 | fl = bm->announce_list[i]; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 726 | fl->buffers_added_to_freelist_function (vm, fl); |
| 727 | } |
Damjan Marion | c47ed03 | 2017-01-25 14:18:03 +0100 | [diff] [blame] | 728 | _vec_len (bm->announce_list) = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 729 | } |
| 730 | } |
| 731 | |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 732 | static void |
| 733 | vlib_buffer_free_internal (vlib_main_t * vm, u32 * buffers, u32 n_buffers) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 734 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 735 | vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */ |
| 736 | 1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 737 | } |
| 738 | |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 739 | static void |
| 740 | vlib_buffer_free_no_next_internal (vlib_main_t * vm, u32 * buffers, |
| 741 | u32 n_buffers) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 742 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 743 | vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */ |
| 744 | 0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 745 | } |
| 746 | |
| 747 | /* Copy template packet data into buffers as they are allocated. */ |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 748 | static void __attribute__ ((unused)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 749 | vlib_packet_template_buffer_init (vlib_main_t * vm, |
| 750 | vlib_buffer_free_list_t * fl, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 751 | u32 * buffers, u32 n_buffers) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 752 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 753 | vlib_packet_template_t *t = |
| 754 | uword_to_pointer (fl->buffer_init_function_opaque, |
| 755 | vlib_packet_template_t *); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 756 | uword i; |
| 757 | |
| 758 | for (i = 0; i < n_buffers; i++) |
| 759 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 760 | vlib_buffer_t *b = vlib_get_buffer (vm, buffers[i]); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 761 | ASSERT (b->current_length == vec_len (t->packet_data)); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 762 | clib_memcpy (vlib_buffer_get_current (b), t->packet_data, |
| 763 | b->current_length); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 764 | } |
| 765 | } |
| 766 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 767 | void |
| 768 | vlib_packet_template_init (vlib_main_t * vm, |
| 769 | vlib_packet_template_t * t, |
| 770 | void *packet_data, |
| 771 | uword n_packet_data_bytes, |
| 772 | uword min_n_buffers_each_physmem_alloc, |
| 773 | char *fmt, ...) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 774 | { |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 775 | vlib_buffer_main_t *bm = vm->buffer_main; |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 776 | va_list va; |
| 777 | __attribute__ ((unused)) u8 *name; |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 778 | vlib_buffer_free_list_t *fl; |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 779 | |
| 780 | va_start (va, fmt); |
| 781 | name = va_format (0, fmt, &va); |
| 782 | va_end (va); |
| 783 | |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 784 | if (bm->cb.vlib_packet_template_init_cb) |
| 785 | bm->cb.vlib_packet_template_init_cb (vm, (void *) t, packet_data, |
| 786 | n_packet_data_bytes, |
| 787 | min_n_buffers_each_physmem_alloc, |
| 788 | name); |
| 789 | |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 790 | vlib_worker_thread_barrier_sync (vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 791 | |
| 792 | memset (t, 0, sizeof (t[0])); |
| 793 | |
| 794 | vec_add (t->packet_data, packet_data, n_packet_data_bytes); |
| 795 | t->min_n_buffers_each_physmem_alloc = min_n_buffers_each_physmem_alloc; |
| 796 | |
| 797 | t->free_list_index = vlib_buffer_create_free_list_helper |
| 798 | (vm, n_packet_data_bytes, |
| 799 | /* is_public */ 1, |
| 800 | /* is_default */ 0, |
| 801 | name); |
| 802 | |
| 803 | ASSERT (t->free_list_index != 0); |
| 804 | fl = vlib_buffer_get_free_list (vm, t->free_list_index); |
| 805 | fl->min_n_buffers_each_physmem_alloc = t->min_n_buffers_each_physmem_alloc; |
| 806 | |
| 807 | fl->buffer_init_function = vlib_packet_template_buffer_init; |
| 808 | fl->buffer_init_function_opaque = pointer_to_uword (t); |
| 809 | |
| 810 | fl->buffer_init_template.current_data = 0; |
| 811 | fl->buffer_init_template.current_length = n_packet_data_bytes; |
| 812 | fl->buffer_init_template.flags = 0; |
Damjan Marion | c47ed03 | 2017-01-25 14:18:03 +0100 | [diff] [blame] | 813 | fl->buffer_init_template.n_add_refs = 0; |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 814 | vlib_worker_thread_barrier_release (vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 815 | } |
| 816 | |
| 817 | void * |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 818 | vlib_packet_template_get_packet (vlib_main_t * vm, |
| 819 | vlib_packet_template_t * t, u32 * bi_result) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 820 | { |
| 821 | u32 bi; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 822 | vlib_buffer_t *b; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 823 | |
| 824 | if (vlib_buffer_alloc (vm, &bi, 1) != 1) |
| 825 | return 0; |
| 826 | |
| 827 | *bi_result = bi; |
| 828 | |
| 829 | b = vlib_get_buffer (vm, bi); |
Damjan Marion | f1213b8 | 2016-03-13 02:22:06 +0100 | [diff] [blame] | 830 | clib_memcpy (vlib_buffer_get_current (b), |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 831 | t->packet_data, vec_len (t->packet_data)); |
| 832 | b->current_length = vec_len (t->packet_data); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 833 | |
| 834 | return b->data; |
| 835 | } |
| 836 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 837 | void |
| 838 | vlib_packet_template_get_packet_helper (vlib_main_t * vm, |
| 839 | vlib_packet_template_t * t) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 840 | { |
| 841 | word n = t->min_n_buffers_each_physmem_alloc; |
| 842 | word l = vec_len (t->packet_data); |
| 843 | word n_alloc; |
| 844 | |
| 845 | ASSERT (l > 0); |
| 846 | ASSERT (vec_len (t->free_buffers) == 0); |
| 847 | |
| 848 | vec_validate (t->free_buffers, n - 1); |
| 849 | n_alloc = vlib_buffer_alloc_from_free_list (vm, t->free_buffers, |
| 850 | n, t->free_list_index); |
| 851 | _vec_len (t->free_buffers) = n_alloc; |
| 852 | } |
| 853 | |
| 854 | /* Append given data to end of buffer, possibly allocating new buffers. */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 855 | u32 |
| 856 | vlib_buffer_add_data (vlib_main_t * vm, |
| 857 | u32 free_list_index, |
| 858 | u32 buffer_index, void *data, u32 n_data_bytes) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 859 | { |
| 860 | u32 n_buffer_bytes, n_left, n_left_this_buffer, bi; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 861 | vlib_buffer_t *b; |
| 862 | void *d; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 863 | |
| 864 | bi = buffer_index; |
| 865 | if (bi == 0 |
| 866 | && 1 != vlib_buffer_alloc_from_free_list (vm, &bi, 1, free_list_index)) |
| 867 | goto out_of_buffers; |
| 868 | |
| 869 | d = data; |
| 870 | n_left = n_data_bytes; |
| 871 | n_buffer_bytes = vlib_buffer_free_list_buffer_size (vm, free_list_index); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 872 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 873 | b = vlib_get_buffer (vm, bi); |
| 874 | b->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID; |
| 875 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 876 | /* Get to the end of the chain before we try to append data... */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 877 | while (b->flags & VLIB_BUFFER_NEXT_PRESENT) |
| 878 | b = vlib_get_buffer (vm, b->next_buffer); |
| 879 | |
| 880 | while (1) |
| 881 | { |
| 882 | u32 n; |
| 883 | |
| 884 | ASSERT (n_buffer_bytes >= b->current_length); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 885 | n_left_this_buffer = |
| 886 | n_buffer_bytes - (b->current_data + b->current_length); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 887 | n = clib_min (n_left_this_buffer, n_left); |
Damjan Marion | f1213b8 | 2016-03-13 02:22:06 +0100 | [diff] [blame] | 888 | clib_memcpy (vlib_buffer_get_current (b) + b->current_length, d, n); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 889 | b->current_length += n; |
| 890 | n_left -= n; |
| 891 | if (n_left == 0) |
| 892 | break; |
| 893 | |
| 894 | d += n; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 895 | if (1 != |
| 896 | vlib_buffer_alloc_from_free_list (vm, &b->next_buffer, 1, |
| 897 | free_list_index)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 898 | goto out_of_buffers; |
| 899 | |
| 900 | b->flags |= VLIB_BUFFER_NEXT_PRESENT; |
| 901 | |
| 902 | b = vlib_get_buffer (vm, b->next_buffer); |
| 903 | } |
| 904 | |
| 905 | return bi; |
| 906 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 907 | out_of_buffers: |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 908 | clib_error ("out of buffers"); |
| 909 | return bi; |
| 910 | } |
| 911 | |
Pierre Pfister | 328e99b | 2016-02-12 13:18:42 +0000 | [diff] [blame] | 912 | u16 |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 913 | vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm, |
| 914 | u32 free_list_index, |
| 915 | vlib_buffer_t * first, |
| 916 | vlib_buffer_t ** last, |
| 917 | void *data, u16 data_len) |
| 918 | { |
Pierre Pfister | 328e99b | 2016-02-12 13:18:42 +0000 | [diff] [blame] | 919 | vlib_buffer_t *l = *last; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 920 | u32 n_buffer_bytes = |
| 921 | vlib_buffer_free_list_buffer_size (vm, free_list_index); |
Pierre Pfister | 328e99b | 2016-02-12 13:18:42 +0000 | [diff] [blame] | 922 | u16 copied = 0; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 923 | ASSERT (n_buffer_bytes >= l->current_length + l->current_data); |
| 924 | while (data_len) |
| 925 | { |
| 926 | u16 max = n_buffer_bytes - l->current_length - l->current_data; |
| 927 | if (max == 0) |
| 928 | { |
| 929 | if (1 != |
| 930 | vlib_buffer_alloc_from_free_list (vm, &l->next_buffer, 1, |
| 931 | free_list_index)) |
| 932 | return copied; |
| 933 | *last = l = vlib_buffer_chain_buffer (vm, first, l, l->next_buffer); |
| 934 | max = n_buffer_bytes - l->current_length - l->current_data; |
| 935 | } |
Pierre Pfister | 328e99b | 2016-02-12 13:18:42 +0000 | [diff] [blame] | 936 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 937 | u16 len = (data_len > max) ? max : data_len; |
| 938 | clib_memcpy (vlib_buffer_get_current (l) + l->current_length, |
| 939 | data + copied, len); |
| 940 | vlib_buffer_chain_increase_length (first, l, len); |
| 941 | data_len -= len; |
| 942 | copied += len; |
| 943 | } |
Pierre Pfister | 328e99b | 2016-02-12 13:18:42 +0000 | [diff] [blame] | 944 | return copied; |
| 945 | } |
| 946 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 947 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 948 | static u8 * |
| 949 | format_vlib_buffer_free_list (u8 * s, va_list * va) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 950 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 951 | vlib_buffer_free_list_t *f = va_arg (*va, vlib_buffer_free_list_t *); |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 952 | u32 threadnum = va_arg (*va, u32); |
| 953 | uword bytes_alloc, bytes_free, n_free, size; |
| 954 | |
| 955 | if (!f) |
| 956 | return format (s, "%=7s%=30s%=12s%=12s%=12s%=12s%=12s%=12s", |
| 957 | "Thread", "Name", "Index", "Size", "Alloc", "Free", |
| 958 | "#Alloc", "#Free"); |
| 959 | |
| 960 | size = sizeof (vlib_buffer_t) + f->n_data_bytes; |
Damjan Marion | bd69a5f | 2017-02-05 23:44:42 +0100 | [diff] [blame] | 961 | n_free = vec_len (f->buffers); |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 962 | bytes_alloc = size * f->n_alloc; |
| 963 | bytes_free = size * n_free; |
| 964 | |
| 965 | s = format (s, "%7d%30s%12d%12d%=12U%=12U%=12d%=12d", threadnum, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 966 | f->name, f->index, f->n_data_bytes, |
| 967 | format_memory_size, bytes_alloc, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 968 | format_memory_size, bytes_free, f->n_alloc, n_free); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 969 | |
| 970 | return s; |
| 971 | } |
| 972 | |
| 973 | static clib_error_t * |
| 974 | show_buffers (vlib_main_t * vm, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 975 | unformat_input_t * input, vlib_cli_command_t * cmd) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 976 | { |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 977 | vlib_buffer_main_t *bm; |
| 978 | vlib_buffer_free_list_t *f; |
| 979 | vlib_main_t *curr_vm; |
| 980 | u32 vm_index = 0; |
| 981 | |
| 982 | vlib_cli_output (vm, "%U", format_vlib_buffer_free_list, 0, 0); |
| 983 | |
| 984 | do |
| 985 | { |
Dave Barach | 80f54e2 | 2017-03-08 19:08:56 -0500 | [diff] [blame] | 986 | curr_vm = vlib_mains[vm_index]; |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 987 | bm = curr_vm->buffer_main; |
| 988 | |
| 989 | /* *INDENT-OFF* */ |
| 990 | pool_foreach (f, bm->buffer_free_list_pool, ({ |
| 991 | vlib_cli_output (vm, "%U", format_vlib_buffer_free_list, f, vm_index); |
| 992 | })); |
| 993 | /* *INDENT-ON* */ |
| 994 | |
| 995 | vm_index++; |
| 996 | } |
| 997 | while (vm_index < vec_len (vlib_mains)); |
| 998 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 999 | return 0; |
| 1000 | } |
| 1001 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1002 | /* *INDENT-OFF* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1003 | VLIB_CLI_COMMAND (show_buffers_command, static) = { |
| 1004 | .path = "show buffers", |
| 1005 | .short_help = "Show packet buffer allocation", |
| 1006 | .function = show_buffers, |
| 1007 | }; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1008 | /* *INDENT-ON* */ |
| 1009 | |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 1010 | void |
| 1011 | vlib_buffer_cb_init (struct vlib_main_t *vm) |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 1012 | { |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 1013 | vlib_buffer_main_t *bm = vm->buffer_main; |
| 1014 | bm->cb.vlib_buffer_alloc_cb = &vlib_buffer_alloc_internal; |
| 1015 | bm->cb.vlib_buffer_alloc_from_free_list_cb = |
| 1016 | &vlib_buffer_alloc_from_free_list_internal; |
| 1017 | bm->cb.vlib_buffer_free_cb = &vlib_buffer_free_internal; |
| 1018 | bm->cb.vlib_buffer_free_no_next_cb = &vlib_buffer_free_no_next_internal; |
| 1019 | bm->cb.vlib_buffer_delete_free_list_cb = |
| 1020 | &vlib_buffer_delete_free_list_internal; |
| 1021 | bm->extern_buffer_mgmt = 0; |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 1022 | } |
| 1023 | |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 1024 | int |
| 1025 | vlib_buffer_cb_register (struct vlib_main_t *vm, vlib_buffer_callbacks_t * cb) |
| 1026 | { |
| 1027 | vlib_buffer_main_t *bm = vm->buffer_main; |
| 1028 | if (bm->extern_buffer_mgmt) |
| 1029 | return -1; |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 1030 | |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 1031 | #define _(x) bm->cb.x = cb->x |
| 1032 | _(vlib_buffer_alloc_cb); |
| 1033 | _(vlib_buffer_alloc_from_free_list_cb); |
| 1034 | _(vlib_buffer_free_cb); |
| 1035 | _(vlib_buffer_free_no_next_cb); |
| 1036 | _(vlib_buffer_delete_free_list_cb); |
| 1037 | #undef _ |
| 1038 | bm->extern_buffer_mgmt = 1; |
| 1039 | return 0; |
| 1040 | } |
Damjan Marion | 1cd8f3c | 2016-11-24 02:07:32 +0100 | [diff] [blame] | 1041 | |
Chris Luke | d4024f5 | 2016-09-06 09:32:36 -0400 | [diff] [blame] | 1042 | /** @endcond */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1043 | /* |
| 1044 | * fd.io coding-style-patch-verification: ON |
| 1045 | * |
| 1046 | * Local Variables: |
| 1047 | * eval: (c-set-style "gnu") |
| 1048 | * End: |
| 1049 | */ |