blob: 71fcfee2190bcb4ef96b8da56bde6ff3b1178ed0 [file] [log] [blame]
Ed Warnickecb9cada2015-12-08 15:45:58 -07001/*
2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15/*
16 * buffer.c: allocate/free network buffers.
17 *
18 * Copyright (c) 2008 Eliot Dresselhaus
19 *
20 * Permission is hereby granted, free of charge, to any person obtaining
21 * a copy of this software and associated documentation files (the
22 * "Software"), to deal in the Software without restriction, including
23 * without limitation the rights to use, copy, modify, merge, publish,
24 * distribute, sublicense, and/or sell copies of the Software, and to
25 * permit persons to whom the Software is furnished to do so, subject to
26 * the following conditions:
27 *
28 * The above copyright notice and this permission notice shall be
29 * included in all copies or substantial portions of the Software.
30 *
31 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38 */
39
Chris Luked4024f52016-09-06 09:32:36 -040040/**
Chris Luked4024f52016-09-06 09:32:36 -040041 * @file
42 *
43 * Allocate/free network buffers.
44 */
45
Damjan Marion4b661402024-02-29 16:14:27 +010046#include <vppinfra/bitmap.h>
47#include <vppinfra/unix.h>
Ed Warnickecb9cada2015-12-08 15:45:58 -070048#include <vlib/vlib.h>
Damjan Marion374e2c52017-03-09 20:38:15 +010049#include <vlib/unix/unix.h>
Damjan Marion8973b072022-03-01 15:51:18 +010050#include <vlib/stats/stats.h>
Ed Warnickecb9cada2015-12-08 15:45:58 -070051
Damjan Marion910d3692019-01-21 11:48:34 +010052#define VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA 16384
53#define VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA_UNPRIV 8192
Damjan Marion901d16c2018-10-23 19:50:20 +020054
Damjan Marion910d3692019-01-21 11:48:34 +010055#ifdef CLIB_HAVE_VEC128
56/* Assumptions by vlib_buffer_free_inline: */
57STATIC_ASSERT_FITS_IN (vlib_buffer_t, flags, 16);
58STATIC_ASSERT_FITS_IN (vlib_buffer_t, ref_count, 16);
59STATIC_ASSERT_FITS_IN (vlib_buffer_t, buffer_pool_index, 16);
60#endif
Damjan Marion04a7f052017-07-10 15:06:17 +020061
Damjan Marion910d3692019-01-21 11:48:34 +010062u16 __vlib_buffer_external_hdr_size = 0;
Damjan Marion567e61d2018-10-24 17:08:26 +020063
Dave Barach9b8ffd92016-07-08 08:13:45 -040064uword
65vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm,
66 vlib_buffer_t * b_first)
Ed Warnickecb9cada2015-12-08 15:45:58 -070067{
Dave Barach9b8ffd92016-07-08 08:13:45 -040068 vlib_buffer_t *b = b_first;
Ed Warnickecb9cada2015-12-08 15:45:58 -070069 uword l_first = b_first->current_length;
70 uword l = 0;
71 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
72 {
73 b = vlib_get_buffer (vm, b->next_buffer);
74 l += b->current_length;
75 }
76 b_first->total_length_not_including_first_buffer = l;
77 b_first->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
78 return l + l_first;
79}
80
Dave Barach9b8ffd92016-07-08 08:13:45 -040081u8 *
Benoît Ganne43543172019-10-21 15:13:54 +020082format_vlib_buffer_no_chain (u8 * s, va_list * args)
Ed Warnickecb9cada2015-12-08 15:45:58 -070083{
Dave Barach9b8ffd92016-07-08 08:13:45 -040084 vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
Christophe Fontained3c008d2017-10-02 18:10:54 +020085 u32 indent = format_get_indent (s);
Damjan Mariondac03522018-02-01 15:30:13 +010086 u8 *a = 0;
Damjan Marion1cd8f3c2016-11-24 02:07:32 +010087
Damjan Mariondac03522018-02-01 15:30:13 +010088#define _(bit, name, v) \
89 if (v && (b->flags & VLIB_BUFFER_##name)) \
90 a = format (a, "%s ", v);
91 foreach_vlib_buffer_flag
92#undef _
Damjan Marion36eb7c22019-01-18 20:45:30 +010093 s = format (s, "current data %d, length %d, buffer-pool %d, "
Damjan Marion910d3692019-01-21 11:48:34 +010094 "ref-count %u", b->current_data, b->current_length,
95 b->buffer_pool_index, b->ref_count);
Damjan Marion1cd8f3c2016-11-24 02:07:32 +010096
Mohsin Kazmi30d46e72022-01-11 11:34:56 +000097 if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
Damjan Marion1cd8f3c2016-11-24 02:07:32 +010098 s = format (s, ", totlen-nifb %d",
99 b->total_length_not_including_first_buffer);
100
101 if (b->flags & VLIB_BUFFER_IS_TRACED)
Dave Baracha638c182019-06-21 18:24:07 -0400102 s = format (s, ", trace handle 0x%x", b->trace_handle);
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100103
Damjan Mariondac03522018-02-01 15:30:13 +0100104 if (a)
105 s = format (s, "\n%U%v", format_white_space, indent, a);
106 vec_free (a);
107
Benoît Ganne43543172019-10-21 15:13:54 +0200108 return s;
109}
110
111u8 *
112format_vlib_buffer (u8 * s, va_list * args)
113{
114 vlib_main_t *vm = vlib_get_main ();
115 vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
116 u32 indent = format_get_indent (s);
117
118 s = format (s, "%U", format_vlib_buffer_no_chain, b);
119
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100120 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
121 {
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100122 u32 next_buffer = b->next_buffer;
123 b = vlib_get_buffer (vm, next_buffer);
124
Damjan Marionc47ed032017-01-25 14:18:03 +0100125 s =
Damjan Marion910d3692019-01-21 11:48:34 +0100126 format (s, "\n%Unext-buffer 0x%x, segment length %d, ref-count %u",
Damjan Marionc47ed032017-01-25 14:18:03 +0100127 format_white_space, indent, next_buffer, b->current_length,
Damjan Marion910d3692019-01-21 11:48:34 +0100128 b->ref_count);
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100129 }
130
Ed Warnickecb9cada2015-12-08 15:45:58 -0700131 return s;
132}
133
Dave Barach9b8ffd92016-07-08 08:13:45 -0400134u8 *
135format_vlib_buffer_and_data (u8 * s, va_list * args)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700136{
Dave Barach9b8ffd92016-07-08 08:13:45 -0400137 vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
138
Ed Warnickecb9cada2015-12-08 15:45:58 -0700139 s = format (s, "%U, %U",
140 format_vlib_buffer, b,
141 format_hex_bytes, vlib_buffer_get_current (b), 64);
142
143 return s;
144}
145
Dave Barach9b8ffd92016-07-08 08:13:45 -0400146static u8 *
147format_vlib_buffer_known_state (u8 * s, va_list * args)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700148{
149 vlib_buffer_known_state_t state = va_arg (*args, vlib_buffer_known_state_t);
Dave Barach9b8ffd92016-07-08 08:13:45 -0400150 char *t;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700151
152 switch (state)
153 {
154 case VLIB_BUFFER_UNKNOWN:
155 t = "unknown";
156 break;
157
158 case VLIB_BUFFER_KNOWN_ALLOCATED:
159 t = "known-allocated";
160 break;
161
162 case VLIB_BUFFER_KNOWN_FREE:
163 t = "known-free";
164 break;
165
166 default:
167 t = "invalid";
168 break;
169 }
170
171 return format (s, "%s", t);
172}
173
Dave Barach9b8ffd92016-07-08 08:13:45 -0400174u8 *
175format_vlib_buffer_contents (u8 * s, va_list * va)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700176{
Dave Barach9b8ffd92016-07-08 08:13:45 -0400177 vlib_main_t *vm = va_arg (*va, vlib_main_t *);
178 vlib_buffer_t *b = va_arg (*va, vlib_buffer_t *);
179
Ed Warnickecb9cada2015-12-08 15:45:58 -0700180 while (1)
181 {
Dave Barach9b8ffd92016-07-08 08:13:45 -0400182 vec_add (s, vlib_buffer_get_current (b), b->current_length);
183 if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))
Ed Warnickecb9cada2015-12-08 15:45:58 -0700184 break;
185 b = vlib_get_buffer (vm, b->next_buffer);
186 }
187
188 return s;
189}
190
191static u8 *
192vlib_validate_buffer_helper (vlib_main_t * vm,
193 u32 bi,
Dave Barach9b8ffd92016-07-08 08:13:45 -0400194 uword follow_buffer_next, uword ** unique_hash)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700195{
Damjan Marion910d3692019-01-21 11:48:34 +0100196 vlib_buffer_main_t *bm = vm->buffer_main;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400197 vlib_buffer_t *b = vlib_get_buffer (vm, bi);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700198
Damjan Marion910d3692019-01-21 11:48:34 +0100199 if (vec_len (bm->buffer_pools) <= b->buffer_pool_index)
200 return format (0, "unknown buffer pool 0x%x", b->buffer_pool_index);
201
Dave Barach9b8ffd92016-07-08 08:13:45 -0400202 if ((signed) b->current_data < (signed) -VLIB_BUFFER_PRE_DATA_SIZE)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700203 return format (0, "current data %d before pre-data", b->current_data);
Damjan Marion878c6092017-01-04 13:19:27 +0100204
Damjan Marion8934a042019-02-09 23:29:26 +0100205 if (b->current_data + b->current_length >
206 vlib_buffer_get_default_data_size (vm))
Damjan Marion910d3692019-01-21 11:48:34 +0100207 return format (0, "%d-%d beyond end of buffer %d", b->current_data,
Damjan Marion8934a042019-02-09 23:29:26 +0100208 b->current_length, vlib_buffer_get_default_data_size (vm));
Ed Warnickecb9cada2015-12-08 15:45:58 -0700209
Dave Barach9b8ffd92016-07-08 08:13:45 -0400210 if (follow_buffer_next && (b->flags & VLIB_BUFFER_NEXT_PRESENT))
Ed Warnickecb9cada2015-12-08 15:45:58 -0700211 {
212 vlib_buffer_known_state_t k;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400213 u8 *msg, *result;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700214
Damjan Mariond50e3472019-01-20 00:03:56 +0100215 k = vlib_buffer_is_known (vm, b->next_buffer);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700216 if (k != VLIB_BUFFER_KNOWN_ALLOCATED)
217 return format (0, "next 0x%x: %U",
Dave Barach9b8ffd92016-07-08 08:13:45 -0400218 b->next_buffer, format_vlib_buffer_known_state, k);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700219
220 if (unique_hash)
221 {
222 if (hash_get (*unique_hash, b->next_buffer))
223 return format (0, "duplicate buffer 0x%x", b->next_buffer);
224
225 hash_set1 (*unique_hash, b->next_buffer);
226 }
227
228 msg = vlib_validate_buffer (vm, b->next_buffer, follow_buffer_next);
229 if (msg)
230 {
231 result = format (0, "next 0x%x: %v", b->next_buffer, msg);
232 vec_free (msg);
233 return result;
234 }
235 }
236
237 return 0;
238}
239
240u8 *
241vlib_validate_buffer (vlib_main_t * vm, u32 bi, uword follow_buffer_next)
Dave Barach9b8ffd92016-07-08 08:13:45 -0400242{
243 return vlib_validate_buffer_helper (vm, bi, follow_buffer_next,
244 /* unique_hash */ 0);
245}
Ed Warnickecb9cada2015-12-08 15:45:58 -0700246
247u8 *
248vlib_validate_buffers (vlib_main_t * vm,
249 u32 * buffers,
250 uword next_buffer_stride,
251 uword n_buffers,
252 vlib_buffer_known_state_t known_state,
253 uword follow_buffer_next)
254{
Dave Barach9b8ffd92016-07-08 08:13:45 -0400255 uword i, *hash;
256 u32 bi, *b = buffers;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700257 vlib_buffer_known_state_t k;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400258 u8 *msg = 0, *result = 0;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700259
260 hash = hash_create (0, 0);
261 for (i = 0; i < n_buffers; i++)
262 {
263 bi = b[0];
264 b += next_buffer_stride;
265
266 /* Buffer is not unique. */
267 if (hash_get (hash, bi))
268 {
269 msg = format (0, "not unique");
270 goto done;
271 }
272
Damjan Mariond50e3472019-01-20 00:03:56 +0100273 k = vlib_buffer_is_known (vm, bi);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700274 if (k != known_state)
275 {
276 msg = format (0, "is %U; expected %U",
277 format_vlib_buffer_known_state, k,
278 format_vlib_buffer_known_state, known_state);
279 goto done;
280 }
281
282 msg = vlib_validate_buffer_helper (vm, bi, follow_buffer_next, &hash);
283 if (msg)
284 goto done;
285
286 hash_set1 (hash, bi);
287 }
288
Dave Barach9b8ffd92016-07-08 08:13:45 -0400289done:
Ed Warnickecb9cada2015-12-08 15:45:58 -0700290 if (msg)
291 {
292 result = format (0, "0x%x: %v", bi, msg);
293 vec_free (msg);
294 }
295 hash_free (hash);
296 return result;
297}
298
Paul Vinciguerra8feeaff2019-03-27 11:25:48 -0700299/* When debugging validate that given buffers are either known allocated
Ed Warnickecb9cada2015-12-08 15:45:58 -0700300 or known free. */
Damjan Marionc8a26c62017-11-24 20:15:23 +0100301void
Ed Warnickecb9cada2015-12-08 15:45:58 -0700302vlib_buffer_validate_alloc_free (vlib_main_t * vm,
303 u32 * buffers,
304 uword n_buffers,
305 vlib_buffer_known_state_t expected_state)
306{
Damjan Marion910d3692019-01-21 11:48:34 +0100307 vlib_buffer_main_t *bm = vm->buffer_main;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400308 u32 *b;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700309 uword i, bi, is_free;
310
311 if (CLIB_DEBUG == 0)
312 return;
313
Ed Warnickecb9cada2015-12-08 15:45:58 -0700314 is_free = expected_state == VLIB_BUFFER_KNOWN_ALLOCATED;
315 b = buffers;
316 for (i = 0; i < n_buffers; i++)
317 {
318 vlib_buffer_known_state_t known;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400319
Ed Warnickecb9cada2015-12-08 15:45:58 -0700320 bi = b[0];
321 b += 1;
Damjan Mariond50e3472019-01-20 00:03:56 +0100322 known = vlib_buffer_is_known (vm, bi);
Damjan Marion910d3692019-01-21 11:48:34 +0100323
324 if (known == VLIB_BUFFER_UNKNOWN &&
325 expected_state == VLIB_BUFFER_KNOWN_FREE)
326 known = VLIB_BUFFER_KNOWN_FREE;
327
Ed Warnickecb9cada2015-12-08 15:45:58 -0700328 if (known != expected_state)
329 {
Klement Sekerae82c73a2019-01-30 18:29:28 +0100330 clib_panic ("%s %U buffer 0x%x", is_free ? "freeing" : "allocating",
331 format_vlib_buffer_known_state, known, bi);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700332 }
333
Damjan Marion910d3692019-01-21 11:48:34 +0100334 clib_spinlock_lock (&bm->buffer_known_hash_lockp);
335 hash_set (bm->buffer_known_hash, bi, is_free ? VLIB_BUFFER_KNOWN_FREE :
336 VLIB_BUFFER_KNOWN_ALLOCATED);
337 clib_spinlock_unlock (&bm->buffer_known_hash_lockp);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700338 }
339}
Ed Warnickecb9cada2015-12-08 15:45:58 -0700340
Dave Barach9b8ffd92016-07-08 08:13:45 -0400341void
342vlib_packet_template_init (vlib_main_t * vm,
343 vlib_packet_template_t * t,
344 void *packet_data,
345 uword n_packet_data_bytes,
Damjan Mariond1274cb2018-03-13 21:32:17 +0100346 uword min_n_buffers_each_alloc, char *fmt, ...)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700347{
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100348 va_list va;
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100349
350 va_start (va, fmt);
Damjan Marion671e60e2018-12-30 18:09:59 +0100351 t->name = va_format (0, fmt, &va);
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100352 va_end (va);
353
354 vlib_worker_thread_barrier_sync (vm);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700355
Dave Barachb7b92992018-10-17 10:38:51 -0400356 clib_memset (t, 0, sizeof (t[0]));
Ed Warnickecb9cada2015-12-08 15:45:58 -0700357
358 vec_add (t->packet_data, packet_data, n_packet_data_bytes);
Damjan Mariond1274cb2018-03-13 21:32:17 +0100359 t->min_n_buffers_each_alloc = min_n_buffers_each_alloc;
Damjan Marion878c6092017-01-04 13:19:27 +0100360 vlib_worker_thread_barrier_release (vm);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700361}
362
363void *
Dave Barach9b8ffd92016-07-08 08:13:45 -0400364vlib_packet_template_get_packet (vlib_main_t * vm,
365 vlib_packet_template_t * t, u32 * bi_result)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700366{
367 u32 bi;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400368 vlib_buffer_t *b;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700369
370 if (vlib_buffer_alloc (vm, &bi, 1) != 1)
371 return 0;
372
373 *bi_result = bi;
374
375 b = vlib_get_buffer (vm, bi);
Dave Barach178cf492018-11-13 16:34:13 -0500376 clib_memcpy_fast (vlib_buffer_get_current (b),
377 t->packet_data, vec_len (t->packet_data));
Dave Barach9b8ffd92016-07-08 08:13:45 -0400378 b->current_length = vec_len (t->packet_data);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700379
380 return b->data;
381}
382
Ed Warnickecb9cada2015-12-08 15:45:58 -0700383/* Append given data to end of buffer, possibly allocating new buffers. */
Dave Barach3a63fc52019-01-07 09:15:47 -0500384int
Damjan Marionab9b7ec2019-01-18 20:24:44 +0100385vlib_buffer_add_data (vlib_main_t * vm, u32 * buffer_index, void *data,
386 u32 n_data_bytes)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700387{
388 u32 n_buffer_bytes, n_left, n_left_this_buffer, bi;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400389 vlib_buffer_t *b;
390 void *d;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700391
Dave Barach3a63fc52019-01-07 09:15:47 -0500392 bi = *buffer_index;
Damjan Marionab9b7ec2019-01-18 20:24:44 +0100393 if (bi == ~0 && 1 != vlib_buffer_alloc (vm, &bi, 1))
Ed Warnickecb9cada2015-12-08 15:45:58 -0700394 goto out_of_buffers;
395
396 d = data;
397 n_left = n_data_bytes;
Damjan Marion8934a042019-02-09 23:29:26 +0100398 n_buffer_bytes = vlib_buffer_get_default_data_size (vm);
Dave Barach9b8ffd92016-07-08 08:13:45 -0400399
Ed Warnickecb9cada2015-12-08 15:45:58 -0700400 b = vlib_get_buffer (vm, bi);
401 b->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
402
Dave Barach9b8ffd92016-07-08 08:13:45 -0400403 /* Get to the end of the chain before we try to append data... */
Ed Warnickecb9cada2015-12-08 15:45:58 -0700404 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
405 b = vlib_get_buffer (vm, b->next_buffer);
406
407 while (1)
408 {
409 u32 n;
410
411 ASSERT (n_buffer_bytes >= b->current_length);
Dave Barach9b8ffd92016-07-08 08:13:45 -0400412 n_left_this_buffer =
413 n_buffer_bytes - (b->current_data + b->current_length);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700414 n = clib_min (n_left_this_buffer, n_left);
Dave Barach178cf492018-11-13 16:34:13 -0500415 clib_memcpy_fast (vlib_buffer_get_current (b) + b->current_length, d,
416 n);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700417 b->current_length += n;
418 n_left -= n;
419 if (n_left == 0)
420 break;
421
422 d += n;
Damjan Marionab9b7ec2019-01-18 20:24:44 +0100423 if (1 != vlib_buffer_alloc (vm, &b->next_buffer, 1))
Ed Warnickecb9cada2015-12-08 15:45:58 -0700424 goto out_of_buffers;
425
426 b->flags |= VLIB_BUFFER_NEXT_PRESENT;
427
428 b = vlib_get_buffer (vm, b->next_buffer);
429 }
430
Dave Barach3a63fc52019-01-07 09:15:47 -0500431 *buffer_index = bi;
432 return 0;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700433
Dave Barach9b8ffd92016-07-08 08:13:45 -0400434out_of_buffers:
Dave Barach3a63fc52019-01-07 09:15:47 -0500435 clib_warning ("out of buffers");
436 return 1;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700437}
438
Pierre Pfister328e99b2016-02-12 13:18:42 +0000439u16
Dave Barach9b8ffd92016-07-08 08:13:45 -0400440vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm,
Dave Barach9b8ffd92016-07-08 08:13:45 -0400441 vlib_buffer_t * first,
Damjan Mariondac03522018-02-01 15:30:13 +0100442 vlib_buffer_t ** last, void *data,
443 u16 data_len)
Dave Barach9b8ffd92016-07-08 08:13:45 -0400444{
Pierre Pfister328e99b2016-02-12 13:18:42 +0000445 vlib_buffer_t *l = *last;
Damjan Marion8934a042019-02-09 23:29:26 +0100446 u32 n_buffer_bytes = vlib_buffer_get_default_data_size (vm);
Pierre Pfister328e99b2016-02-12 13:18:42 +0000447 u16 copied = 0;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400448 ASSERT (n_buffer_bytes >= l->current_length + l->current_data);
449 while (data_len)
450 {
451 u16 max = n_buffer_bytes - l->current_length - l->current_data;
452 if (max == 0)
453 {
Damjan Marion910d3692019-01-21 11:48:34 +0100454 if (1 != vlib_buffer_alloc_from_pool (vm, &l->next_buffer, 1,
455 first->buffer_pool_index))
Dave Barach9b8ffd92016-07-08 08:13:45 -0400456 return copied;
Eyal Barib688fb12018-11-12 16:13:49 +0200457 *last = l = vlib_buffer_chain_buffer (vm, l, l->next_buffer);
Dave Barach9b8ffd92016-07-08 08:13:45 -0400458 max = n_buffer_bytes - l->current_length - l->current_data;
459 }
Pierre Pfister328e99b2016-02-12 13:18:42 +0000460
Dave Barach9b8ffd92016-07-08 08:13:45 -0400461 u16 len = (data_len > max) ? max : data_len;
Dave Barach178cf492018-11-13 16:34:13 -0500462 clib_memcpy_fast (vlib_buffer_get_current (l) + l->current_length,
463 data + copied, len);
Dave Barach9b8ffd92016-07-08 08:13:45 -0400464 vlib_buffer_chain_increase_length (first, l, len);
465 data_len -= len;
466 copied += len;
467 }
Pierre Pfister328e99b2016-02-12 13:18:42 +0000468 return copied;
469}
470
Benoît Gannec16fe462020-06-15 12:24:36 +0200471static uword
472vlib_buffer_alloc_size (uword ext_hdr_size, uword data_size)
473{
Damjan Marion0be1b762020-09-04 12:34:58 +0200474 uword alloc_size = ext_hdr_size + sizeof (vlib_buffer_t) + data_size;
Damjan Marion038dad72024-01-19 21:19:57 +0100475 alloc_size = round_pow2 (alloc_size, VLIB_BUFFER_ALIGN);
Damjan Marion0be1b762020-09-04 12:34:58 +0200476
Vratko Polak04fd51c2024-02-06 12:45:59 +0100477 /* in case when we have even number of 'cachelines', we add one more for
478 * better cache occupancy */
479 alloc_size |= VLIB_BUFFER_ALIGN;
480
Damjan Marion0be1b762020-09-04 12:34:58 +0200481 return alloc_size;
Benoît Gannec16fe462020-06-15 12:24:36 +0200482}
483
Damjan Marionb592d1b2019-02-28 23:16:11 +0100484u8
Damjan Marion65dc34b2023-10-06 10:59:32 +0200485vlib_buffer_pool_create (vlib_main_t *vm, u32 data_size, u32 physmem_map_index,
486 char *fmt, ...)
Damjan Marion04a7f052017-07-10 15:06:17 +0200487{
Damjan Mariond50e3472019-01-20 00:03:56 +0100488 vlib_buffer_main_t *bm = vm->buffer_main;
Damjan Marion910d3692019-01-21 11:48:34 +0100489 vlib_buffer_pool_t *bp;
Damjan Marion68b4da62018-09-30 18:26:20 +0200490 vlib_physmem_map_t *m = vlib_physmem_get_map (vm, physmem_map_index);
491 uword start = pointer_to_uword (m->base);
Damjan Marion8e8d3c82018-10-23 22:54:40 +0200492 uword size = (uword) m->n_pages << m->log2_page_size;
Damjan Marion65dc34b2023-10-06 10:59:32 +0200493 uword page_mask = ~pow2_mask (m->log2_page_size);
494 u8 *p;
495 u32 alloc_size;
496 va_list va;
Damjan Marion910d3692019-01-21 11:48:34 +0100497
Damjan Marionb592d1b2019-02-28 23:16:11 +0100498 if (vec_len (bm->buffer_pools) >= 255)
499 return ~0;
Damjan Marion910d3692019-01-21 11:48:34 +0100500
Tianyu Li70b1cbd2021-06-23 07:35:03 +0000501 vec_add2_aligned (bm->buffer_pools, bp, 1, CLIB_CACHE_LINE_BYTES);
Damjan Marion04a7f052017-07-10 15:06:17 +0200502
503 if (bm->buffer_mem_size == 0)
504 {
505 bm->buffer_mem_start = start;
506 bm->buffer_mem_size = size;
507 }
508 else if (start < bm->buffer_mem_start)
509 {
510 bm->buffer_mem_size += bm->buffer_mem_start - start;
511 bm->buffer_mem_start = start;
512 if (size > bm->buffer_mem_size)
513 bm->buffer_mem_size = size;
514 }
515 else if (start > bm->buffer_mem_start)
516 {
517 uword new_size = start - bm->buffer_mem_start + size;
518 if (new_size > bm->buffer_mem_size)
519 bm->buffer_mem_size = new_size;
520 }
521
522 if ((u64) bm->buffer_mem_size >
523 ((u64) 1 << (32 + CLIB_LOG2_CACHE_LINE_BYTES)))
524 {
525 clib_panic ("buffer memory size out of range!");
526 }
Damjan Marion149ba772017-10-12 13:09:26 +0200527
Damjan Marion910d3692019-01-21 11:48:34 +0100528 bp->start = start;
529 bp->size = size;
530 bp->index = bp - bm->buffer_pools;
531 bp->buffer_template.buffer_pool_index = bp->index;
532 bp->buffer_template.ref_count = 1;
533 bp->physmem_map_index = physmem_map_index;
Damjan Marion910d3692019-01-21 11:48:34 +0100534 bp->data_size = data_size;
535 bp->numa_node = m->numa_node;
Damjan Marion65dc34b2023-10-06 10:59:32 +0200536 bp->log2_page_size = m->log2_page_size;
537
538 va_start (va, fmt);
539 bp->name = va_format (0, fmt, &va);
540 va_end (va);
Damjan Mariond1274cb2018-03-13 21:32:17 +0100541
Damjan Marion6ffb7c62021-03-26 13:06:13 +0100542 vec_validate_aligned (bp->threads, vlib_get_n_threads () - 1,
Damjan Marion910d3692019-01-21 11:48:34 +0100543 CLIB_CACHE_LINE_BYTES);
544
Benoît Gannec16fe462020-06-15 12:24:36 +0200545 alloc_size = vlib_buffer_alloc_size (bm->ext_hdr_size, data_size);
Damjan Marion65dc34b2023-10-06 10:59:32 +0200546 bp->alloc_size = alloc_size;
Damjan Marion910d3692019-01-21 11:48:34 +0100547
548 /* preallocate buffer indices memory */
Damjan Marion65dc34b2023-10-06 10:59:32 +0200549 bp->buffers = clib_mem_alloc_aligned (
550 round_pow2 ((size / alloc_size) * sizeof (u32), CLIB_CACHE_LINE_BYTES),
551 CLIB_CACHE_LINE_BYTES);
Damjan Marion910d3692019-01-21 11:48:34 +0100552
553 clib_spinlock_init (&bp->lock);
554
Damjan Marion65dc34b2023-10-06 10:59:32 +0200555 p = m->base;
Damjan Marion910d3692019-01-21 11:48:34 +0100556
Damjan Marion65dc34b2023-10-06 10:59:32 +0200557 /* start with naturally aligned address */
558 p += alloc_size - (uword) p % alloc_size;
Damjan Marion910d3692019-01-21 11:48:34 +0100559
Damjan Marion65dc34b2023-10-06 10:59:32 +0200560 /*
561 * Waste 1 buffer (maximum) so that 0 is never a valid buffer index.
562 * Allows various places to ASSERT (bi != 0). Much easier
563 * than debugging downstream crashes in successor nodes.
564 */
565 if (p == m->base)
566 p += alloc_size;
Dave Barachc74b43c2020-04-09 17:24:07 -0400567
Damjan Marion65dc34b2023-10-06 10:59:32 +0200568 for (; p < (u8 *) m->base + size - alloc_size; p += alloc_size)
569 {
570 vlib_buffer_t *b;
571 u32 bi;
Damjan Marion910d3692019-01-21 11:48:34 +0100572
Damjan Marion65dc34b2023-10-06 10:59:32 +0200573 /* skip if buffer spans across page boundary */
574 if (((uword) p & page_mask) != ((uword) (p + alloc_size) & page_mask))
575 continue;
Damjan Marion910d3692019-01-21 11:48:34 +0100576
Damjan Marion65dc34b2023-10-06 10:59:32 +0200577 b = (vlib_buffer_t *) (p + bm->ext_hdr_size);
Damjan Marionbf236632023-10-13 09:59:00 +0000578 b->template = bp->buffer_template;
Damjan Marion65dc34b2023-10-06 10:59:32 +0200579 bi = vlib_get_buffer_index (vm, b);
580 bp->buffers[bp->n_avail++] = bi;
581 vlib_get_buffer (vm, bi);
582 }
Damjan Marionb6e8b1a2019-03-12 18:14:15 +0100583
Damjan Marion65dc34b2023-10-06 10:59:32 +0200584 bp->n_buffers = bp->n_avail;
Damjan Marion910d3692019-01-21 11:48:34 +0100585
Damjan Marionb592d1b2019-02-28 23:16:11 +0100586 return bp->index;
Damjan Marion04a7f052017-07-10 15:06:17 +0200587}
Ed Warnickecb9cada2015-12-08 15:45:58 -0700588
Dave Barach9b8ffd92016-07-08 08:13:45 -0400589static u8 *
Damjan Marion910d3692019-01-21 11:48:34 +0100590format_vlib_buffer_pool (u8 * s, va_list * va)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700591{
Damjan Marion910d3692019-01-21 11:48:34 +0100592 vlib_main_t *vm = va_arg (*va, vlib_main_t *);
593 vlib_buffer_pool_t *bp = va_arg (*va, vlib_buffer_pool_t *);
594 vlib_buffer_pool_thread_t *bpt;
595 u32 cached = 0;
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100596
Damjan Marion910d3692019-01-21 11:48:34 +0100597 if (!bp)
598 return format (s, "%-20s%=6s%=6s%=6s%=11s%=6s%=8s%=8s%=8s",
599 "Pool Name", "Index", "NUMA", "Size", "Data Size",
600 "Total", "Avail", "Cached", "Used");
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100601
Damjan Marion910d3692019-01-21 11:48:34 +0100602 vec_foreach (bpt, bp->threads)
Damjan Marionb6e8b1a2019-03-12 18:14:15 +0100603 cached += bpt->n_cached;
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100604
Damjan Marion55a1f282023-10-24 23:59:48 +0200605 s = format (s, "%-20v%=6d%=6d%=6u%=11u%=6u%=8u%=8u%=8u", bp->name, bp->index,
606 bp->numa_node,
607 bp->data_size + sizeof (vlib_buffer_t) +
608 vm->buffer_main->ext_hdr_size,
Damjan Marionb6e8b1a2019-03-12 18:14:15 +0100609 bp->data_size, bp->n_buffers, bp->n_avail, cached,
610 bp->n_buffers - bp->n_avail - cached);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700611
612 return s;
613}
614
Benoît Gannee09a2332021-03-09 15:37:49 +0100615u8 *
616format_vlib_buffer_pool_all (u8 *s, va_list *va)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700617{
Benoît Gannee09a2332021-03-09 15:37:49 +0100618 vlib_main_t *vm = va_arg (*va, vlib_main_t *);
Damjan Marion910d3692019-01-21 11:48:34 +0100619 vlib_buffer_main_t *bm = vm->buffer_main;
620 vlib_buffer_pool_t *bp;
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100621
Benoît Gannee09a2332021-03-09 15:37:49 +0100622 s = format (s, "%U", format_vlib_buffer_pool, vm, 0);
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100623
Damjan Marion910d3692019-01-21 11:48:34 +0100624 vec_foreach (bp, bm->buffer_pools)
Benoît Gannee09a2332021-03-09 15:37:49 +0100625 s = format (s, "\n%U", format_vlib_buffer_pool, vm, bp);
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100626
Benoît Gannee09a2332021-03-09 15:37:49 +0100627 return s;
628}
629
630static clib_error_t *
631show_buffers (vlib_main_t *vm, unformat_input_t *input,
632 vlib_cli_command_t *cmd)
633{
634 vlib_cli_output (vm, "%U", format_vlib_buffer_pool_all, vm);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700635 return 0;
636}
637
638VLIB_CLI_COMMAND (show_buffers_command, static) = {
639 .path = "show buffers",
640 .short_help = "Show packet buffer allocation",
641 .function = show_buffers,
642};
Dave Barach9b8ffd92016-07-08 08:13:45 -0400643
Damjan Marion49d66f12017-07-20 18:10:35 +0200644clib_error_t *
Damjan Marion321bd102022-06-01 00:45:18 +0200645vlib_buffer_num_workers_change (vlib_main_t *vm)
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100646{
Damjan Marion910d3692019-01-21 11:48:34 +0100647 vlib_buffer_main_t *bm = vm->buffer_main;
648 vlib_buffer_pool_t *bp;
649
Damjan Marion910d3692019-01-21 11:48:34 +0100650 vec_foreach (bp, bm->buffer_pools)
Damjan Marion321bd102022-06-01 00:45:18 +0200651 vec_validate_aligned (bp->threads, vlib_get_n_threads () - 1,
652 CLIB_CACHE_LINE_BYTES);
Damjan Marion910d3692019-01-21 11:48:34 +0100653
654 return 0;
655}
656
Damjan Marion321bd102022-06-01 00:45:18 +0200657VLIB_NUM_WORKERS_CHANGE_FN (vlib_buffer_num_workers_change);
Damjan Marion910d3692019-01-21 11:48:34 +0100658
659static clib_error_t *
Nathan Skrzypczak61559022020-11-23 16:25:21 +0100660vlib_buffer_main_init_numa_alloc (struct vlib_main_t *vm, u32 numa_node,
661 u32 * physmem_map_index,
662 clib_mem_page_sz_t log2_page_size,
663 u8 unpriv)
Damjan Marion910d3692019-01-21 11:48:34 +0100664{
665 vlib_buffer_main_t *bm = vm->buffer_main;
Lukas Stocknerb4808de2024-06-18 14:52:52 +0200666 u32 default_buffers_per_numa = bm->default_buffers_per_numa;
Lukas Stockner63ff7a62024-06-04 16:14:55 +0200667 u32 buffers_per_numa = bm->buffers_per_numa[numa_node];
Damjan Marion49d66f12017-07-20 18:10:35 +0200668 clib_error_t *error;
Nathan Skrzypczak61559022020-11-23 16:25:21 +0100669 u32 buffer_size;
Damjan Marion910d3692019-01-21 11:48:34 +0100670 uword n_pages, pagesize;
Nathan Skrzypczak61559022020-11-23 16:25:21 +0100671 u8 *name = 0;
Damjan Marion567e61d2018-10-24 17:08:26 +0200672
Nathan Skrzypczak61559022020-11-23 16:25:21 +0100673 ASSERT (log2_page_size != CLIB_MEM_PAGE_SZ_UNKNOWN);
Damjan Marion49d66f12017-07-20 18:10:35 +0200674
Nathan Skrzypczak61559022020-11-23 16:25:21 +0100675 pagesize = clib_mem_page_bytes (log2_page_size);
676 buffer_size = vlib_buffer_alloc_size (bm->ext_hdr_size,
677 vlib_buffer_get_default_data_size
678 (vm));
Benoît Gannec16fe462020-06-15 12:24:36 +0200679 if (buffer_size > pagesize)
Nathan Skrzypczak61559022020-11-23 16:25:21 +0100680 return clib_error_return (0, "buffer size (%llu) is greater than page "
681 "size (%llu)", buffer_size, pagesize);
Benoît Gannec16fe462020-06-15 12:24:36 +0200682
Lukas Stocknerb4808de2024-06-18 14:52:52 +0200683 if (default_buffers_per_numa == 0)
684 default_buffers_per_numa = unpriv ?
685 VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA_UNPRIV :
686 VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA;
Lukas Stockner63ff7a62024-06-04 16:14:55 +0200687
Lukas Stocknerb4808de2024-06-18 14:52:52 +0200688 if (buffers_per_numa == ~0)
689 buffers_per_numa = default_buffers_per_numa;
Nathan Skrzypczak61559022020-11-23 16:25:21 +0100690
691 name = format (0, "buffers-numa-%d%c", numa_node, 0);
Damjan Marion910d3692019-01-21 11:48:34 +0100692 n_pages = (buffers_per_numa - 1) / (pagesize / buffer_size) + 1;
693 error = vlib_physmem_shared_map_create (vm, (char *) name,
694 n_pages * pagesize,
695 min_log2 (pagesize), numa_node,
Nathan Skrzypczak61559022020-11-23 16:25:21 +0100696 physmem_map_index);
697 vec_free (name);
698 return error;
699}
Damjan Marion567e61d2018-10-24 17:08:26 +0200700
Nathan Skrzypczak61559022020-11-23 16:25:21 +0100701static clib_error_t *
702vlib_buffer_main_init_numa_node (struct vlib_main_t *vm, u32 numa_node,
703 u8 * index)
704{
705 vlib_buffer_main_t *bm = vm->buffer_main;
706 u32 physmem_map_index;
707 clib_error_t *error;
Nathan Skrzypczak61559022020-11-23 16:25:21 +0100708
709 if (bm->log2_page_size == CLIB_MEM_PAGE_SZ_UNKNOWN)
Damjan Marion567e61d2018-10-24 17:08:26 +0200710 {
Nathan Skrzypczak61559022020-11-23 16:25:21 +0100711 error = vlib_buffer_main_init_numa_alloc (vm, numa_node,
712 &physmem_map_index,
713 CLIB_MEM_PAGE_SZ_DEFAULT_HUGE,
714 0 /* unpriv */ );
715 if (!error)
716 goto buffer_pool_create;
717
718 /* If alloc failed, retry without hugepages */
719 vlib_log_warn (bm->log_default,
720 "numa[%u] falling back to non-hugepage backed "
721 "buffer pool (%U)", numa_node, format_clib_error, error);
Damjan Marion567e61d2018-10-24 17:08:26 +0200722 clib_error_free (error);
Nathan Skrzypczak61559022020-11-23 16:25:21 +0100723
724 error = vlib_buffer_main_init_numa_alloc (vm, numa_node,
725 &physmem_map_index,
726 CLIB_MEM_PAGE_SZ_DEFAULT,
727 1 /* unpriv */ );
Damjan Marion567e61d2018-10-24 17:08:26 +0200728 }
Nathan Skrzypczak61559022020-11-23 16:25:21 +0100729 else
730 error = vlib_buffer_main_init_numa_alloc (vm, numa_node,
731 &physmem_map_index,
732 bm->log2_page_size,
733 0 /* unpriv */ );
Damjan Marion567e61d2018-10-24 17:08:26 +0200734 if (error)
Nathan Skrzypczak61559022020-11-23 16:25:21 +0100735 return error;
Damjan Marion49d66f12017-07-20 18:10:35 +0200736
Nathan Skrzypczak61559022020-11-23 16:25:21 +0100737buffer_pool_create:
Damjan Marion65dc34b2023-10-06 10:59:32 +0200738 *index =
739 vlib_buffer_pool_create (vm, vlib_buffer_get_default_data_size (vm),
740 physmem_map_index, "default-numa-%d", numa_node);
Damjan Marionb592d1b2019-02-28 23:16:11 +0100741
742 if (*index == (u8) ~ 0)
Nathan Skrzypczak61559022020-11-23 16:25:21 +0100743 error = clib_error_return (0, "maximum number of buffer pools reached");
Nathan Skrzypczak61559022020-11-23 16:25:21 +0100744
745
Guanghua Zhang785daf42019-08-18 08:35:04 +0800746 return error;
Damjan Marion5de3fec2019-02-06 14:22:32 +0100747}
748
749void
750vlib_buffer_main_alloc (vlib_main_t * vm)
751{
752 vlib_buffer_main_t *bm;
753
754 if (vm->buffer_main)
755 return;
756
757 vm->buffer_main = bm = clib_mem_alloc (sizeof (bm[0]));
758 clib_memset (vm->buffer_main, 0, sizeof (bm[0]));
759 bm->default_data_size = VLIB_BUFFER_DEFAULT_DATA_SIZE;
Damjan Marion910d3692019-01-21 11:48:34 +0100760}
761
Filip Tehlard2bbdef2019-02-22 05:05:53 -0800762static u32
763buffer_get_cached (vlib_buffer_pool_t * bp)
764{
765 u32 cached = 0;
766 vlib_buffer_pool_thread_t *bpt;
767
Filip Tehlarb806d0e2019-02-28 04:36:10 -0800768 clib_spinlock_lock (&bp->lock);
769
Filip Tehlard2bbdef2019-02-22 05:05:53 -0800770 vec_foreach (bpt, bp->threads)
Damjan Marionb6e8b1a2019-03-12 18:14:15 +0100771 cached += bpt->n_cached;
Filip Tehlard2bbdef2019-02-22 05:05:53 -0800772
Filip Tehlarb806d0e2019-02-28 04:36:10 -0800773 clib_spinlock_unlock (&bp->lock);
774
Filip Tehlard2bbdef2019-02-22 05:05:53 -0800775 return cached;
776}
777
778static vlib_buffer_pool_t *
Ole Troan849cbe22019-02-26 13:10:56 +0100779buffer_get_by_index (vlib_buffer_main_t * bm, u32 index)
Filip Tehlard2bbdef2019-02-22 05:05:53 -0800780{
781 vlib_buffer_pool_t *bp;
Ole Troan849cbe22019-02-26 13:10:56 +0100782 if (!bm->buffer_pools || vec_len (bm->buffer_pools) < index)
783 return 0;
784 bp = vec_elt_at_index (bm->buffer_pools, index);
Steven Luong95155102019-03-05 15:29:22 -0800785
Ole Troan849cbe22019-02-26 13:10:56 +0100786 return bp;
Filip Tehlard2bbdef2019-02-22 05:05:53 -0800787}
788
789static void
Damjan Marion8973b072022-03-01 15:51:18 +0100790buffer_gauges_collect_used_fn (vlib_stats_collector_data_t *d)
Filip Tehlard2bbdef2019-02-22 05:05:53 -0800791{
792 vlib_main_t *vm = vlib_get_main ();
Damjan Marion8973b072022-03-01 15:51:18 +0100793 vlib_buffer_pool_t *bp =
794 buffer_get_by_index (vm->buffer_main, d->private_data);
Filip Tehlard2bbdef2019-02-22 05:05:53 -0800795 if (!bp)
796 return;
797
Damjan Marion8973b072022-03-01 15:51:18 +0100798 d->entry->value = bp->n_buffers - bp->n_avail - buffer_get_cached (bp);
Filip Tehlard2bbdef2019-02-22 05:05:53 -0800799}
800
801static void
Damjan Marion8973b072022-03-01 15:51:18 +0100802buffer_gauges_collect_available_fn (vlib_stats_collector_data_t *d)
Filip Tehlard2bbdef2019-02-22 05:05:53 -0800803{
804 vlib_main_t *vm = vlib_get_main ();
Damjan Marion8973b072022-03-01 15:51:18 +0100805 vlib_buffer_pool_t *bp =
806 buffer_get_by_index (vm->buffer_main, d->private_data);
Filip Tehlard2bbdef2019-02-22 05:05:53 -0800807 if (!bp)
808 return;
809
Damjan Marion8973b072022-03-01 15:51:18 +0100810 d->entry->value = bp->n_avail;
Filip Tehlard2bbdef2019-02-22 05:05:53 -0800811}
812
813static void
Damjan Marion8973b072022-03-01 15:51:18 +0100814buffer_gauges_collect_cached_fn (vlib_stats_collector_data_t *d)
Filip Tehlard2bbdef2019-02-22 05:05:53 -0800815{
816 vlib_main_t *vm = vlib_get_main ();
Damjan Marion8973b072022-03-01 15:51:18 +0100817 vlib_buffer_pool_t *bp =
818 buffer_get_by_index (vm->buffer_main, d->private_data);
Filip Tehlard2bbdef2019-02-22 05:05:53 -0800819 if (!bp)
820 return;
821
Damjan Marion8973b072022-03-01 15:51:18 +0100822 d->entry->value = buffer_get_cached (bp);
Filip Tehlard2bbdef2019-02-22 05:05:53 -0800823}
824
Damjan Marion910d3692019-01-21 11:48:34 +0100825clib_error_t *
Filip Tehlard2bbdef2019-02-22 05:05:53 -0800826vlib_buffer_main_init (struct vlib_main_t * vm)
Damjan Marion910d3692019-01-21 11:48:34 +0100827{
828 vlib_buffer_main_t *bm;
829 clib_error_t *err;
Damjan Marionb592d1b2019-02-28 23:16:11 +0100830 clib_bitmap_t *bmp = 0, *bmp_has_memory = 0;
Damjan Marion910d3692019-01-21 11:48:34 +0100831 u32 numa_node;
Filip Tehlard2bbdef2019-02-22 05:05:53 -0800832 vlib_buffer_pool_t *bp;
Damjan Marionb592d1b2019-02-28 23:16:11 +0100833 u8 *name = 0, first_valid_buffer_pool_index = ~0;
Damjan Marion910d3692019-01-21 11:48:34 +0100834
Damjan Marion5de3fec2019-02-06 14:22:32 +0100835 vlib_buffer_main_alloc (vm);
Damjan Marion910d3692019-01-21 11:48:34 +0100836
837 bm = vm->buffer_main;
838 bm->log_default = vlib_log_register_class ("buffer", 0);
839 bm->ext_hdr_size = __vlib_buffer_external_hdr_size;
840
841 clib_spinlock_init (&bm->buffer_known_hash_lockp);
842
Damjan Marion4b661402024-02-29 16:14:27 +0100843 bmp = os_get_online_cpu_node_bitmap ();
Damjan Marion3eb6cbe2024-02-12 19:44:58 +0000844 bmp_has_memory = os_get_cpu_with_memory_bitmap ();
Damjan Marionb592d1b2019-02-28 23:16:11 +0100845
846 if (bmp && bmp_has_memory)
847 bmp = clib_bitmap_and (bmp, bmp_has_memory);
848
849 /* no info from sysfs, assuming that only numa 0 exists */
850 if (bmp == 0)
851 bmp = clib_bitmap_set (bmp, 0, 1);
852
853 if (clib_bitmap_last_set (bmp) >= VLIB_BUFFER_MAX_NUMA_NODES)
854 clib_panic ("system have more than %u NUMA nodes",
855 VLIB_BUFFER_MAX_NUMA_NODES);
Damjan Marion910d3692019-01-21 11:48:34 +0100856
Damjan Marionf0ca1e82020-12-13 23:26:56 +0100857 clib_bitmap_foreach (numa_node, bmp)
Damjan Marionb592d1b2019-02-28 23:16:11 +0100858 {
859 u8 *index = bm->default_buffer_pool_index_for_numa + numa_node;
860 index[0] = ~0;
Lukas Stocknerb4808de2024-06-18 14:52:52 +0200861
862 if (bm->buffers_per_numa[numa_node] == 0)
863 continue;
864
Damjan Marionb592d1b2019-02-28 23:16:11 +0100865 if ((err = vlib_buffer_main_init_numa_node (vm, numa_node, index)))
866 {
867 clib_error_report (err);
868 clib_error_free (err);
869 continue;
870 }
871
872 if (first_valid_buffer_pool_index == 0xff)
873 first_valid_buffer_pool_index = index[0];
Damjan Marionf0ca1e82020-12-13 23:26:56 +0100874 }
Damjan Marion910d3692019-01-21 11:48:34 +0100875
Damjan Marionb592d1b2019-02-28 23:16:11 +0100876 if (first_valid_buffer_pool_index == (u8) ~ 0)
877 {
878 err = clib_error_return (0, "failed to allocate buffer pool(s)");
879 goto done;
880 }
881
Damjan Marionf0ca1e82020-12-13 23:26:56 +0100882 clib_bitmap_foreach (numa_node, bmp)
Damjan Marionb592d1b2019-02-28 23:16:11 +0100883 {
884 if (bm->default_buffer_pool_index_for_numa[numa_node] == (u8) ~0)
885 bm->default_buffer_pool_index_for_numa[numa_node] =
886 first_valid_buffer_pool_index;
Damjan Marionf0ca1e82020-12-13 23:26:56 +0100887 }
Damjan Marion910d3692019-01-21 11:48:34 +0100888
Filip Tehlard2bbdef2019-02-22 05:05:53 -0800889 vec_foreach (bp, bm->buffer_pools)
890 {
Damjan Marion8973b072022-03-01 15:51:18 +0100891 vlib_stats_collector_reg_t reg = { .private_data = bp - bm->buffer_pools };
Damjan Marionb592d1b2019-02-28 23:16:11 +0100892 if (bp->n_buffers == 0)
893 continue;
894
Damjan Marion8973b072022-03-01 15:51:18 +0100895 reg.entry_index =
Damjan Marion55a1f282023-10-24 23:59:48 +0200896 vlib_stats_add_gauge ("/buffer-pools/%v/cached", bp->name);
Damjan Marion8973b072022-03-01 15:51:18 +0100897 reg.collect_fn = buffer_gauges_collect_cached_fn;
898 vlib_stats_register_collector_fn (&reg);
Damjan Marionb592d1b2019-02-28 23:16:11 +0100899
Damjan Marion55a1f282023-10-24 23:59:48 +0200900 reg.entry_index = vlib_stats_add_gauge ("/buffer-pools/%v/used", bp->name);
Damjan Marion8973b072022-03-01 15:51:18 +0100901 reg.collect_fn = buffer_gauges_collect_used_fn;
902 vlib_stats_register_collector_fn (&reg);
Damjan Marionb592d1b2019-02-28 23:16:11 +0100903
Damjan Marion8973b072022-03-01 15:51:18 +0100904 reg.entry_index =
Damjan Marion55a1f282023-10-24 23:59:48 +0200905 vlib_stats_add_gauge ("/buffer-pools/%v/available", bp->name);
Damjan Marion8973b072022-03-01 15:51:18 +0100906 reg.collect_fn = buffer_gauges_collect_available_fn;
907 vlib_stats_register_collector_fn (&reg);
Filip Tehlard2bbdef2019-02-22 05:05:53 -0800908 }
909
Damjan Marion910d3692019-01-21 11:48:34 +0100910done:
911 vec_free (bmp);
Damjan Marionb592d1b2019-02-28 23:16:11 +0100912 vec_free (bmp_has_memory);
913 vec_free (name);
Damjan Marion910d3692019-01-21 11:48:34 +0100914 return err;
Damjan Marion878c6092017-01-04 13:19:27 +0100915}
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100916
Damjan Marion49d66f12017-07-20 18:10:35 +0200917static clib_error_t *
Lukas Stockner63ff7a62024-06-04 16:14:55 +0200918vlib_buffers_numa_configure (vlib_buffer_main_t *bm, u32 numa_node,
919 unformat_input_t *input)
920{
Lukas Stocknerb4808de2024-06-18 14:52:52 +0200921 u32 buffers = ~0;
Lukas Stockner63ff7a62024-06-04 16:14:55 +0200922
923 if (numa_node >= VLIB_BUFFER_MAX_NUMA_NODES)
924 return clib_error_return (0, "invalid numa node");
925
926 if (!input)
927 return 0;
928
929 unformat_skip_white_space (input);
930 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
931 {
932 if (unformat (input, "buffers %u", &buffers))
933 ;
934 else
935 return unformat_parse_error (input);
936 }
937
938 bm->buffers_per_numa[numa_node] = buffers;
939 return 0;
940}
941
942static clib_error_t *
Damjan Marion49d66f12017-07-20 18:10:35 +0200943vlib_buffers_configure (vlib_main_t * vm, unformat_input_t * input)
944{
Damjan Marion910d3692019-01-21 11:48:34 +0100945 vlib_buffer_main_t *bm;
Lukas Stockner63ff7a62024-06-04 16:14:55 +0200946 u32 numa_node;
947 unformat_input_t sub_input;
948 clib_error_t *error = 0;
Damjan Marion910d3692019-01-21 11:48:34 +0100949
Damjan Marion5de3fec2019-02-06 14:22:32 +0100950 vlib_buffer_main_alloc (vm);
Damjan Marion910d3692019-01-21 11:48:34 +0100951
952 bm = vm->buffer_main;
Nathan Skrzypczak61559022020-11-23 16:25:21 +0100953 bm->log2_page_size = CLIB_MEM_PAGE_SZ_UNKNOWN;
Lukas Stocknerb4808de2024-06-18 14:52:52 +0200954 memset (bm->buffers_per_numa, ~0, sizeof (bm->buffers_per_numa));
Damjan Marion49d66f12017-07-20 18:10:35 +0200955
956 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
957 {
Lukas Stockner63ff7a62024-06-04 16:14:55 +0200958 if (unformat (input, "buffers-per-numa %u",
959 &bm->default_buffers_per_numa))
Damjan Marion910d3692019-01-21 11:48:34 +0100960 ;
Nathan Skrzypczak61559022020-11-23 16:25:21 +0100961 else if (unformat (input, "page-size %U", unformat_log2_page_size,
962 &bm->log2_page_size))
963 ;
Damjan Marion5de3fec2019-02-06 14:22:32 +0100964 else if (unformat (input, "default data-size %u",
965 &bm->default_data_size))
966 ;
Lukas Stockner63ff7a62024-06-04 16:14:55 +0200967 else if (unformat (input, "numa %u %U", &numa_node,
968 unformat_vlib_cli_sub_input, &sub_input))
969 {
970 error = vlib_buffers_numa_configure (bm, numa_node, &sub_input);
971 unformat_free (&sub_input);
972
973 if (error)
974 return error;
975 }
Damjan Marion49d66f12017-07-20 18:10:35 +0200976 else
977 return unformat_parse_error (input);
978 }
979
980 unformat_free (input);
981 return 0;
982}
983
984VLIB_EARLY_CONFIG_FUNCTION (vlib_buffers_configure, "buffers");
985
Dave Barachc74b43c2020-04-09 17:24:07 -0400986#if VLIB_BUFFER_ALLOC_FAULT_INJECTOR > 0
987u32
988vlib_buffer_alloc_may_fail (vlib_main_t * vm, u32 n_buffers)
989{
990 f64 r;
991
992 r = random_f64 (&vm->buffer_alloc_success_seed);
993
994 /* Fail this request? */
995 if (r > vm->buffer_alloc_success_rate)
996 n_buffers--;
997 /* 5% chance of returning nothing at all */
998 if (r > vm->buffer_alloc_success_rate && r > 0.95)
999 n_buffers = 0;
1000
1001 return n_buffers;
1002}
1003#endif
Damjan Marion49d66f12017-07-20 18:10:35 +02001004
Benoît Gannee09a2332021-03-09 15:37:49 +01001005__clib_export int
1006vlib_buffer_set_alloc_free_callback (
1007 vlib_main_t *vm, vlib_buffer_alloc_free_callback_t *alloc_callback_fn,
1008 vlib_buffer_alloc_free_callback_t *free_callback_fn)
1009{
1010 vlib_buffer_main_t *bm = vm->buffer_main;
1011 if ((alloc_callback_fn && bm->alloc_callback_fn) ||
1012 (free_callback_fn && bm->free_callback_fn))
1013 return 1;
1014 bm->alloc_callback_fn = alloc_callback_fn;
1015 bm->free_callback_fn = free_callback_fn;
1016 return 0;
1017}
1018
Chris Luked4024f52016-09-06 09:32:36 -04001019/** @endcond */
Dave Barach9b8ffd92016-07-08 08:13:45 -04001020/*
1021 * fd.io coding-style-patch-verification: ON
1022 *
1023 * Local Variables:
1024 * eval: (c-set-style "gnu")
1025 * End:
1026 */