blob: 547c26445af7ef13170244ba4e8c49d0cba01215 [file] [log] [blame]
Ed Warnickecb9cada2015-12-08 15:45:58 -07001/*
2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15/*
16 * buffer.c: allocate/free network buffers.
17 *
18 * Copyright (c) 2008 Eliot Dresselhaus
19 *
20 * Permission is hereby granted, free of charge, to any person obtaining
21 * a copy of this software and associated documentation files (the
22 * "Software"), to deal in the Software without restriction, including
23 * without limitation the rights to use, copy, modify, merge, publish,
24 * distribute, sublicense, and/or sell copies of the Software, and to
25 * permit persons to whom the Software is furnished to do so, subject to
26 * the following conditions:
27 *
28 * The above copyright notice and this permission notice shall be
29 * included in all copies or substantial portions of the Software.
30 *
31 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38 */
39
Chris Luked4024f52016-09-06 09:32:36 -040040/**
Chris Luked4024f52016-09-06 09:32:36 -040041 * @file
42 *
43 * Allocate/free network buffers.
44 */
45
Ed Warnickecb9cada2015-12-08 15:45:58 -070046#include <vlib/vlib.h>
Damjan Marion374e2c52017-03-09 20:38:15 +010047#include <vlib/unix/unix.h>
Ed Warnickecb9cada2015-12-08 15:45:58 -070048
Damjan Marion04a7f052017-07-10 15:06:17 +020049vlib_buffer_callbacks_t *vlib_buffer_callbacks = 0;
Damjan Marion901d16c2018-10-23 19:50:20 +020050
51/* when running unpriviledged we are limited by RLIMIT_MEMLOCK which is
52 typically set to 16MB so setting default size for buffer memory to 14MB
53 */
54static u32 vlib_buffer_physmem_sz = 14 << 20;
Damjan Marion04a7f052017-07-10 15:06:17 +020055
Damjan Marion567e61d2018-10-24 17:08:26 +020056/* logging */
57static vlib_log_class_t buffer_log_default;
58
Dave Barach9b8ffd92016-07-08 08:13:45 -040059uword
60vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm,
61 vlib_buffer_t * b_first)
Ed Warnickecb9cada2015-12-08 15:45:58 -070062{
Dave Barach9b8ffd92016-07-08 08:13:45 -040063 vlib_buffer_t *b = b_first;
Ed Warnickecb9cada2015-12-08 15:45:58 -070064 uword l_first = b_first->current_length;
65 uword l = 0;
66 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
67 {
68 b = vlib_get_buffer (vm, b->next_buffer);
69 l += b->current_length;
70 }
71 b_first->total_length_not_including_first_buffer = l;
72 b_first->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
73 return l + l_first;
74}
75
Dave Barach9b8ffd92016-07-08 08:13:45 -040076u8 *
77format_vlib_buffer (u8 * s, va_list * args)
Ed Warnickecb9cada2015-12-08 15:45:58 -070078{
Dave Barach9b8ffd92016-07-08 08:13:45 -040079 vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
Christophe Fontained3c008d2017-10-02 18:10:54 +020080 u32 indent = format_get_indent (s);
Damjan Mariondac03522018-02-01 15:30:13 +010081 u8 *a = 0;
Damjan Marion1cd8f3c2016-11-24 02:07:32 +010082
Damjan Mariondac03522018-02-01 15:30:13 +010083#define _(bit, name, v) \
84 if (v && (b->flags & VLIB_BUFFER_##name)) \
85 a = format (a, "%s ", v);
86 foreach_vlib_buffer_flag
87#undef _
Damjan Marion36eb7c22019-01-18 20:45:30 +010088 s = format (s, "current data %d, length %d, buffer-pool %d, "
89 "clone-count %u", b->current_data, b->current_length,
90 b->buffer_pool_index, b->n_add_refs);
Damjan Marion1cd8f3c2016-11-24 02:07:32 +010091
92 if (b->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID)
93 s = format (s, ", totlen-nifb %d",
94 b->total_length_not_including_first_buffer);
95
96 if (b->flags & VLIB_BUFFER_IS_TRACED)
97 s = format (s, ", trace 0x%x", b->trace_index);
98
Damjan Mariondac03522018-02-01 15:30:13 +010099 if (a)
100 s = format (s, "\n%U%v", format_white_space, indent, a);
101 vec_free (a);
102
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100103 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
104 {
105 vlib_main_t *vm = vlib_get_main ();
106 u32 next_buffer = b->next_buffer;
107 b = vlib_get_buffer (vm, next_buffer);
108
Damjan Marionc47ed032017-01-25 14:18:03 +0100109 s =
110 format (s, "\n%Unext-buffer 0x%x, segment length %d, clone-count %u",
111 format_white_space, indent, next_buffer, b->current_length,
112 b->n_add_refs);
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100113 }
114
Ed Warnickecb9cada2015-12-08 15:45:58 -0700115 return s;
116}
117
Dave Barach9b8ffd92016-07-08 08:13:45 -0400118u8 *
119format_vlib_buffer_and_data (u8 * s, va_list * args)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700120{
Dave Barach9b8ffd92016-07-08 08:13:45 -0400121 vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
122
Ed Warnickecb9cada2015-12-08 15:45:58 -0700123 s = format (s, "%U, %U",
124 format_vlib_buffer, b,
125 format_hex_bytes, vlib_buffer_get_current (b), 64);
126
127 return s;
128}
129
Dave Barach9b8ffd92016-07-08 08:13:45 -0400130static u8 *
131format_vlib_buffer_known_state (u8 * s, va_list * args)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700132{
133 vlib_buffer_known_state_t state = va_arg (*args, vlib_buffer_known_state_t);
Dave Barach9b8ffd92016-07-08 08:13:45 -0400134 char *t;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700135
136 switch (state)
137 {
138 case VLIB_BUFFER_UNKNOWN:
139 t = "unknown";
140 break;
141
142 case VLIB_BUFFER_KNOWN_ALLOCATED:
143 t = "known-allocated";
144 break;
145
146 case VLIB_BUFFER_KNOWN_FREE:
147 t = "known-free";
148 break;
149
150 default:
151 t = "invalid";
152 break;
153 }
154
155 return format (s, "%s", t);
156}
157
Dave Barach9b8ffd92016-07-08 08:13:45 -0400158u8 *
159format_vlib_buffer_contents (u8 * s, va_list * va)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700160{
Dave Barach9b8ffd92016-07-08 08:13:45 -0400161 vlib_main_t *vm = va_arg (*va, vlib_main_t *);
162 vlib_buffer_t *b = va_arg (*va, vlib_buffer_t *);
163
Ed Warnickecb9cada2015-12-08 15:45:58 -0700164 while (1)
165 {
Dave Barach9b8ffd92016-07-08 08:13:45 -0400166 vec_add (s, vlib_buffer_get_current (b), b->current_length);
167 if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))
Ed Warnickecb9cada2015-12-08 15:45:58 -0700168 break;
169 b = vlib_get_buffer (vm, b->next_buffer);
170 }
171
172 return s;
173}
174
175static u8 *
176vlib_validate_buffer_helper (vlib_main_t * vm,
177 u32 bi,
Dave Barach9b8ffd92016-07-08 08:13:45 -0400178 uword follow_buffer_next, uword ** unique_hash)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700179{
Dave Barach9b8ffd92016-07-08 08:13:45 -0400180 vlib_buffer_t *b = vlib_get_buffer (vm, bi);
Dave Barach9b8ffd92016-07-08 08:13:45 -0400181 vlib_buffer_free_list_t *fl;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700182
Damjan Marion36eb7c22019-01-18 20:45:30 +0100183 fl = pool_elt_at_index (vm->buffer_free_list_pool,
184 VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700185
Dave Barach9b8ffd92016-07-08 08:13:45 -0400186 if ((signed) b->current_data < (signed) -VLIB_BUFFER_PRE_DATA_SIZE)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700187 return format (0, "current data %d before pre-data", b->current_data);
Damjan Marion878c6092017-01-04 13:19:27 +0100188
Ed Warnickecb9cada2015-12-08 15:45:58 -0700189 if (b->current_data + b->current_length > fl->n_data_bytes)
190 return format (0, "%d-%d beyond end of buffer %d",
Dave Barach9b8ffd92016-07-08 08:13:45 -0400191 b->current_data, b->current_length, fl->n_data_bytes);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700192
Dave Barach9b8ffd92016-07-08 08:13:45 -0400193 if (follow_buffer_next && (b->flags & VLIB_BUFFER_NEXT_PRESENT))
Ed Warnickecb9cada2015-12-08 15:45:58 -0700194 {
195 vlib_buffer_known_state_t k;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400196 u8 *msg, *result;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700197
Damjan Mariond50e3472019-01-20 00:03:56 +0100198 k = vlib_buffer_is_known (vm, b->next_buffer);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700199 if (k != VLIB_BUFFER_KNOWN_ALLOCATED)
200 return format (0, "next 0x%x: %U",
Dave Barach9b8ffd92016-07-08 08:13:45 -0400201 b->next_buffer, format_vlib_buffer_known_state, k);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700202
203 if (unique_hash)
204 {
205 if (hash_get (*unique_hash, b->next_buffer))
206 return format (0, "duplicate buffer 0x%x", b->next_buffer);
207
208 hash_set1 (*unique_hash, b->next_buffer);
209 }
210
211 msg = vlib_validate_buffer (vm, b->next_buffer, follow_buffer_next);
212 if (msg)
213 {
214 result = format (0, "next 0x%x: %v", b->next_buffer, msg);
215 vec_free (msg);
216 return result;
217 }
218 }
219
220 return 0;
221}
222
223u8 *
224vlib_validate_buffer (vlib_main_t * vm, u32 bi, uword follow_buffer_next)
Dave Barach9b8ffd92016-07-08 08:13:45 -0400225{
226 return vlib_validate_buffer_helper (vm, bi, follow_buffer_next,
227 /* unique_hash */ 0);
228}
Ed Warnickecb9cada2015-12-08 15:45:58 -0700229
230u8 *
231vlib_validate_buffers (vlib_main_t * vm,
232 u32 * buffers,
233 uword next_buffer_stride,
234 uword n_buffers,
235 vlib_buffer_known_state_t known_state,
236 uword follow_buffer_next)
237{
Dave Barach9b8ffd92016-07-08 08:13:45 -0400238 uword i, *hash;
239 u32 bi, *b = buffers;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700240 vlib_buffer_known_state_t k;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400241 u8 *msg = 0, *result = 0;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700242
243 hash = hash_create (0, 0);
244 for (i = 0; i < n_buffers; i++)
245 {
246 bi = b[0];
247 b += next_buffer_stride;
248
249 /* Buffer is not unique. */
250 if (hash_get (hash, bi))
251 {
252 msg = format (0, "not unique");
253 goto done;
254 }
255
Damjan Mariond50e3472019-01-20 00:03:56 +0100256 k = vlib_buffer_is_known (vm, bi);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700257 if (k != known_state)
258 {
259 msg = format (0, "is %U; expected %U",
260 format_vlib_buffer_known_state, k,
261 format_vlib_buffer_known_state, known_state);
262 goto done;
263 }
264
265 msg = vlib_validate_buffer_helper (vm, bi, follow_buffer_next, &hash);
266 if (msg)
267 goto done;
268
269 hash_set1 (hash, bi);
270 }
271
Dave Barach9b8ffd92016-07-08 08:13:45 -0400272done:
Ed Warnickecb9cada2015-12-08 15:45:58 -0700273 if (msg)
274 {
275 result = format (0, "0x%x: %v", bi, msg);
276 vec_free (msg);
277 }
278 hash_free (hash);
279 return result;
280}
281
Dave Barach80f54e22017-03-08 19:08:56 -0500282/*
283 * Hand-craft a static vector w/ length 1, so vec_len(vlib_mains) =1
284 * and vlib_mains[0] = &vlib_global_main from the beginning of time.
285 *
286 * The only place which should ever expand vlib_mains is start_workers()
287 * in threads.c. It knows about the bootstrap vector.
288 */
289/* *INDENT-OFF* */
290static struct
291{
292 vec_header_t h;
293 vlib_main_t *vm;
294} __attribute__ ((packed)) __bootstrap_vlib_main_vector
295 __attribute__ ((aligned (CLIB_CACHE_LINE_BYTES))) =
296{
297 .h.len = 1,
298 .vm = &vlib_global_main,
299};
300/* *INDENT-ON* */
301
302vlib_main_t **vlib_mains = &__bootstrap_vlib_main_vector.vm;
303
Ed Warnickecb9cada2015-12-08 15:45:58 -0700304
305/* When dubugging validate that given buffers are either known allocated
306 or known free. */
Damjan Marionc8a26c62017-11-24 20:15:23 +0100307void
Ed Warnickecb9cada2015-12-08 15:45:58 -0700308vlib_buffer_validate_alloc_free (vlib_main_t * vm,
309 u32 * buffers,
310 uword n_buffers,
311 vlib_buffer_known_state_t expected_state)
312{
Dave Barach9b8ffd92016-07-08 08:13:45 -0400313 u32 *b;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700314 uword i, bi, is_free;
315
316 if (CLIB_DEBUG == 0)
317 return;
318
Damjan Marionc8a26c62017-11-24 20:15:23 +0100319 if (vlib_buffer_callbacks)
320 return;
321
Ed Warnickecb9cada2015-12-08 15:45:58 -0700322 is_free = expected_state == VLIB_BUFFER_KNOWN_ALLOCATED;
323 b = buffers;
324 for (i = 0; i < n_buffers; i++)
325 {
326 vlib_buffer_known_state_t known;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400327
Ed Warnickecb9cada2015-12-08 15:45:58 -0700328 bi = b[0];
329 b += 1;
Damjan Mariond50e3472019-01-20 00:03:56 +0100330 known = vlib_buffer_is_known (vm, bi);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700331 if (known != expected_state)
332 {
333 ASSERT (0);
334 vlib_panic_with_msg
335 (vm, "%s %U buffer 0x%x",
336 is_free ? "freeing" : "allocating",
Dave Barach9b8ffd92016-07-08 08:13:45 -0400337 format_vlib_buffer_known_state, known, bi);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700338 }
339
Damjan Mariond50e3472019-01-20 00:03:56 +0100340 vlib_buffer_set_known_state (vm, bi, is_free ? VLIB_BUFFER_KNOWN_FREE :
341 VLIB_BUFFER_KNOWN_ALLOCATED);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700342 }
343}
Ed Warnickecb9cada2015-12-08 15:45:58 -0700344
Ed Warnickecb9cada2015-12-08 15:45:58 -0700345/* Add buffer free list. */
Damjan Mariondac03522018-02-01 15:30:13 +0100346static vlib_buffer_free_list_index_t
Ed Warnickecb9cada2015-12-08 15:45:58 -0700347vlib_buffer_create_free_list_helper (vlib_main_t * vm,
348 u32 n_data_bytes,
Dave Barach9b8ffd92016-07-08 08:13:45 -0400349 u32 is_public, u32 is_default, u8 * name)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700350{
Damjan Mariond50e3472019-01-20 00:03:56 +0100351 vlib_buffer_main_t *bm = vm->buffer_main;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400352 vlib_buffer_free_list_t *f;
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100353 int i;
354
Damjan Marion586afd72017-04-05 19:18:20 +0200355 ASSERT (vlib_get_thread_index () == 0);
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100356
Damjan Mariond1274cb2018-03-13 21:32:17 +0100357 if (!is_default && pool_elts (vm->buffer_free_list_pool) == 0)
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100358 {
Damjan Mariondac03522018-02-01 15:30:13 +0100359 vlib_buffer_free_list_index_t default_free_free_list_index;
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100360
361 /* *INDENT-OFF* */
362 default_free_free_list_index =
363 vlib_buffer_create_free_list_helper
364 (vm,
365 /* default buffer size */ VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES,
366 /* is_public */ 1,
367 /* is_default */ 1,
368 (u8 *) "default");
369 /* *INDENT-ON* */
370 ASSERT (default_free_free_list_index ==
371 VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
372
373 if (n_data_bytes == VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES && is_public)
374 return default_free_free_list_index;
375 }
376
Damjan Mariond1274cb2018-03-13 21:32:17 +0100377 pool_get_aligned (vm->buffer_free_list_pool, f, CLIB_CACHE_LINE_BYTES);
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100378
Dave Barachb7b92992018-10-17 10:38:51 -0400379 clib_memset (f, 0, sizeof (f[0]));
Damjan Mariond1274cb2018-03-13 21:32:17 +0100380 f->index = f - vm->buffer_free_list_pool;
Eyal Barice55bcd2018-11-25 15:42:47 +0200381 vec_validate (f->buffers, 0);
382 vec_reset_length (f->buffers);
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100383 f->n_data_bytes = vlib_buffer_round_size (n_data_bytes);
Damjan Mariond1274cb2018-03-13 21:32:17 +0100384 f->min_n_buffers_each_alloc = VLIB_FRAME_SIZE;
385 f->buffer_pool_index = 0;
Chris Luke7447d072017-07-05 12:57:10 -0400386 f->name = clib_mem_is_vec (name) ? name : format (0, "%s", name);
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100387
388 /* Setup free buffer template. */
Damjan Marionc47ed032017-01-25 14:18:03 +0100389 f->buffer_init_template.n_add_refs = 0;
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100390
391 if (is_public)
392 {
393 uword *p = hash_get (bm->free_list_by_size, f->n_data_bytes);
394 if (!p)
395 hash_set (bm->free_list_by_size, f->n_data_bytes, f->index);
396 }
397
398 for (i = 1; i < vec_len (vlib_mains); i++)
399 {
Damjan Mariond1274cb2018-03-13 21:32:17 +0100400 vlib_main_t *wvm = vlib_mains[i];
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100401 vlib_buffer_free_list_t *wf;
Damjan Mariond1274cb2018-03-13 21:32:17 +0100402 pool_get_aligned (wvm->buffer_free_list_pool,
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100403 wf, CLIB_CACHE_LINE_BYTES);
Damjan Mariond1274cb2018-03-13 21:32:17 +0100404 ASSERT (f - vm->buffer_free_list_pool ==
405 wf - wvm->buffer_free_list_pool);
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100406 wf[0] = f[0];
Damjan Marionbd69a5f2017-02-05 23:44:42 +0100407 wf->buffers = 0;
Eyal Barice55bcd2018-11-25 15:42:47 +0200408 vec_validate (wf->buffers, 0);
409 vec_reset_length (wf->buffers);
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100410 wf->n_alloc = 0;
411 }
Ed Warnickecb9cada2015-12-08 15:45:58 -0700412
413 return f->index;
414}
415
Damjan Mariondac03522018-02-01 15:30:13 +0100416vlib_buffer_free_list_index_t
Dave Barach9b8ffd92016-07-08 08:13:45 -0400417vlib_buffer_create_free_list (vlib_main_t * vm, u32 n_data_bytes,
418 char *fmt, ...)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700419{
420 va_list va;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400421 u8 *name;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700422
423 va_start (va, fmt);
424 name = va_format (0, fmt, &va);
425 va_end (va);
426
427 return vlib_buffer_create_free_list_helper (vm, n_data_bytes,
428 /* is_public */ 0,
429 /* is_default */ 0,
430 name);
431}
432
Ed Warnickecb9cada2015-12-08 15:45:58 -0700433static void
434del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f)
435{
Damjan Mariond50e3472019-01-20 00:03:56 +0100436 vlib_buffer_pool_t *bp = vlib_buffer_pool_get (vm, f->buffer_pool_index);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700437
Damjan Mariond1274cb2018-03-13 21:32:17 +0100438 vec_add_aligned (bp->buffers, f->buffers, vec_len (f->buffers),
439 CLIB_CACHE_LINE_BYTES);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700440 vec_free (f->name);
Damjan Marionbd69a5f2017-02-05 23:44:42 +0100441 vec_free (f->buffers);
Damjan Mariond1274cb2018-03-13 21:32:17 +0100442
443 /* Poison it. */
Dave Barachb7b92992018-10-17 10:38:51 -0400444 clib_memset (f, 0xab, sizeof (f[0]));
Ed Warnickecb9cada2015-12-08 15:45:58 -0700445}
446
447/* Add buffer free list. */
Dave Barach9b8ffd92016-07-08 08:13:45 -0400448void
Damjan Mariondac03522018-02-01 15:30:13 +0100449vlib_buffer_delete_free_list_internal (vlib_main_t * vm,
450 vlib_buffer_free_list_index_t index)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700451{
Dave Barach9b8ffd92016-07-08 08:13:45 -0400452 vlib_buffer_free_list_t *f;
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100453 int i;
454
Damjan Marion586afd72017-04-05 19:18:20 +0200455 ASSERT (vlib_get_thread_index () == 0);
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100456
Damjan Mariondac03522018-02-01 15:30:13 +0100457 f = vlib_buffer_get_free_list (vm, index);
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100458
Damjan Marionbd69a5f2017-02-05 23:44:42 +0100459 ASSERT (vec_len (f->buffers) == f->n_alloc);
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100460
461 del_free_list (vm, f);
462
Damjan Mariond1274cb2018-03-13 21:32:17 +0100463 pool_put (vm->buffer_free_list_pool, f);
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100464
465 for (i = 1; i < vec_len (vlib_mains); i++)
466 {
Damjan Mariond1274cb2018-03-13 21:32:17 +0100467 vlib_main_t *wvm = vlib_mains[i];
468 f = vlib_buffer_get_free_list (vlib_mains[i], index);
469 del_free_list (wvm, f);
470 pool_put (wvm->buffer_free_list_pool, f);
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100471 }
Ed Warnickecb9cada2015-12-08 15:45:58 -0700472}
473
Damjan Mariond1274cb2018-03-13 21:32:17 +0100474static_always_inline void *
Damjan Marion68b4da62018-09-30 18:26:20 +0200475vlib_buffer_pool_get_buffer (vlib_main_t * vm, vlib_buffer_pool_t * bp)
Damjan Mariond1274cb2018-03-13 21:32:17 +0100476{
Damjan Marion68b4da62018-09-30 18:26:20 +0200477 return vlib_physmem_alloc_from_map (vm, bp->physmem_map_index,
478 bp->buffer_size, CLIB_CACHE_LINE_BYTES);
Damjan Mariond1274cb2018-03-13 21:32:17 +0100479}
480
Ed Warnickecb9cada2015-12-08 15:45:58 -0700481/* Make sure free list has at least given number of free buffers. */
482static uword
Damjan Marionc8a26c62017-11-24 20:15:23 +0100483vlib_buffer_fill_free_list_internal (vlib_main_t * vm,
484 vlib_buffer_free_list_t * fl,
485 uword min_free_buffers)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700486{
Damjan Mariond1274cb2018-03-13 21:32:17 +0100487 vlib_buffer_t *b;
Damjan Mariond50e3472019-01-20 00:03:56 +0100488 vlib_buffer_pool_t *bp = vlib_buffer_pool_get (vm, fl->buffer_pool_index);
Damjan Mariond1274cb2018-03-13 21:32:17 +0100489 int n;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400490 u32 *bi;
Damjan Mariond1274cb2018-03-13 21:32:17 +0100491 u32 n_alloc = 0;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700492
Ed Warnickecb9cada2015-12-08 15:45:58 -0700493 /* Already have enough free buffers on free list? */
Damjan Marionbd69a5f2017-02-05 23:44:42 +0100494 n = min_free_buffers - vec_len (fl->buffers);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700495 if (n <= 0)
496 return min_free_buffers;
497
Damjan Mariond1274cb2018-03-13 21:32:17 +0100498 if (vec_len (bp->buffers) > 0)
Damjan Marionb6a8ed72017-08-29 00:15:35 +0200499 {
500 int n_copy, n_left;
Damjan Mariond1274cb2018-03-13 21:32:17 +0100501 clib_spinlock_lock (&bp->lock);
502 n_copy = clib_min (vec_len (bp->buffers), n);
503 n_left = vec_len (bp->buffers) - n_copy;
504 vec_add_aligned (fl->buffers, bp->buffers + n_left, n_copy,
Damjan Marionb6a8ed72017-08-29 00:15:35 +0200505 CLIB_CACHE_LINE_BYTES);
Damjan Mariond1274cb2018-03-13 21:32:17 +0100506 _vec_len (bp->buffers) = n_left;
507 clib_spinlock_unlock (&bp->lock);
Damjan Marionb6a8ed72017-08-29 00:15:35 +0200508 n = min_free_buffers - vec_len (fl->buffers);
509 if (n <= 0)
510 return min_free_buffers;
511 }
512
Ed Warnickecb9cada2015-12-08 15:45:58 -0700513 /* Always allocate round number of buffers. */
Damjan Marionbd69a5f2017-02-05 23:44:42 +0100514 n = round_pow2 (n, CLIB_CACHE_LINE_BYTES / sizeof (u32));
Ed Warnickecb9cada2015-12-08 15:45:58 -0700515
516 /* Always allocate new buffers in reasonably large sized chunks. */
Damjan Mariond1274cb2018-03-13 21:32:17 +0100517 n = clib_max (n, fl->min_n_buffers_each_alloc);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700518
Damjan Mariond1274cb2018-03-13 21:32:17 +0100519 clib_spinlock_lock (&bp->lock);
520 while (n_alloc < n)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700521 {
Damjan Marion68b4da62018-09-30 18:26:20 +0200522 if ((b = vlib_buffer_pool_get_buffer (vm, bp)) == 0)
Damjan Mariond1274cb2018-03-13 21:32:17 +0100523 goto done;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400524
Damjan Mariond1274cb2018-03-13 21:32:17 +0100525 n_alloc += 1;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700526
Damjan Mariond1274cb2018-03-13 21:32:17 +0100527 vec_add2_aligned (fl->buffers, bi, 1, CLIB_CACHE_LINE_BYTES);
528 bi[0] = vlib_get_buffer_index (vm, b);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700529
Damjan Mariond1274cb2018-03-13 21:32:17 +0100530 if (CLIB_DEBUG > 0)
Damjan Mariond50e3472019-01-20 00:03:56 +0100531 vlib_buffer_set_known_state (vm, bi[0], VLIB_BUFFER_KNOWN_FREE);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700532
Dave Barachb7b92992018-10-17 10:38:51 -0400533 clib_memset (b, 0, sizeof (vlib_buffer_t));
Damjan Mariond1274cb2018-03-13 21:32:17 +0100534 vlib_buffer_init_for_free_list (b, fl);
Dave Barach9b8ffd92016-07-08 08:13:45 -0400535
Ed Warnickecb9cada2015-12-08 15:45:58 -0700536 if (fl->buffer_init_function)
Damjan Mariond1274cb2018-03-13 21:32:17 +0100537 fl->buffer_init_function (vm, fl, bi, 1);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700538 }
Damjan Mariond1274cb2018-03-13 21:32:17 +0100539
540done:
541 clib_spinlock_unlock (&bp->lock);
542 fl->n_alloc += n_alloc;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700543 return n_alloc;
544}
545
Dave Barach9b8ffd92016-07-08 08:13:45 -0400546void *
547vlib_set_buffer_free_callback (vlib_main_t * vm, void *fp)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700548{
Damjan Mariond50e3472019-01-20 00:03:56 +0100549 vlib_buffer_main_t *bm = vm->buffer_main;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400550 void *rv = bm->buffer_free_callback;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700551
552 bm->buffer_free_callback = fp;
553 return rv;
554}
555
Ed Warnickecb9cada2015-12-08 15:45:58 -0700556static_always_inline void
Damjan Marioneac3b112018-03-05 09:36:31 +0100557recycle_or_free (vlib_main_t * vm, vlib_buffer_main_t * bm, u32 bi,
558 vlib_buffer_t * b, u32 follow_buffer_next)
559{
560 vlib_buffer_free_list_t *fl;
Damjan Marion8e715292018-09-03 15:41:45 +0200561 u32 flags, next;
562
Damjan Marion36eb7c22019-01-18 20:45:30 +0100563 fl = pool_elt_at_index (vm->buffer_free_list_pool,
564 VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
Damjan Marioneac3b112018-03-05 09:36:31 +0100565
Damjan Marion8e715292018-09-03 15:41:45 +0200566 do
Damjan Marioneac3b112018-03-05 09:36:31 +0100567 {
Damjan Marion8e715292018-09-03 15:41:45 +0200568 vlib_buffer_t *nb = vlib_get_buffer (vm, bi);
569 flags = nb->flags;
570 next = nb->next_buffer;
571 if (nb->n_add_refs)
572 nb->n_add_refs--;
573 else
Damjan Marioneac3b112018-03-05 09:36:31 +0100574 {
Damjan Marion8e715292018-09-03 15:41:45 +0200575 vlib_buffer_validate_alloc_free (vm, &bi, 1,
576 VLIB_BUFFER_KNOWN_ALLOCATED);
577 vlib_buffer_add_to_free_list (vm, fl, bi, 1);
Damjan Marioneac3b112018-03-05 09:36:31 +0100578 }
Damjan Marion8e715292018-09-03 15:41:45 +0200579 bi = next;
Damjan Marioneac3b112018-03-05 09:36:31 +0100580 }
Damjan Marion8e715292018-09-03 15:41:45 +0200581 while (follow_buffer_next && (flags & VLIB_BUFFER_NEXT_PRESENT));
Damjan Marioneac3b112018-03-05 09:36:31 +0100582}
583
584static_always_inline void
Ed Warnickecb9cada2015-12-08 15:45:58 -0700585vlib_buffer_free_inline (vlib_main_t * vm,
Dave Barach9b8ffd92016-07-08 08:13:45 -0400586 u32 * buffers, u32 n_buffers, u32 follow_buffer_next)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700587{
Damjan Mariond50e3472019-01-20 00:03:56 +0100588 vlib_buffer_main_t *bm = vm->buffer_main;
Damjan Marioneac3b112018-03-05 09:36:31 +0100589 vlib_buffer_t *p, *b0, *b1, *b2, *b3;
590 int i = 0;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400591 u32 (*cb) (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
592 u32 follow_buffer_next);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700593
Ed Warnickecb9cada2015-12-08 15:45:58 -0700594 cb = bm->buffer_free_callback;
595
596 if (PREDICT_FALSE (cb != 0))
Dave Barach9b8ffd92016-07-08 08:13:45 -0400597 n_buffers = (*cb) (vm, buffers, n_buffers, follow_buffer_next);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700598
Dave Barach9b8ffd92016-07-08 08:13:45 -0400599 if (!n_buffers)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700600 return;
601
Damjan Marioneac3b112018-03-05 09:36:31 +0100602 while (i + 11 < n_buffers)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700603 {
Damjan Marioneac3b112018-03-05 09:36:31 +0100604 p = vlib_get_buffer (vm, buffers[i + 8]);
605 vlib_prefetch_buffer_header (p, LOAD);
606 p = vlib_get_buffer (vm, buffers[i + 9]);
607 vlib_prefetch_buffer_header (p, LOAD);
608 p = vlib_get_buffer (vm, buffers[i + 10]);
609 vlib_prefetch_buffer_header (p, LOAD);
610 p = vlib_get_buffer (vm, buffers[i + 11]);
611 vlib_prefetch_buffer_header (p, LOAD);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700612
Damjan Marioneac3b112018-03-05 09:36:31 +0100613 b0 = vlib_get_buffer (vm, buffers[i]);
614 b1 = vlib_get_buffer (vm, buffers[i + 1]);
615 b2 = vlib_get_buffer (vm, buffers[i + 2]);
616 b3 = vlib_get_buffer (vm, buffers[i + 3]);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700617
Damjan Marioneac3b112018-03-05 09:36:31 +0100618 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
619 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b1);
620 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b2);
621 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b3);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700622
Damjan Marioneac3b112018-03-05 09:36:31 +0100623 recycle_or_free (vm, bm, buffers[i], b0, follow_buffer_next);
624 recycle_or_free (vm, bm, buffers[i + 1], b1, follow_buffer_next);
625 recycle_or_free (vm, bm, buffers[i + 2], b2, follow_buffer_next);
626 recycle_or_free (vm, bm, buffers[i + 3], b3, follow_buffer_next);
Damjan Marionc47ed032017-01-25 14:18:03 +0100627
Damjan Marioneac3b112018-03-05 09:36:31 +0100628 i += 4;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700629 }
Damjan Marioneac3b112018-03-05 09:36:31 +0100630
631 while (i < n_buffers)
632 {
633 b0 = vlib_get_buffer (vm, buffers[i]);
634 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
635 recycle_or_free (vm, bm, buffers[i], b0, follow_buffer_next);
636 i++;
637 }
Ed Warnickecb9cada2015-12-08 15:45:58 -0700638}
639
Damjan Marion878c6092017-01-04 13:19:27 +0100640static void
641vlib_buffer_free_internal (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700642{
Dave Barach9b8ffd92016-07-08 08:13:45 -0400643 vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */
644 1);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700645}
646
Damjan Marion878c6092017-01-04 13:19:27 +0100647static void
648vlib_buffer_free_no_next_internal (vlib_main_t * vm, u32 * buffers,
649 u32 n_buffers)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700650{
Dave Barach9b8ffd92016-07-08 08:13:45 -0400651 vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */
652 0);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700653}
654
Dave Barach9b8ffd92016-07-08 08:13:45 -0400655void
656vlib_packet_template_init (vlib_main_t * vm,
657 vlib_packet_template_t * t,
658 void *packet_data,
659 uword n_packet_data_bytes,
Damjan Mariond1274cb2018-03-13 21:32:17 +0100660 uword min_n_buffers_each_alloc, char *fmt, ...)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700661{
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100662 va_list va;
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100663
664 va_start (va, fmt);
Damjan Marion671e60e2018-12-30 18:09:59 +0100665 t->name = va_format (0, fmt, &va);
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100666 va_end (va);
667
668 vlib_worker_thread_barrier_sync (vm);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700669
Dave Barachb7b92992018-10-17 10:38:51 -0400670 clib_memset (t, 0, sizeof (t[0]));
Ed Warnickecb9cada2015-12-08 15:45:58 -0700671
672 vec_add (t->packet_data, packet_data, n_packet_data_bytes);
Damjan Mariond1274cb2018-03-13 21:32:17 +0100673 t->min_n_buffers_each_alloc = min_n_buffers_each_alloc;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700674
Damjan Marion878c6092017-01-04 13:19:27 +0100675 vlib_worker_thread_barrier_release (vm);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700676}
677
678void *
Dave Barach9b8ffd92016-07-08 08:13:45 -0400679vlib_packet_template_get_packet (vlib_main_t * vm,
680 vlib_packet_template_t * t, u32 * bi_result)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700681{
682 u32 bi;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400683 vlib_buffer_t *b;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700684
685 if (vlib_buffer_alloc (vm, &bi, 1) != 1)
686 return 0;
687
688 *bi_result = bi;
689
690 b = vlib_get_buffer (vm, bi);
Dave Barach178cf492018-11-13 16:34:13 -0500691 clib_memcpy_fast (vlib_buffer_get_current (b),
692 t->packet_data, vec_len (t->packet_data));
Dave Barach9b8ffd92016-07-08 08:13:45 -0400693 b->current_length = vec_len (t->packet_data);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700694
695 return b->data;
696}
697
Ed Warnickecb9cada2015-12-08 15:45:58 -0700698/* Append given data to end of buffer, possibly allocating new buffers. */
Dave Barach3a63fc52019-01-07 09:15:47 -0500699int
Damjan Marionab9b7ec2019-01-18 20:24:44 +0100700vlib_buffer_add_data (vlib_main_t * vm, u32 * buffer_index, void *data,
701 u32 n_data_bytes)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700702{
703 u32 n_buffer_bytes, n_left, n_left_this_buffer, bi;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400704 vlib_buffer_t *b;
705 void *d;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700706
Dave Barach3a63fc52019-01-07 09:15:47 -0500707 bi = *buffer_index;
Damjan Marionab9b7ec2019-01-18 20:24:44 +0100708 if (bi == ~0 && 1 != vlib_buffer_alloc (vm, &bi, 1))
Ed Warnickecb9cada2015-12-08 15:45:58 -0700709 goto out_of_buffers;
710
711 d = data;
712 n_left = n_data_bytes;
Damjan Marionab9b7ec2019-01-18 20:24:44 +0100713 n_buffer_bytes = VLIB_BUFFER_DATA_SIZE;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400714
Ed Warnickecb9cada2015-12-08 15:45:58 -0700715 b = vlib_get_buffer (vm, bi);
716 b->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
717
Dave Barach9b8ffd92016-07-08 08:13:45 -0400718 /* Get to the end of the chain before we try to append data... */
Ed Warnickecb9cada2015-12-08 15:45:58 -0700719 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
720 b = vlib_get_buffer (vm, b->next_buffer);
721
722 while (1)
723 {
724 u32 n;
725
726 ASSERT (n_buffer_bytes >= b->current_length);
Dave Barach9b8ffd92016-07-08 08:13:45 -0400727 n_left_this_buffer =
728 n_buffer_bytes - (b->current_data + b->current_length);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700729 n = clib_min (n_left_this_buffer, n_left);
Dave Barach178cf492018-11-13 16:34:13 -0500730 clib_memcpy_fast (vlib_buffer_get_current (b) + b->current_length, d,
731 n);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700732 b->current_length += n;
733 n_left -= n;
734 if (n_left == 0)
735 break;
736
737 d += n;
Damjan Marionab9b7ec2019-01-18 20:24:44 +0100738 if (1 != vlib_buffer_alloc (vm, &b->next_buffer, 1))
Ed Warnickecb9cada2015-12-08 15:45:58 -0700739 goto out_of_buffers;
740
741 b->flags |= VLIB_BUFFER_NEXT_PRESENT;
742
743 b = vlib_get_buffer (vm, b->next_buffer);
744 }
745
Dave Barach3a63fc52019-01-07 09:15:47 -0500746 *buffer_index = bi;
747 return 0;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700748
Dave Barach9b8ffd92016-07-08 08:13:45 -0400749out_of_buffers:
Dave Barach3a63fc52019-01-07 09:15:47 -0500750 clib_warning ("out of buffers");
751 return 1;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700752}
753
Pierre Pfister328e99b2016-02-12 13:18:42 +0000754u16
Dave Barach9b8ffd92016-07-08 08:13:45 -0400755vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm,
Damjan Mariondac03522018-02-01 15:30:13 +0100756 vlib_buffer_free_list_index_t
757 free_list_index,
Dave Barach9b8ffd92016-07-08 08:13:45 -0400758 vlib_buffer_t * first,
Damjan Mariondac03522018-02-01 15:30:13 +0100759 vlib_buffer_t ** last, void *data,
760 u16 data_len)
Dave Barach9b8ffd92016-07-08 08:13:45 -0400761{
Pierre Pfister328e99b2016-02-12 13:18:42 +0000762 vlib_buffer_t *l = *last;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400763 u32 n_buffer_bytes =
764 vlib_buffer_free_list_buffer_size (vm, free_list_index);
Pierre Pfister328e99b2016-02-12 13:18:42 +0000765 u16 copied = 0;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400766 ASSERT (n_buffer_bytes >= l->current_length + l->current_data);
767 while (data_len)
768 {
769 u16 max = n_buffer_bytes - l->current_length - l->current_data;
770 if (max == 0)
771 {
772 if (1 !=
773 vlib_buffer_alloc_from_free_list (vm, &l->next_buffer, 1,
774 free_list_index))
775 return copied;
Eyal Barib688fb12018-11-12 16:13:49 +0200776 *last = l = vlib_buffer_chain_buffer (vm, l, l->next_buffer);
Dave Barach9b8ffd92016-07-08 08:13:45 -0400777 max = n_buffer_bytes - l->current_length - l->current_data;
778 }
Pierre Pfister328e99b2016-02-12 13:18:42 +0000779
Dave Barach9b8ffd92016-07-08 08:13:45 -0400780 u16 len = (data_len > max) ? max : data_len;
Dave Barach178cf492018-11-13 16:34:13 -0500781 clib_memcpy_fast (vlib_buffer_get_current (l) + l->current_length,
782 data + copied, len);
Dave Barach9b8ffd92016-07-08 08:13:45 -0400783 vlib_buffer_chain_increase_length (first, l, len);
784 data_len -= len;
785 copied += len;
786 }
Pierre Pfister328e99b2016-02-12 13:18:42 +0000787 return copied;
788}
789
Damjan Marion149ba772017-10-12 13:09:26 +0200790u8
Damjan Marion68b4da62018-09-30 18:26:20 +0200791vlib_buffer_register_physmem_map (vlib_main_t * vm, u32 physmem_map_index)
Damjan Marion04a7f052017-07-10 15:06:17 +0200792{
Damjan Mariond50e3472019-01-20 00:03:56 +0100793 vlib_buffer_main_t *bm = vm->buffer_main;
Damjan Marion149ba772017-10-12 13:09:26 +0200794 vlib_buffer_pool_t *p;
Damjan Marion68b4da62018-09-30 18:26:20 +0200795 vlib_physmem_map_t *m = vlib_physmem_get_map (vm, physmem_map_index);
796 uword start = pointer_to_uword (m->base);
Damjan Marion8e8d3c82018-10-23 22:54:40 +0200797 uword size = (uword) m->n_pages << m->log2_page_size;
Damjan Marion04a7f052017-07-10 15:06:17 +0200798
799 if (bm->buffer_mem_size == 0)
800 {
801 bm->buffer_mem_start = start;
802 bm->buffer_mem_size = size;
803 }
804 else if (start < bm->buffer_mem_start)
805 {
806 bm->buffer_mem_size += bm->buffer_mem_start - start;
807 bm->buffer_mem_start = start;
808 if (size > bm->buffer_mem_size)
809 bm->buffer_mem_size = size;
810 }
811 else if (start > bm->buffer_mem_start)
812 {
813 uword new_size = start - bm->buffer_mem_start + size;
814 if (new_size > bm->buffer_mem_size)
815 bm->buffer_mem_size = new_size;
816 }
817
818 if ((u64) bm->buffer_mem_size >
819 ((u64) 1 << (32 + CLIB_LOG2_CACHE_LINE_BYTES)))
820 {
821 clib_panic ("buffer memory size out of range!");
822 }
Damjan Marion149ba772017-10-12 13:09:26 +0200823
824 vec_add2 (bm->buffer_pools, p, 1);
825 p->start = start;
826 p->size = size;
Damjan Marion68b4da62018-09-30 18:26:20 +0200827 p->physmem_map_index = physmem_map_index;
Damjan Mariond1274cb2018-03-13 21:32:17 +0100828
Damjan Mariond1274cb2018-03-13 21:32:17 +0100829 ASSERT (p - bm->buffer_pools < 256);
Damjan Marion149ba772017-10-12 13:09:26 +0200830 return p - bm->buffer_pools;
Damjan Marion04a7f052017-07-10 15:06:17 +0200831}
Ed Warnickecb9cada2015-12-08 15:45:58 -0700832
Dave Barach9b8ffd92016-07-08 08:13:45 -0400833static u8 *
834format_vlib_buffer_free_list (u8 * s, va_list * va)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700835{
Dave Barach9b8ffd92016-07-08 08:13:45 -0400836 vlib_buffer_free_list_t *f = va_arg (*va, vlib_buffer_free_list_t *);
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100837 u32 threadnum = va_arg (*va, u32);
838 uword bytes_alloc, bytes_free, n_free, size;
839
840 if (!f)
841 return format (s, "%=7s%=30s%=12s%=12s%=12s%=12s%=12s%=12s",
842 "Thread", "Name", "Index", "Size", "Alloc", "Free",
843 "#Alloc", "#Free");
844
845 size = sizeof (vlib_buffer_t) + f->n_data_bytes;
Damjan Marionbd69a5f2017-02-05 23:44:42 +0100846 n_free = vec_len (f->buffers);
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100847 bytes_alloc = size * f->n_alloc;
848 bytes_free = size * n_free;
849
Chris Luke7447d072017-07-05 12:57:10 -0400850 s = format (s, "%7d%30v%12d%12d%=12U%=12U%=12d%=12d", threadnum,
Ed Warnickecb9cada2015-12-08 15:45:58 -0700851 f->name, f->index, f->n_data_bytes,
852 format_memory_size, bytes_alloc,
Dave Barach9b8ffd92016-07-08 08:13:45 -0400853 format_memory_size, bytes_free, f->n_alloc, n_free);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700854
855 return s;
856}
857
858static clib_error_t *
859show_buffers (vlib_main_t * vm,
Dave Barach9b8ffd92016-07-08 08:13:45 -0400860 unformat_input_t * input, vlib_cli_command_t * cmd)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700861{
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100862 vlib_buffer_free_list_t *f;
863 vlib_main_t *curr_vm;
864 u32 vm_index = 0;
865
866 vlib_cli_output (vm, "%U", format_vlib_buffer_free_list, 0, 0);
867
868 do
869 {
Dave Barach80f54e22017-03-08 19:08:56 -0500870 curr_vm = vlib_mains[vm_index];
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100871
872 /* *INDENT-OFF* */
Damjan Mariond1274cb2018-03-13 21:32:17 +0100873 pool_foreach (f, curr_vm->buffer_free_list_pool, ({
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100874 vlib_cli_output (vm, "%U", format_vlib_buffer_free_list, f, vm_index);
875 }));
876 /* *INDENT-ON* */
877
878 vm_index++;
879 }
880 while (vm_index < vec_len (vlib_mains));
881
Ed Warnickecb9cada2015-12-08 15:45:58 -0700882 return 0;
883}
884
Dave Barach9b8ffd92016-07-08 08:13:45 -0400885/* *INDENT-OFF* */
Ed Warnickecb9cada2015-12-08 15:45:58 -0700886VLIB_CLI_COMMAND (show_buffers_command, static) = {
887 .path = "show buffers",
888 .short_help = "Show packet buffer allocation",
889 .function = show_buffers,
890};
Dave Barach9b8ffd92016-07-08 08:13:45 -0400891/* *INDENT-ON* */
892
Damjan Marion49d66f12017-07-20 18:10:35 +0200893clib_error_t *
894vlib_buffer_main_init (struct vlib_main_t * vm)
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100895{
Damjan Mariond50e3472019-01-20 00:03:56 +0100896 vlib_buffer_main_t *bm;
Damjan Marion49d66f12017-07-20 18:10:35 +0200897 clib_error_t *error;
Damjan Marion68b4da62018-09-30 18:26:20 +0200898 u32 physmem_map_index;
899 u8 pool_index;
Damjan Marion567e61d2018-10-24 17:08:26 +0200900 int log2_page_size = 0;
901
902 buffer_log_default = vlib_log_register_class ("buffer", 0);
Damjan Marion49d66f12017-07-20 18:10:35 +0200903
Damjan Mariond50e3472019-01-20 00:03:56 +0100904 bm = vm->buffer_main = clib_mem_alloc (sizeof (bm[0]));
905 clib_memset (bm, 0, sizeof (bm[0]));
906
Damjan Marion49d66f12017-07-20 18:10:35 +0200907 if (vlib_buffer_callbacks)
908 {
909 /* external plugin has registered own buffer callbacks
910 so we just copy them and quit */
Dave Barach178cf492018-11-13 16:34:13 -0500911 clib_memcpy_fast (&bm->cb, vlib_buffer_callbacks,
912 sizeof (vlib_buffer_callbacks_t));
Damjan Marion49d66f12017-07-20 18:10:35 +0200913 bm->callbacks_registered = 1;
914 return 0;
915 }
Damjan Marion04a7f052017-07-10 15:06:17 +0200916
Damjan Marionc8a26c62017-11-24 20:15:23 +0100917 bm->cb.vlib_buffer_fill_free_list_cb = &vlib_buffer_fill_free_list_internal;
Damjan Marion878c6092017-01-04 13:19:27 +0100918 bm->cb.vlib_buffer_free_cb = &vlib_buffer_free_internal;
919 bm->cb.vlib_buffer_free_no_next_cb = &vlib_buffer_free_no_next_internal;
920 bm->cb.vlib_buffer_delete_free_list_cb =
921 &vlib_buffer_delete_free_list_internal;
Damjan Marion6b0f5892017-07-27 04:01:24 -0400922 clib_spinlock_init (&bm->buffer_known_hash_lockp);
Damjan Marion49d66f12017-07-20 18:10:35 +0200923
Damjan Marion567e61d2018-10-24 17:08:26 +0200924retry:
925 error = vlib_physmem_shared_map_create (vm, "buffers",
926 vlib_buffer_physmem_sz,
927 log2_page_size,
928 CLIB_PMALLOC_NUMA_LOCAL,
929 &physmem_map_index);
930
931 if (error && log2_page_size == 0)
932 {
933 vlib_log_warn (buffer_log_default, "%U", format_clib_error, error);
934 clib_error_free (error);
935 vlib_log_warn (buffer_log_default, "falling back to non-hugepage "
936 "backed buffer pool");
937 log2_page_size = min_log2 (clib_mem_get_page_size ());
938 goto retry;
939 }
940
941 if (error)
Damjan Marion68b4da62018-09-30 18:26:20 +0200942 return error;
Damjan Marion49d66f12017-07-20 18:10:35 +0200943
Damjan Marion68b4da62018-09-30 18:26:20 +0200944 pool_index = vlib_buffer_register_physmem_map (vm, physmem_map_index);
Damjan Mariond50e3472019-01-20 00:03:56 +0100945 vlib_buffer_pool_t *bp = vlib_buffer_pool_get (vm, pool_index);
Damjan Marion68b4da62018-09-30 18:26:20 +0200946 clib_spinlock_init (&bp->lock);
947 bp->buffer_size = VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES +
948 sizeof (vlib_buffer_t);
Damjan Marion49d66f12017-07-20 18:10:35 +0200949
Damjan Marion68b4da62018-09-30 18:26:20 +0200950 return 0;
Damjan Marion878c6092017-01-04 13:19:27 +0100951}
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100952
Damjan Marion49d66f12017-07-20 18:10:35 +0200953static clib_error_t *
954vlib_buffers_configure (vlib_main_t * vm, unformat_input_t * input)
955{
956 u32 size_in_mb;
957
958 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
959 {
960 if (unformat (input, "memory-size-in-mb %d", &size_in_mb))
961 vlib_buffer_physmem_sz = size_in_mb << 20;
962 else
963 return unformat_parse_error (input);
964 }
965
966 unformat_free (input);
967 return 0;
968}
969
970VLIB_EARLY_CONFIG_FUNCTION (vlib_buffers_configure, "buffers");
971
972
Chris Luked4024f52016-09-06 09:32:36 -0400973/** @endcond */
Dave Barach9b8ffd92016-07-08 08:13:45 -0400974/*
975 * fd.io coding-style-patch-verification: ON
976 *
977 * Local Variables:
978 * eval: (c-set-style "gnu")
979 * End:
980 */