blob: 4bf6d125b2116b7f3c7045fef1ca7d95fb3565c2 [file] [log] [blame]
Ed Warnickecb9cada2015-12-08 15:45:58 -07001/*
2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15/*
16 * buffer.c: allocate/free network buffers.
17 *
18 * Copyright (c) 2008 Eliot Dresselhaus
19 *
20 * Permission is hereby granted, free of charge, to any person obtaining
21 * a copy of this software and associated documentation files (the
22 * "Software"), to deal in the Software without restriction, including
23 * without limitation the rights to use, copy, modify, merge, publish,
24 * distribute, sublicense, and/or sell copies of the Software, and to
25 * permit persons to whom the Software is furnished to do so, subject to
26 * the following conditions:
27 *
28 * The above copyright notice and this permission notice shall be
29 * included in all copies or substantial portions of the Software.
30 *
31 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38 */
39
Chris Luked4024f52016-09-06 09:32:36 -040040/**
41 * @cond (!DPDK)
42 * @file
43 *
44 * Allocate/free network buffers.
45 */
46
Damjan Marion1cd8f3c2016-11-24 02:07:32 +010047#if DPDK > 0
48#include <rte_config.h>
49
50#include <rte_common.h>
51#include <rte_log.h>
52#include <rte_memory.h>
53#include <rte_memzone.h>
54#include <rte_tailq.h>
55#include <rte_eal.h>
56#include <rte_per_lcore.h>
57#include <rte_launch.h>
58#include <rte_atomic.h>
59#include <rte_cycles.h>
60#include <rte_prefetch.h>
61#include <rte_lcore.h>
62#include <rte_per_lcore.h>
63#include <rte_branch_prediction.h>
64#include <rte_interrupts.h>
65#include <rte_pci.h>
66#include <rte_random.h>
67#include <rte_debug.h>
68#include <rte_ether.h>
69#include <rte_ethdev.h>
70#include <rte_ring.h>
71#include <rte_mempool.h>
72#include <rte_mbuf.h>
73#include <rte_version.h>
74#endif
75
Ed Warnickecb9cada2015-12-08 15:45:58 -070076#include <vlib/vlib.h>
77
Damjan Marion1cd8f3c2016-11-24 02:07:32 +010078#if DPDK > 0
79#pragma weak rte_mem_virt2phy
80#pragma weak rte_eal_has_hugepages
81#pragma weak rte_socket_id
82#pragma weak rte_pktmbuf_pool_create
83#endif
84
Dave Barach9b8ffd92016-07-08 08:13:45 -040085uword
86vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm,
87 vlib_buffer_t * b_first)
Ed Warnickecb9cada2015-12-08 15:45:58 -070088{
Dave Barach9b8ffd92016-07-08 08:13:45 -040089 vlib_buffer_t *b = b_first;
Ed Warnickecb9cada2015-12-08 15:45:58 -070090 uword l_first = b_first->current_length;
91 uword l = 0;
92 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
93 {
94 b = vlib_get_buffer (vm, b->next_buffer);
95 l += b->current_length;
96 }
97 b_first->total_length_not_including_first_buffer = l;
98 b_first->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
99 return l + l_first;
100}
101
Dave Barach9b8ffd92016-07-08 08:13:45 -0400102u8 *
103format_vlib_buffer (u8 * s, va_list * args)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700104{
Dave Barach9b8ffd92016-07-08 08:13:45 -0400105 vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100106#if DPDK > 0
107 uword indent = format_get_indent (s);
108
109 s = format (s, "current data %d, length %d, free-list %d",
110 b->current_data, b->current_length, b->free_list_index);
111
112 if (b->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID)
113 s = format (s, ", totlen-nifb %d",
114 b->total_length_not_including_first_buffer);
115
116 if (b->flags & VLIB_BUFFER_IS_TRACED)
117 s = format (s, ", trace 0x%x", b->trace_index);
118
119 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
120 {
121 vlib_main_t *vm = vlib_get_main ();
122 u32 next_buffer = b->next_buffer;
123 b = vlib_get_buffer (vm, next_buffer);
124
125 s = format (s, "\n%Unext-buffer 0x%x, segment length %d",
126 format_white_space, indent, next_buffer, b->current_length);
127 }
128
129#else
Dave Barach9b8ffd92016-07-08 08:13:45 -0400130
Ed Warnickecb9cada2015-12-08 15:45:58 -0700131 s = format (s, "current data %d, length %d, free-list %d",
Dave Barach9b8ffd92016-07-08 08:13:45 -0400132 b->current_data, b->current_length, b->free_list_index);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700133
134 if (b->flags & VLIB_BUFFER_IS_TRACED)
135 s = format (s, ", trace 0x%x", b->trace_index);
136
137 if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
138 s = format (s, ", next-buffer 0x%x", b->next_buffer);
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100139#endif
Ed Warnickecb9cada2015-12-08 15:45:58 -0700140
141 return s;
142}
143
Dave Barach9b8ffd92016-07-08 08:13:45 -0400144u8 *
145format_vlib_buffer_and_data (u8 * s, va_list * args)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700146{
Dave Barach9b8ffd92016-07-08 08:13:45 -0400147 vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
148
Ed Warnickecb9cada2015-12-08 15:45:58 -0700149 s = format (s, "%U, %U",
150 format_vlib_buffer, b,
151 format_hex_bytes, vlib_buffer_get_current (b), 64);
152
153 return s;
154}
155
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100156#if DPDK == 0
Dave Barach9b8ffd92016-07-08 08:13:45 -0400157static u8 *
158format_vlib_buffer_known_state (u8 * s, va_list * args)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700159{
160 vlib_buffer_known_state_t state = va_arg (*args, vlib_buffer_known_state_t);
Dave Barach9b8ffd92016-07-08 08:13:45 -0400161 char *t;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700162
163 switch (state)
164 {
165 case VLIB_BUFFER_UNKNOWN:
166 t = "unknown";
167 break;
168
169 case VLIB_BUFFER_KNOWN_ALLOCATED:
170 t = "known-allocated";
171 break;
172
173 case VLIB_BUFFER_KNOWN_FREE:
174 t = "known-free";
175 break;
176
177 default:
178 t = "invalid";
179 break;
180 }
181
182 return format (s, "%s", t);
183}
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100184#endif
Ed Warnickecb9cada2015-12-08 15:45:58 -0700185
Dave Barach9b8ffd92016-07-08 08:13:45 -0400186u8 *
187format_vlib_buffer_contents (u8 * s, va_list * va)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700188{
Dave Barach9b8ffd92016-07-08 08:13:45 -0400189 vlib_main_t *vm = va_arg (*va, vlib_main_t *);
190 vlib_buffer_t *b = va_arg (*va, vlib_buffer_t *);
191
Ed Warnickecb9cada2015-12-08 15:45:58 -0700192 while (1)
193 {
Dave Barach9b8ffd92016-07-08 08:13:45 -0400194 vec_add (s, vlib_buffer_get_current (b), b->current_length);
195 if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))
Ed Warnickecb9cada2015-12-08 15:45:58 -0700196 break;
197 b = vlib_get_buffer (vm, b->next_buffer);
198 }
199
200 return s;
201}
202
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100203#if DPDK == 0
Ed Warnickecb9cada2015-12-08 15:45:58 -0700204static u8 *
205vlib_validate_buffer_helper (vlib_main_t * vm,
206 u32 bi,
Dave Barach9b8ffd92016-07-08 08:13:45 -0400207 uword follow_buffer_next, uword ** unique_hash)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700208{
Dave Barach9b8ffd92016-07-08 08:13:45 -0400209 vlib_buffer_t *b = vlib_get_buffer (vm, bi);
210 vlib_buffer_main_t *bm = vm->buffer_main;
211 vlib_buffer_free_list_t *fl;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700212
Dave Barach9b8ffd92016-07-08 08:13:45 -0400213 if (pool_is_free_index (bm->buffer_free_list_pool, b->free_list_index))
Ed Warnickecb9cada2015-12-08 15:45:58 -0700214 return format (0, "unknown free list 0x%x", b->free_list_index);
215
Dave Barach9b8ffd92016-07-08 08:13:45 -0400216 fl = pool_elt_at_index (bm->buffer_free_list_pool, b->free_list_index);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700217
Dave Barach9b8ffd92016-07-08 08:13:45 -0400218 if ((signed) b->current_data < (signed) -VLIB_BUFFER_PRE_DATA_SIZE)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700219 return format (0, "current data %d before pre-data", b->current_data);
220#if DPDK == 0
221 if (b->current_data + b->current_length > fl->n_data_bytes)
222 return format (0, "%d-%d beyond end of buffer %d",
Dave Barach9b8ffd92016-07-08 08:13:45 -0400223 b->current_data, b->current_length, fl->n_data_bytes);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700224#endif
225
Dave Barach9b8ffd92016-07-08 08:13:45 -0400226 if (follow_buffer_next && (b->flags & VLIB_BUFFER_NEXT_PRESENT))
Ed Warnickecb9cada2015-12-08 15:45:58 -0700227 {
228 vlib_buffer_known_state_t k;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400229 u8 *msg, *result;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700230
231 k = vlib_buffer_is_known (vm, b->next_buffer);
232 if (k != VLIB_BUFFER_KNOWN_ALLOCATED)
233 return format (0, "next 0x%x: %U",
Dave Barach9b8ffd92016-07-08 08:13:45 -0400234 b->next_buffer, format_vlib_buffer_known_state, k);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700235
236 if (unique_hash)
237 {
238 if (hash_get (*unique_hash, b->next_buffer))
239 return format (0, "duplicate buffer 0x%x", b->next_buffer);
240
241 hash_set1 (*unique_hash, b->next_buffer);
242 }
243
244 msg = vlib_validate_buffer (vm, b->next_buffer, follow_buffer_next);
245 if (msg)
246 {
247 result = format (0, "next 0x%x: %v", b->next_buffer, msg);
248 vec_free (msg);
249 return result;
250 }
251 }
252
253 return 0;
254}
255
256u8 *
257vlib_validate_buffer (vlib_main_t * vm, u32 bi, uword follow_buffer_next)
Dave Barach9b8ffd92016-07-08 08:13:45 -0400258{
259 return vlib_validate_buffer_helper (vm, bi, follow_buffer_next,
260 /* unique_hash */ 0);
261}
Ed Warnickecb9cada2015-12-08 15:45:58 -0700262
263u8 *
264vlib_validate_buffers (vlib_main_t * vm,
265 u32 * buffers,
266 uword next_buffer_stride,
267 uword n_buffers,
268 vlib_buffer_known_state_t known_state,
269 uword follow_buffer_next)
270{
Dave Barach9b8ffd92016-07-08 08:13:45 -0400271 uword i, *hash;
272 u32 bi, *b = buffers;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700273 vlib_buffer_known_state_t k;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400274 u8 *msg = 0, *result = 0;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700275
276 hash = hash_create (0, 0);
277 for (i = 0; i < n_buffers; i++)
278 {
279 bi = b[0];
280 b += next_buffer_stride;
281
282 /* Buffer is not unique. */
283 if (hash_get (hash, bi))
284 {
285 msg = format (0, "not unique");
286 goto done;
287 }
288
289 k = vlib_buffer_is_known (vm, bi);
290 if (k != known_state)
291 {
292 msg = format (0, "is %U; expected %U",
293 format_vlib_buffer_known_state, k,
294 format_vlib_buffer_known_state, known_state);
295 goto done;
296 }
297
298 msg = vlib_validate_buffer_helper (vm, bi, follow_buffer_next, &hash);
299 if (msg)
300 goto done;
301
302 hash_set1 (hash, bi);
303 }
304
Dave Barach9b8ffd92016-07-08 08:13:45 -0400305done:
Ed Warnickecb9cada2015-12-08 15:45:58 -0700306 if (msg)
307 {
308 result = format (0, "0x%x: %v", bi, msg);
309 vec_free (msg);
310 }
311 hash_free (hash);
312 return result;
313}
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100314#endif
Ed Warnickecb9cada2015-12-08 15:45:58 -0700315
316vlib_main_t **vlib_mains;
317
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100318#if DPDK == 0
Ed Warnickecb9cada2015-12-08 15:45:58 -0700319/* When dubugging validate that given buffers are either known allocated
320 or known free. */
321static void
322vlib_buffer_validate_alloc_free (vlib_main_t * vm,
323 u32 * buffers,
324 uword n_buffers,
325 vlib_buffer_known_state_t expected_state)
326{
Dave Barach9b8ffd92016-07-08 08:13:45 -0400327 u32 *b;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700328 uword i, bi, is_free;
329
330 if (CLIB_DEBUG == 0)
331 return;
332
Dave Barach9b8ffd92016-07-08 08:13:45 -0400333 ASSERT (os_get_cpu_number () == 0);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700334
335 /* smp disaster check */
336 if (vlib_mains)
Dave Barach9b8ffd92016-07-08 08:13:45 -0400337 ASSERT (vm == vlib_mains[0]);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700338
339 is_free = expected_state == VLIB_BUFFER_KNOWN_ALLOCATED;
340 b = buffers;
341 for (i = 0; i < n_buffers; i++)
342 {
343 vlib_buffer_known_state_t known;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400344
Ed Warnickecb9cada2015-12-08 15:45:58 -0700345 bi = b[0];
346 b += 1;
347 known = vlib_buffer_is_known (vm, bi);
348 if (known != expected_state)
349 {
350 ASSERT (0);
351 vlib_panic_with_msg
352 (vm, "%s %U buffer 0x%x",
353 is_free ? "freeing" : "allocating",
Dave Barach9b8ffd92016-07-08 08:13:45 -0400354 format_vlib_buffer_known_state, known, bi);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700355 }
356
357 vlib_buffer_set_known_state
358 (vm, bi,
359 is_free ? VLIB_BUFFER_KNOWN_FREE : VLIB_BUFFER_KNOWN_ALLOCATED);
360 }
361}
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100362#endif
Ed Warnickecb9cada2015-12-08 15:45:58 -0700363
364#define BUFFERS_PER_COPY (sizeof (vlib_copy_unit_t) / sizeof (u32))
365
366/* Make sure we have at least given number of unaligned buffers. */
367static void
368fill_unaligned (vlib_main_t * vm,
369 vlib_buffer_free_list_t * free_list,
370 uword n_unaligned_buffers)
371{
372 word la = vec_len (free_list->aligned_buffers);
373 word lu = vec_len (free_list->unaligned_buffers);
374
375 /* Aligned come in aligned copy-sized chunks. */
376 ASSERT (la % BUFFERS_PER_COPY == 0);
377
378 ASSERT (la >= n_unaligned_buffers);
379
380 while (lu < n_unaligned_buffers)
381 {
382 /* Copy 4 buffers from end of aligned vector to unaligned vector. */
383 vec_add (free_list->unaligned_buffers,
384 free_list->aligned_buffers + la - BUFFERS_PER_COPY,
385 BUFFERS_PER_COPY);
386 la -= BUFFERS_PER_COPY;
387 lu += BUFFERS_PER_COPY;
388 }
389 _vec_len (free_list->aligned_buffers) = la;
390}
391
392/* After free aligned buffers may not contain even sized chunks. */
393static void
394trim_aligned (vlib_buffer_free_list_t * f)
395{
396 uword l, n_trim;
397
398 /* Add unaligned to aligned before trim. */
399 l = vec_len (f->unaligned_buffers);
400 if (l > 0)
401 {
402 vec_add_aligned (f->aligned_buffers, f->unaligned_buffers, l,
403 /* align */ sizeof (vlib_copy_unit_t));
404
405 _vec_len (f->unaligned_buffers) = 0;
406 }
407
408 /* Remove unaligned buffers from end of aligned vector and save for next trim. */
409 l = vec_len (f->aligned_buffers);
410 n_trim = l % BUFFERS_PER_COPY;
411 if (n_trim)
412 {
413 /* Trim aligned -> unaligned. */
414 vec_add (f->unaligned_buffers, f->aligned_buffers + l - n_trim, n_trim);
415
416 /* Remove from aligned. */
417 _vec_len (f->aligned_buffers) = l - n_trim;
418 }
419}
420
421static void
422merge_free_lists (vlib_buffer_free_list_t * dst,
423 vlib_buffer_free_list_t * src)
424{
425 uword l;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400426 u32 *d;
427
Ed Warnickecb9cada2015-12-08 15:45:58 -0700428 trim_aligned (src);
429 trim_aligned (dst);
430
431 l = vec_len (src->aligned_buffers);
432 if (l > 0)
433 {
434 vec_add2_aligned (dst->aligned_buffers, d, l,
435 /* align */ sizeof (vlib_copy_unit_t));
Damjan Mariona5e7c292016-11-24 01:28:45 +0100436 clib_memcpy (d, src->aligned_buffers, l * sizeof (d[0]));
Ed Warnickecb9cada2015-12-08 15:45:58 -0700437 vec_free (src->aligned_buffers);
438 }
439
440 l = vec_len (src->unaligned_buffers);
441 if (l > 0)
442 {
443 vec_add (dst->unaligned_buffers, src->unaligned_buffers, l);
444 vec_free (src->unaligned_buffers);
445 }
446}
447
448always_inline u32
449vlib_buffer_get_free_list_with_size (vlib_main_t * vm, u32 size)
450{
Dave Barach9b8ffd92016-07-08 08:13:45 -0400451 vlib_buffer_main_t *bm = vm->buffer_main;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700452
453 size = vlib_buffer_round_size (size);
Dave Barach9b8ffd92016-07-08 08:13:45 -0400454 uword *p = hash_get (bm->free_list_by_size, size);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700455 return p ? p[0] : ~0;
456}
457
458/* Add buffer free list. */
459static u32
460vlib_buffer_create_free_list_helper (vlib_main_t * vm,
461 u32 n_data_bytes,
Dave Barach9b8ffd92016-07-08 08:13:45 -0400462 u32 is_public, u32 is_default, u8 * name)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700463{
Dave Barach9b8ffd92016-07-08 08:13:45 -0400464 vlib_buffer_main_t *bm = vm->buffer_main;
465 vlib_buffer_free_list_t *f;
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100466#if DPDK > 0
467 int i;
468
469 ASSERT (os_get_cpu_number () == 0);
470
471 if (!is_default && pool_elts (bm->buffer_free_list_pool) == 0)
472 {
473 u32 default_free_free_list_index;
474
475 /* *INDENT-OFF* */
476 default_free_free_list_index =
477 vlib_buffer_create_free_list_helper
478 (vm,
479 /* default buffer size */ VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES,
480 /* is_public */ 1,
481 /* is_default */ 1,
482 (u8 *) "default");
483 /* *INDENT-ON* */
484 ASSERT (default_free_free_list_index ==
485 VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
486
487 if (n_data_bytes == VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES && is_public)
488 return default_free_free_list_index;
489 }
490
491 pool_get_aligned (bm->buffer_free_list_pool, f, CLIB_CACHE_LINE_BYTES);
492
493 memset (f, 0, sizeof (f[0]));
494 f->index = f - bm->buffer_free_list_pool;
495 f->n_data_bytes = vlib_buffer_round_size (n_data_bytes);
496 f->min_n_buffers_each_physmem_alloc = 16;
497 f->name = clib_mem_is_heap_object (name) ? name : format (0, "%s", name);
498
499 /* Setup free buffer template. */
500 f->buffer_init_template.free_list_index = f->index;
501
502 if (is_public)
503 {
504 uword *p = hash_get (bm->free_list_by_size, f->n_data_bytes);
505 if (!p)
506 hash_set (bm->free_list_by_size, f->n_data_bytes, f->index);
507 }
508
509 for (i = 1; i < vec_len (vlib_mains); i++)
510 {
511 vlib_buffer_main_t *wbm = vlib_mains[i]->buffer_main;
512 vlib_buffer_free_list_t *wf;
513 pool_get_aligned (wbm->buffer_free_list_pool,
514 wf, CLIB_CACHE_LINE_BYTES);
515 ASSERT (f - bm->buffer_free_list_pool ==
516 wf - wbm->buffer_free_list_pool);
517 wf[0] = f[0];
518 wf->aligned_buffers = 0;
519 wf->unaligned_buffers = 0;
520 wf->n_alloc = 0;
521 }
522#else
Ed Warnickecb9cada2015-12-08 15:45:58 -0700523
Dave Barach9b8ffd92016-07-08 08:13:45 -0400524 if (!is_default && pool_elts (bm->buffer_free_list_pool) == 0)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700525 {
526 u32 default_free_free_list_index;
527
Dave Barach9b8ffd92016-07-08 08:13:45 -0400528 default_free_free_list_index = vlib_buffer_create_free_list_helper (vm,
529 /* default buffer size */
530 VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES,
531 /* is_public */
532 1,
533 /* is_default */
534 1,
535 (u8
536 *)
537 "default");
538 ASSERT (default_free_free_list_index ==
539 VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700540
541 if (n_data_bytes == VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES && is_public)
542 return default_free_free_list_index;
543 }
544
545 pool_get_aligned (bm->buffer_free_list_pool, f, CLIB_CACHE_LINE_BYTES);
546
547 memset (f, 0, sizeof (f[0]));
548 f->index = f - bm->buffer_free_list_pool;
549 f->n_data_bytes = vlib_buffer_round_size (n_data_bytes);
550 f->min_n_buffers_each_physmem_alloc = 256;
551 f->name = clib_mem_is_heap_object (name) ? name : format (0, "%s", name);
552
553 /* Setup free buffer template. */
554 f->buffer_init_template.free_list_index = f->index;
555
556 if (is_public)
557 {
Dave Barach9b8ffd92016-07-08 08:13:45 -0400558 uword *p = hash_get (bm->free_list_by_size, f->n_data_bytes);
559 if (!p)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700560 hash_set (bm->free_list_by_size, f->n_data_bytes, f->index);
561 }
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100562#endif
Ed Warnickecb9cada2015-12-08 15:45:58 -0700563
564 return f->index;
565}
566
Dave Barach9b8ffd92016-07-08 08:13:45 -0400567u32
568vlib_buffer_create_free_list (vlib_main_t * vm, u32 n_data_bytes,
569 char *fmt, ...)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700570{
571 va_list va;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400572 u8 *name;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700573
574 va_start (va, fmt);
575 name = va_format (0, fmt, &va);
576 va_end (va);
577
578 return vlib_buffer_create_free_list_helper (vm, n_data_bytes,
579 /* is_public */ 0,
580 /* is_default */ 0,
581 name);
582}
583
Dave Barach9b8ffd92016-07-08 08:13:45 -0400584u32
585vlib_buffer_get_or_create_free_list (vlib_main_t * vm, u32 n_data_bytes,
586 char *fmt, ...)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700587{
588 u32 i = vlib_buffer_get_free_list_with_size (vm, n_data_bytes);
589
590 if (i == ~0)
591 {
592 va_list va;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400593 u8 *name;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700594
595 va_start (va, fmt);
596 name = va_format (0, fmt, &va);
597 va_end (va);
598
599 i = vlib_buffer_create_free_list_helper (vm, n_data_bytes,
600 /* is_public */ 1,
601 /* is_default */ 0,
602 name);
603 }
Dave Barach9b8ffd92016-07-08 08:13:45 -0400604
Ed Warnickecb9cada2015-12-08 15:45:58 -0700605 return i;
606}
607
608static void
609del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f)
610{
611 u32 i;
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100612#if DPDK > 0
613 struct rte_mbuf *mb;
614 vlib_buffer_t *b;
615
616 for (i = 0; i < vec_len (f->unaligned_buffers); i++)
617 {
618 b = vlib_get_buffer (vm, f->unaligned_buffers[i]);
619 mb = rte_mbuf_from_vlib_buffer (b);
620 ASSERT (rte_mbuf_refcnt_read (mb) == 1);
621 rte_pktmbuf_free (mb);
622 }
623 for (i = 0; i < vec_len (f->aligned_buffers); i++)
624 {
625 b = vlib_get_buffer (vm, f->aligned_buffers[i]);
626 mb = rte_mbuf_from_vlib_buffer (b);
627 ASSERT (rte_mbuf_refcnt_read (mb) == 1);
628 rte_pktmbuf_free (mb);
629 }
630 vec_free (f->name);
631#else
Ed Warnickecb9cada2015-12-08 15:45:58 -0700632
633 for (i = 0; i < vec_len (f->buffer_memory_allocated); i++)
634 vm->os_physmem_free (f->buffer_memory_allocated[i]);
635 vec_free (f->name);
636 vec_free (f->buffer_memory_allocated);
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100637#endif
Ed Warnickecb9cada2015-12-08 15:45:58 -0700638 vec_free (f->unaligned_buffers);
639 vec_free (f->aligned_buffers);
640}
641
642/* Add buffer free list. */
Dave Barach9b8ffd92016-07-08 08:13:45 -0400643void
644vlib_buffer_delete_free_list (vlib_main_t * vm, u32 free_list_index)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700645{
Dave Barach9b8ffd92016-07-08 08:13:45 -0400646 vlib_buffer_main_t *bm = vm->buffer_main;
647 vlib_buffer_free_list_t *f;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700648 u32 merge_index;
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100649#if DPDK > 0
650 int i;
651
652 ASSERT (os_get_cpu_number () == 0);
653
654 f = vlib_buffer_get_free_list (vm, free_list_index);
655
656 merge_index = vlib_buffer_get_free_list_with_size (vm, f->n_data_bytes);
657 if (merge_index != ~0 && merge_index != free_list_index)
658 {
659 merge_free_lists (pool_elt_at_index (bm->buffer_free_list_pool,
660 merge_index), f);
661 }
662
663 del_free_list (vm, f);
664
665 /* Poison it. */
666 memset (f, 0xab, sizeof (f[0]));
667
668 pool_put (bm->buffer_free_list_pool, f);
669
670 for (i = 1; i < vec_len (vlib_mains); i++)
671 {
672 bm = vlib_mains[i]->buffer_main;
673 f = vlib_buffer_get_free_list (vlib_mains[i], free_list_index);;
674 memset (f, 0xab, sizeof (f[0]));
675 pool_put (bm->buffer_free_list_pool, f);
676 }
677#else
Ed Warnickecb9cada2015-12-08 15:45:58 -0700678
679 f = vlib_buffer_get_free_list (vm, free_list_index);
680
Dave Barach9b8ffd92016-07-08 08:13:45 -0400681 ASSERT (vec_len (f->unaligned_buffers) + vec_len (f->aligned_buffers) ==
682 f->n_alloc);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700683 merge_index = vlib_buffer_get_free_list_with_size (vm, f->n_data_bytes);
684 if (merge_index != ~0 && merge_index != free_list_index)
685 {
686 merge_free_lists (pool_elt_at_index (bm->buffer_free_list_pool,
687 merge_index), f);
688 }
689
690 del_free_list (vm, f);
691
692 /* Poison it. */
693 memset (f, 0xab, sizeof (f[0]));
694
695 pool_put (bm->buffer_free_list_pool, f);
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100696#endif
Ed Warnickecb9cada2015-12-08 15:45:58 -0700697}
698
699/* Make sure free list has at least given number of free buffers. */
700static uword
701fill_free_list (vlib_main_t * vm,
Dave Barach9b8ffd92016-07-08 08:13:45 -0400702 vlib_buffer_free_list_t * fl, uword min_free_buffers)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700703{
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100704#if DPDK > 0
705 vlib_buffer_t *b;
706 int n, i;
707 u32 bi;
708 u32 n_remaining = 0, n_alloc = 0;
709 unsigned socket_id = rte_socket_id ? rte_socket_id () : 0;
710 struct rte_mempool *rmp = vm->buffer_main->pktmbuf_pools[socket_id];
711 struct rte_mbuf *mb;
712
713 /* Too early? */
714 if (PREDICT_FALSE (rmp == 0))
715 return 0;
716
717 trim_aligned (fl);
718
719 /* Already have enough free buffers on free list? */
720 n = min_free_buffers - vec_len (fl->aligned_buffers);
721 if (n <= 0)
722 return min_free_buffers;
723
724 /* Always allocate round number of buffers. */
725 n = round_pow2 (n, BUFFERS_PER_COPY);
726
727 /* Always allocate new buffers in reasonably large sized chunks. */
728 n = clib_max (n, fl->min_n_buffers_each_physmem_alloc);
729
730 vec_validate (vm->mbuf_alloc_list, n - 1);
731
732 if (rte_mempool_get_bulk (rmp, vm->mbuf_alloc_list, n) < 0)
733 return 0;
734
735 _vec_len (vm->mbuf_alloc_list) = n;
736
737 for (i = 0; i < n; i++)
738 {
739 mb = vm->mbuf_alloc_list[i];
740
741 ASSERT (rte_mbuf_refcnt_read (mb) == 0);
742 rte_mbuf_refcnt_set (mb, 1);
743
744 b = vlib_buffer_from_rte_mbuf (mb);
745 bi = vlib_get_buffer_index (vm, b);
746
747 vec_add1_aligned (fl->aligned_buffers, bi, sizeof (vlib_copy_unit_t));
748 n_alloc++;
749 n_remaining--;
750
751 vlib_buffer_init_for_free_list (b, fl);
752
753 if (fl->buffer_init_function)
754 fl->buffer_init_function (vm, fl, &bi, 1);
755 }
756
757 fl->n_alloc += n;
758
759 return n;
760#else
Dave Barach9b8ffd92016-07-08 08:13:45 -0400761 vlib_buffer_t *buffers, *b;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700762 int n, n_bytes, i;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400763 u32 *bi;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700764 u32 n_remaining, n_alloc, n_this_chunk;
765
766 trim_aligned (fl);
767
768 /* Already have enough free buffers on free list? */
769 n = min_free_buffers - vec_len (fl->aligned_buffers);
770 if (n <= 0)
771 return min_free_buffers;
772
773 /* Always allocate round number of buffers. */
774 n = round_pow2 (n, BUFFERS_PER_COPY);
775
776 /* Always allocate new buffers in reasonably large sized chunks. */
777 n = clib_max (n, fl->min_n_buffers_each_physmem_alloc);
778
779 n_remaining = n;
780 n_alloc = 0;
781 while (n_remaining > 0)
782 {
783 n_this_chunk = clib_min (n_remaining, 16);
Dave Barach9b8ffd92016-07-08 08:13:45 -0400784
Ed Warnickecb9cada2015-12-08 15:45:58 -0700785 n_bytes = n_this_chunk * (sizeof (b[0]) + fl->n_data_bytes);
786
787 /* drb: removed power-of-2 ASSERT */
788 buffers = vm->os_physmem_alloc_aligned (&vm->physmem_main,
Dave Barach9b8ffd92016-07-08 08:13:45 -0400789 n_bytes,
790 sizeof (vlib_buffer_t));
791 if (!buffers)
792 return n_alloc;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700793
794 /* Record chunk as being allocated so we can free it later. */
795 vec_add1 (fl->buffer_memory_allocated, buffers);
796
797 fl->n_alloc += n_this_chunk;
798 n_alloc += n_this_chunk;
799 n_remaining -= n_this_chunk;
800
801 b = buffers;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400802 vec_add2_aligned (fl->aligned_buffers, bi, n_this_chunk,
803 sizeof (vlib_copy_unit_t));
Ed Warnickecb9cada2015-12-08 15:45:58 -0700804 for (i = 0; i < n_this_chunk; i++)
Dave Barach9b8ffd92016-07-08 08:13:45 -0400805 {
806 bi[i] = vlib_get_buffer_index (vm, b);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700807
Dave Barach9b8ffd92016-07-08 08:13:45 -0400808 if (CLIB_DEBUG > 0)
809 vlib_buffer_set_known_state (vm, bi[i], VLIB_BUFFER_KNOWN_FREE);
810 b = vlib_buffer_next_contiguous (b, fl->n_data_bytes);
811 }
Ed Warnickecb9cada2015-12-08 15:45:58 -0700812
813 memset (buffers, 0, n_bytes);
814
815 /* Initialize all new buffers. */
816 b = buffers;
817 for (i = 0; i < n_this_chunk; i++)
Dave Barach9b8ffd92016-07-08 08:13:45 -0400818 {
819 vlib_buffer_init_for_free_list (b, fl);
820 b = vlib_buffer_next_contiguous (b, fl->n_data_bytes);
821 }
822
Ed Warnickecb9cada2015-12-08 15:45:58 -0700823 if (fl->buffer_init_function)
Dave Barach9b8ffd92016-07-08 08:13:45 -0400824 fl->buffer_init_function (vm, fl, bi, n_this_chunk);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700825 }
826 return n_alloc;
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100827#endif
Ed Warnickecb9cada2015-12-08 15:45:58 -0700828}
829
830always_inline uword
831copy_alignment (u32 * x)
Dave Barach9b8ffd92016-07-08 08:13:45 -0400832{
833 return (pointer_to_uword (x) / sizeof (x[0])) % BUFFERS_PER_COPY;
834}
Ed Warnickecb9cada2015-12-08 15:45:58 -0700835
836static u32
837alloc_from_free_list (vlib_main_t * vm,
838 vlib_buffer_free_list_t * free_list,
Dave Barach9b8ffd92016-07-08 08:13:45 -0400839 u32 * alloc_buffers, u32 n_alloc_buffers)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700840{
Dave Barach9b8ffd92016-07-08 08:13:45 -0400841 u32 *dst, *u_src;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700842 uword u_len, n_left;
843 uword n_unaligned_start, n_unaligned_end, n_filled;
844
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100845#if DPDK == 0
Dave Barach9b8ffd92016-07-08 08:13:45 -0400846 ASSERT (os_get_cpu_number () == 0);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700847
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100848#endif
Ed Warnickecb9cada2015-12-08 15:45:58 -0700849 n_left = n_alloc_buffers;
850 dst = alloc_buffers;
851 n_unaligned_start = ((BUFFERS_PER_COPY - copy_alignment (dst))
852 & (BUFFERS_PER_COPY - 1));
853
854 n_filled = fill_free_list (vm, free_list, n_alloc_buffers);
855 if (n_filled == 0)
856 return 0;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400857
Ed Warnickecb9cada2015-12-08 15:45:58 -0700858 n_left = n_filled < n_left ? n_filled : n_left;
859 n_alloc_buffers = n_left;
860
861 if (n_unaligned_start >= n_left)
862 {
863 n_unaligned_start = n_left;
864 n_unaligned_end = 0;
865 }
866 else
867 n_unaligned_end = copy_alignment (dst + n_alloc_buffers);
868
869 fill_unaligned (vm, free_list, n_unaligned_start + n_unaligned_end);
870
871 u_len = vec_len (free_list->unaligned_buffers);
872 u_src = free_list->unaligned_buffers + u_len - 1;
873
874 if (n_unaligned_start)
875 {
876 uword n_copy = n_unaligned_start;
877 if (n_copy > n_left)
878 n_copy = n_left;
879 n_left -= n_copy;
880
881 while (n_copy > 0)
882 {
883 *dst++ = *u_src--;
884 n_copy--;
885 u_len--;
886 }
887
888 /* Now dst should be aligned. */
889 if (n_left > 0)
890 ASSERT (pointer_to_uword (dst) % sizeof (vlib_copy_unit_t) == 0);
891 }
892
893 /* Aligned copy. */
894 {
Dave Barach9b8ffd92016-07-08 08:13:45 -0400895 vlib_copy_unit_t *d, *s;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700896 uword n_copy;
897
Dave Barach9b8ffd92016-07-08 08:13:45 -0400898 if (vec_len (free_list->aligned_buffers) <
899 ((n_left / BUFFERS_PER_COPY) * BUFFERS_PER_COPY))
900 abort ();
Ed Warnickecb9cada2015-12-08 15:45:58 -0700901
902 n_copy = n_left / BUFFERS_PER_COPY;
903 n_left = n_left % BUFFERS_PER_COPY;
904
905 /* Remove buffers from aligned free list. */
906 _vec_len (free_list->aligned_buffers) -= n_copy * BUFFERS_PER_COPY;
907
908 s = (vlib_copy_unit_t *) vec_end (free_list->aligned_buffers);
909 d = (vlib_copy_unit_t *) dst;
910
911 /* Fast path loop. */
912 while (n_copy >= 4)
913 {
914 d[0] = s[0];
915 d[1] = s[1];
916 d[2] = s[2];
917 d[3] = s[3];
918 n_copy -= 4;
919 s += 4;
920 d += 4;
921 }
922
923 while (n_copy >= 1)
924 {
925 d[0] = s[0];
926 n_copy -= 1;
927 s += 1;
928 d += 1;
929 }
930
931 dst = (void *) d;
932 }
933
934 /* Unaligned copy. */
935 ASSERT (n_unaligned_end == n_left);
936 while (n_left > 0)
937 {
938 *dst++ = *u_src--;
939 n_left--;
940 u_len--;
941 }
942
Dave Barach9b8ffd92016-07-08 08:13:45 -0400943 if (!free_list->unaligned_buffers)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700944 ASSERT (u_len == 0);
945 else
946 _vec_len (free_list->unaligned_buffers) = u_len;
947
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100948#if DPDK == 0
Ed Warnickecb9cada2015-12-08 15:45:58 -0700949 /* Verify that buffers are known free. */
950 vlib_buffer_validate_alloc_free (vm, alloc_buffers,
Dave Barach9b8ffd92016-07-08 08:13:45 -0400951 n_alloc_buffers, VLIB_BUFFER_KNOWN_FREE);
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100952#endif
Ed Warnickecb9cada2015-12-08 15:45:58 -0700953
954 return n_alloc_buffers;
955}
956
957/* Allocate a given number of buffers into given array.
958 Returns number actually allocated which will be either zero or
959 number requested. */
Dave Barach9b8ffd92016-07-08 08:13:45 -0400960u32
961vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700962{
Dave Barach9b8ffd92016-07-08 08:13:45 -0400963 vlib_buffer_main_t *bm = vm->buffer_main;
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100964#if DPDK == 0
Dave Barach9b8ffd92016-07-08 08:13:45 -0400965 ASSERT (os_get_cpu_number () == 0);
Damjan Marion1cd8f3c2016-11-24 02:07:32 +0100966#endif
Ed Warnickecb9cada2015-12-08 15:45:58 -0700967
968 return alloc_from_free_list
969 (vm,
970 pool_elt_at_index (bm->buffer_free_list_pool,
971 VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX),
972 buffers, n_buffers);
973}
974
Dave Barach9b8ffd92016-07-08 08:13:45 -0400975u32
976vlib_buffer_alloc_from_free_list (vlib_main_t * vm,
977 u32 * buffers,
978 u32 n_buffers, u32 free_list_index)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700979{
Dave Barach9b8ffd92016-07-08 08:13:45 -0400980 vlib_buffer_main_t *bm = vm->buffer_main;
981 vlib_buffer_free_list_t *f;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700982 f = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index);
983 return alloc_from_free_list (vm, f, buffers, n_buffers);
984}
985
986always_inline void
987add_buffer_to_free_list (vlib_main_t * vm,
Dave Barach9b8ffd92016-07-08 08:13:45 -0400988 vlib_buffer_free_list_t * f,
Ed Warnickecb9cada2015-12-08 15:45:58 -0700989 u32 buffer_index, u8 do_init)
990{
Dave Barach9b8ffd92016-07-08 08:13:45 -0400991 vlib_buffer_t *b;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700992 b = vlib_get_buffer (vm, buffer_index);
Dave Barach9b8ffd92016-07-08 08:13:45 -0400993 if (PREDICT_TRUE (do_init))
994 vlib_buffer_init_for_free_list (b, f);
995 vec_add1_aligned (f->aligned_buffers, buffer_index,
996 sizeof (vlib_copy_unit_t));
Ed Warnickecb9cada2015-12-08 15:45:58 -0700997}
998
999always_inline vlib_buffer_free_list_t *
1000buffer_get_free_list (vlib_main_t * vm, vlib_buffer_t * b, u32 * index)
1001{
Dave Barach9b8ffd92016-07-08 08:13:45 -04001002 vlib_buffer_main_t *bm = vm->buffer_main;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001003 u32 i;
1004
1005 *index = i = b->free_list_index;
1006 return pool_elt_at_index (bm->buffer_free_list_pool, i);
1007}
1008
Dave Barach9b8ffd92016-07-08 08:13:45 -04001009void *
1010vlib_set_buffer_free_callback (vlib_main_t * vm, void *fp)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001011{
Dave Barach9b8ffd92016-07-08 08:13:45 -04001012 vlib_buffer_main_t *bm = vm->buffer_main;
1013 void *rv = bm->buffer_free_callback;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001014
1015 bm->buffer_free_callback = fp;
1016 return rv;
1017}
1018
Damjan Marion1cd8f3c2016-11-24 02:07:32 +01001019#if DPDK == 0
Ed Warnickecb9cada2015-12-08 15:45:58 -07001020void vnet_buffer_free_dpdk_mb (vlib_buffer_t * b) __attribute__ ((weak));
Dave Barach9b8ffd92016-07-08 08:13:45 -04001021void
1022vnet_buffer_free_dpdk_mb (vlib_buffer_t * b)
1023{
1024}
Ed Warnickecb9cada2015-12-08 15:45:58 -07001025
Damjan Marion1cd8f3c2016-11-24 02:07:32 +01001026#endif
Ed Warnickecb9cada2015-12-08 15:45:58 -07001027static_always_inline void
1028vlib_buffer_free_inline (vlib_main_t * vm,
Dave Barach9b8ffd92016-07-08 08:13:45 -04001029 u32 * buffers, u32 n_buffers, u32 follow_buffer_next)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001030{
Damjan Marion1cd8f3c2016-11-24 02:07:32 +01001031#if DPDK > 0
1032 vlib_buffer_main_t *bm = vm->buffer_main;
1033 vlib_buffer_free_list_t *fl;
1034 u32 fi;
1035 int i;
1036 u32 (*cb) (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
1037 u32 follow_buffer_next);
1038
1039 cb = bm->buffer_free_callback;
1040
1041 if (PREDICT_FALSE (cb != 0))
1042 n_buffers = (*cb) (vm, buffers, n_buffers, follow_buffer_next);
1043
1044 if (!n_buffers)
1045 return;
1046
1047 for (i = 0; i < n_buffers; i++)
1048 {
1049 vlib_buffer_t *b;
1050 struct rte_mbuf *mb;
1051
1052 b = vlib_get_buffer (vm, buffers[i]);
1053
1054 fl = buffer_get_free_list (vm, b, &fi);
1055
1056 /* The only current use of this callback: multicast recycle */
1057 if (PREDICT_FALSE (fl->buffers_added_to_freelist_function != 0))
1058 {
1059 int j;
1060
1061 add_buffer_to_free_list
1062 (vm, fl, buffers[i], (b->flags & VLIB_BUFFER_RECYCLE) == 0);
1063
1064 for (j = 0; j < vec_len (bm->announce_list); j++)
1065 {
1066 if (fl == bm->announce_list[j])
1067 goto already_announced;
1068 }
1069 vec_add1 (bm->announce_list, fl);
1070 already_announced:
1071 ;
1072 }
1073 else
1074 {
1075 if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_RECYCLE) == 0))
1076 {
1077 mb = rte_mbuf_from_vlib_buffer (b);
1078 ASSERT (rte_mbuf_refcnt_read (mb) == 1);
1079 rte_pktmbuf_free (mb);
1080 }
1081 }
1082 }
1083 if (vec_len (bm->announce_list))
1084 {
1085 vlib_buffer_free_list_t *fl;
1086 for (i = 0; i < vec_len (bm->announce_list); i++)
1087 {
1088 fl = bm->announce_list[i];
1089 fl->buffers_added_to_freelist_function (vm, fl);
1090 }
1091 _vec_len (bm->announce_list) = 0;
1092 }
1093#else
Dave Barach9b8ffd92016-07-08 08:13:45 -04001094 vlib_buffer_main_t *bm = vm->buffer_main;
1095 vlib_buffer_free_list_t *fl;
1096 static u32 *next_to_free[2]; /* smp bad */
1097 u32 i_next_to_free, *b, *n, *f, fi;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001098 uword n_left;
1099 int i;
Dave Barach9b8ffd92016-07-08 08:13:45 -04001100 static vlib_buffer_free_list_t **announce_list;
1101 vlib_buffer_free_list_t *fl0 = 0, *fl1 = 0;
1102 u32 bi0 = (u32) ~ 0, bi1 = (u32) ~ 0, fi0, fi1 = (u32) ~ 0;
1103 u8 free0, free1 = 0, free_next0, free_next1;
1104 u32 (*cb) (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
1105 u32 follow_buffer_next);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001106
Dave Barach9b8ffd92016-07-08 08:13:45 -04001107 ASSERT (os_get_cpu_number () == 0);
1108
Ed Warnickecb9cada2015-12-08 15:45:58 -07001109 cb = bm->buffer_free_callback;
1110
1111 if (PREDICT_FALSE (cb != 0))
Dave Barach9b8ffd92016-07-08 08:13:45 -04001112 n_buffers = (*cb) (vm, buffers, n_buffers, follow_buffer_next);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001113
Dave Barach9b8ffd92016-07-08 08:13:45 -04001114 if (!n_buffers)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001115 return;
1116
1117 /* Use first buffer to get default free list. */
1118 {
1119 u32 bi0 = buffers[0];
Dave Barach9b8ffd92016-07-08 08:13:45 -04001120 vlib_buffer_t *b0;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001121
1122 b0 = vlib_get_buffer (vm, bi0);
1123 fl = buffer_get_free_list (vm, b0, &fi);
1124 if (fl->buffers_added_to_freelist_function)
Dave Barach9b8ffd92016-07-08 08:13:45 -04001125 vec_add1 (announce_list, fl);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001126 }
1127
1128 vec_validate (next_to_free[0], n_buffers - 1);
1129 vec_validate (next_to_free[1], n_buffers - 1);
1130
1131 i_next_to_free = 0;
1132 n_left = n_buffers;
1133 b = buffers;
1134
Dave Barach9b8ffd92016-07-08 08:13:45 -04001135again:
Ed Warnickecb9cada2015-12-08 15:45:58 -07001136 /* Verify that buffers are known allocated. */
1137 vlib_buffer_validate_alloc_free (vm, b,
Dave Barach9b8ffd92016-07-08 08:13:45 -04001138 n_left, VLIB_BUFFER_KNOWN_ALLOCATED);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001139
1140 vec_add2_aligned (fl->aligned_buffers, f, n_left,
1141 /* align */ sizeof (vlib_copy_unit_t));
1142
1143 n = next_to_free[i_next_to_free];
1144 while (n_left >= 4)
1145 {
Dave Barach9b8ffd92016-07-08 08:13:45 -04001146 vlib_buffer_t *b0, *b1, *binit0, *binit1, dummy_buffers[2];
Ed Warnickecb9cada2015-12-08 15:45:58 -07001147
1148 bi0 = b[0];
1149 bi1 = b[1];
1150
1151 f[0] = bi0;
1152 f[1] = bi1;
1153 f += 2;
1154 b += 2;
1155 n_left -= 2;
1156
1157 /* Prefetch buffers for next iteration. */
1158 vlib_prefetch_buffer_with_index (vm, b[0], WRITE);
1159 vlib_prefetch_buffer_with_index (vm, b[1], WRITE);
1160
1161 b0 = vlib_get_buffer (vm, bi0);
1162 b1 = vlib_get_buffer (vm, bi1);
1163
Dave Barachb5adaea2016-06-17 14:09:56 -04001164 free0 = (b0->flags & VLIB_BUFFER_RECYCLE) == 0;
1165 free1 = (b1->flags & VLIB_BUFFER_RECYCLE) == 0;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001166
1167 /* Must be before init which will over-write buffer flags. */
1168 if (follow_buffer_next)
1169 {
1170 n[0] = b0->next_buffer;
1171 free_next0 = free0 && (b0->flags & VLIB_BUFFER_NEXT_PRESENT) != 0;
1172 n += free_next0;
1173
1174 n[0] = b1->next_buffer;
1175 free_next1 = free1 && (b1->flags & VLIB_BUFFER_NEXT_PRESENT) != 0;
1176 n += free_next1;
1177 }
1178 else
1179 free_next0 = free_next1 = 0;
1180
1181 /* Must be before init which will over-write buffer free list. */
1182 fi0 = b0->free_list_index;
1183 fi1 = b1->free_list_index;
1184
1185 if (PREDICT_FALSE (fi0 != fi || fi1 != fi))
1186 goto slow_path_x2;
1187
1188 binit0 = free0 ? b0 : &dummy_buffers[0];
1189 binit1 = free1 ? b1 : &dummy_buffers[1];
1190
1191 vlib_buffer_init_two_for_free_list (binit0, binit1, fl);
1192 continue;
1193
1194 slow_path_x2:
1195 /* Backup speculation. */
1196 f -= 2;
1197 n -= free_next0 + free_next1;
1198
1199 _vec_len (fl->aligned_buffers) = f - fl->aligned_buffers;
1200
1201 fl0 = pool_elt_at_index (bm->buffer_free_list_pool, fi0);
1202 fl1 = pool_elt_at_index (bm->buffer_free_list_pool, fi1);
1203
1204 add_buffer_to_free_list (vm, fl0, bi0, free0);
Dave Barach9b8ffd92016-07-08 08:13:45 -04001205 if (PREDICT_FALSE (fl0->buffers_added_to_freelist_function != 0))
1206 {
1207 int i;
1208 for (i = 0; i < vec_len (announce_list); i++)
1209 if (fl0 == announce_list[i])
1210 goto no_fl0;
1211 vec_add1 (announce_list, fl0);
1212 }
Ed Warnickecb9cada2015-12-08 15:45:58 -07001213 no_fl0:
Dave Barach9b8ffd92016-07-08 08:13:45 -04001214 if (PREDICT_FALSE (fl1->buffers_added_to_freelist_function != 0))
1215 {
1216 int i;
1217 for (i = 0; i < vec_len (announce_list); i++)
1218 if (fl1 == announce_list[i])
1219 goto no_fl1;
1220 vec_add1 (announce_list, fl1);
1221 }
Ed Warnickecb9cada2015-12-08 15:45:58 -07001222
1223 no_fl1:
1224 add_buffer_to_free_list (vm, fl1, bi1, free1);
1225
1226 /* Possibly change current free list. */
1227 if (fi0 != fi && fi1 != fi)
1228 {
1229 fi = fi1;
1230 fl = pool_elt_at_index (bm->buffer_free_list_pool, fi);
1231 }
1232
1233 vec_add2_aligned (fl->aligned_buffers, f, n_left,
1234 /* align */ sizeof (vlib_copy_unit_t));
1235 }
1236
1237 while (n_left >= 1)
1238 {
Dave Barach9b8ffd92016-07-08 08:13:45 -04001239 vlib_buffer_t *b0, *binit0, dummy_buffers[1];
Ed Warnickecb9cada2015-12-08 15:45:58 -07001240
1241 bi0 = b[0];
1242 f[0] = bi0;
1243 f += 1;
1244 b += 1;
1245 n_left -= 1;
1246
1247 b0 = vlib_get_buffer (vm, bi0);
1248
Dave Barachb5adaea2016-06-17 14:09:56 -04001249 free0 = (b0->flags & VLIB_BUFFER_RECYCLE) == 0;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001250
1251 /* Must be before init which will over-write buffer flags. */
1252 if (follow_buffer_next)
1253 {
1254 n[0] = b0->next_buffer;
1255 free_next0 = free0 && (b0->flags & VLIB_BUFFER_NEXT_PRESENT) != 0;
1256 n += free_next0;
1257 }
1258 else
1259 free_next0 = 0;
1260
1261 /* Must be before init which will over-write buffer free list. */
1262 fi0 = b0->free_list_index;
1263
1264 if (PREDICT_FALSE (fi0 != fi))
1265 goto slow_path_x1;
1266
1267 binit0 = free0 ? b0 : &dummy_buffers[0];
1268
1269 vlib_buffer_init_for_free_list (binit0, fl);
1270 continue;
1271
1272 slow_path_x1:
1273 /* Backup speculation. */
1274 f -= 1;
1275 n -= free_next0;
1276
1277 _vec_len (fl->aligned_buffers) = f - fl->aligned_buffers;
1278
1279 fl0 = pool_elt_at_index (bm->buffer_free_list_pool, fi0);
1280
1281 add_buffer_to_free_list (vm, fl0, bi0, free0);
Dave Barach9b8ffd92016-07-08 08:13:45 -04001282 if (PREDICT_FALSE (fl0->buffers_added_to_freelist_function != 0))
1283 {
1284 int i;
1285 for (i = 0; i < vec_len (announce_list); i++)
1286 if (fl0 == announce_list[i])
1287 goto no_fl00;
1288 vec_add1 (announce_list, fl0);
1289 }
1290
Ed Warnickecb9cada2015-12-08 15:45:58 -07001291 no_fl00:
1292 fi = fi0;
1293 fl = pool_elt_at_index (bm->buffer_free_list_pool, fi);
1294
1295 vec_add2_aligned (fl->aligned_buffers, f, n_left,
1296 /* align */ sizeof (vlib_copy_unit_t));
1297 }
1298
1299 if (follow_buffer_next && ((n_left = n - next_to_free[i_next_to_free]) > 0))
1300 {
1301 b = next_to_free[i_next_to_free];
1302 i_next_to_free ^= 1;
1303 goto again;
1304 }
1305
1306 _vec_len (fl->aligned_buffers) = f - fl->aligned_buffers;
1307
Dave Barach9b8ffd92016-07-08 08:13:45 -04001308 if (vec_len (announce_list))
Ed Warnickecb9cada2015-12-08 15:45:58 -07001309 {
Dave Barach9b8ffd92016-07-08 08:13:45 -04001310 vlib_buffer_free_list_t *fl;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001311 for (i = 0; i < vec_len (announce_list); i++)
Dave Barach9b8ffd92016-07-08 08:13:45 -04001312 {
1313 fl = announce_list[i];
1314 fl->buffers_added_to_freelist_function (vm, fl);
1315 }
1316 _vec_len (announce_list) = 0;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001317 }
Damjan Marion1cd8f3c2016-11-24 02:07:32 +01001318#endif
Ed Warnickecb9cada2015-12-08 15:45:58 -07001319}
1320
Dave Barach9b8ffd92016-07-08 08:13:45 -04001321void
1322vlib_buffer_free (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001323{
Dave Barach9b8ffd92016-07-08 08:13:45 -04001324 vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */
1325 1);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001326}
1327
Dave Barach9b8ffd92016-07-08 08:13:45 -04001328void
1329vlib_buffer_free_no_next (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001330{
Dave Barach9b8ffd92016-07-08 08:13:45 -04001331 vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */
1332 0);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001333}
1334
Damjan Marion1cd8f3c2016-11-24 02:07:32 +01001335#if DPDK == 0
Ed Warnickecb9cada2015-12-08 15:45:58 -07001336/* Copy template packet data into buffers as they are allocated. */
1337static void
1338vlib_packet_template_buffer_init (vlib_main_t * vm,
1339 vlib_buffer_free_list_t * fl,
Dave Barach9b8ffd92016-07-08 08:13:45 -04001340 u32 * buffers, u32 n_buffers)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001341{
Dave Barach9b8ffd92016-07-08 08:13:45 -04001342 vlib_packet_template_t *t =
1343 uword_to_pointer (fl->buffer_init_function_opaque,
1344 vlib_packet_template_t *);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001345 uword i;
1346
1347 for (i = 0; i < n_buffers; i++)
1348 {
Dave Barach9b8ffd92016-07-08 08:13:45 -04001349 vlib_buffer_t *b = vlib_get_buffer (vm, buffers[i]);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001350 ASSERT (b->current_length == vec_len (t->packet_data));
Dave Barach9b8ffd92016-07-08 08:13:45 -04001351 clib_memcpy (vlib_buffer_get_current (b), t->packet_data,
1352 b->current_length);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001353 }
1354}
Damjan Marion1cd8f3c2016-11-24 02:07:32 +01001355#endif
Ed Warnickecb9cada2015-12-08 15:45:58 -07001356
Dave Barach9b8ffd92016-07-08 08:13:45 -04001357void
1358vlib_packet_template_init (vlib_main_t * vm,
1359 vlib_packet_template_t * t,
1360 void *packet_data,
1361 uword n_packet_data_bytes,
1362 uword min_n_buffers_each_physmem_alloc,
1363 char *fmt, ...)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001364{
Damjan Marion1cd8f3c2016-11-24 02:07:32 +01001365#if DPDK > 0
1366 va_list va;
1367 __attribute__ ((unused)) u8 *name;
1368
1369 va_start (va, fmt);
1370 name = va_format (0, fmt, &va);
1371 va_end (va);
1372
1373 vlib_worker_thread_barrier_sync (vm);
1374 memset (t, 0, sizeof (t[0]));
1375
1376 vec_add (t->packet_data, packet_data, n_packet_data_bytes);
1377
1378 vlib_worker_thread_barrier_release (vm);
1379#else
Dave Barach9b8ffd92016-07-08 08:13:45 -04001380 vlib_buffer_free_list_t *fl;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001381 va_list va;
Dave Barach9b8ffd92016-07-08 08:13:45 -04001382 u8 *name;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001383
1384 va_start (va, fmt);
1385 name = va_format (0, fmt, &va);
1386 va_end (va);
1387
1388 memset (t, 0, sizeof (t[0]));
1389
1390 vec_add (t->packet_data, packet_data, n_packet_data_bytes);
1391 t->min_n_buffers_each_physmem_alloc = min_n_buffers_each_physmem_alloc;
1392
1393 t->free_list_index = vlib_buffer_create_free_list_helper
1394 (vm, n_packet_data_bytes,
1395 /* is_public */ 1,
1396 /* is_default */ 0,
1397 name);
1398
1399 ASSERT (t->free_list_index != 0);
1400 fl = vlib_buffer_get_free_list (vm, t->free_list_index);
1401 fl->min_n_buffers_each_physmem_alloc = t->min_n_buffers_each_physmem_alloc;
1402
1403 fl->buffer_init_function = vlib_packet_template_buffer_init;
1404 fl->buffer_init_function_opaque = pointer_to_uword (t);
1405
1406 fl->buffer_init_template.current_data = 0;
1407 fl->buffer_init_template.current_length = n_packet_data_bytes;
1408 fl->buffer_init_template.flags = 0;
Damjan Marion1cd8f3c2016-11-24 02:07:32 +01001409#endif
Ed Warnickecb9cada2015-12-08 15:45:58 -07001410}
1411
1412void *
Dave Barach9b8ffd92016-07-08 08:13:45 -04001413vlib_packet_template_get_packet (vlib_main_t * vm,
1414 vlib_packet_template_t * t, u32 * bi_result)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001415{
1416 u32 bi;
Dave Barach9b8ffd92016-07-08 08:13:45 -04001417 vlib_buffer_t *b;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001418
1419 if (vlib_buffer_alloc (vm, &bi, 1) != 1)
1420 return 0;
1421
1422 *bi_result = bi;
1423
1424 b = vlib_get_buffer (vm, bi);
Damjan Marionf1213b82016-03-13 02:22:06 +01001425 clib_memcpy (vlib_buffer_get_current (b),
Dave Barach9b8ffd92016-07-08 08:13:45 -04001426 t->packet_data, vec_len (t->packet_data));
1427 b->current_length = vec_len (t->packet_data);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001428
1429 return b->data;
1430}
1431
Damjan Marion1cd8f3c2016-11-24 02:07:32 +01001432#if DPDK == 0
Dave Barach9b8ffd92016-07-08 08:13:45 -04001433void
1434vlib_packet_template_get_packet_helper (vlib_main_t * vm,
1435 vlib_packet_template_t * t)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001436{
1437 word n = t->min_n_buffers_each_physmem_alloc;
1438 word l = vec_len (t->packet_data);
1439 word n_alloc;
1440
1441 ASSERT (l > 0);
1442 ASSERT (vec_len (t->free_buffers) == 0);
1443
1444 vec_validate (t->free_buffers, n - 1);
1445 n_alloc = vlib_buffer_alloc_from_free_list (vm, t->free_buffers,
1446 n, t->free_list_index);
1447 _vec_len (t->free_buffers) = n_alloc;
1448}
1449
Damjan Marion1cd8f3c2016-11-24 02:07:32 +01001450#endif
Ed Warnickecb9cada2015-12-08 15:45:58 -07001451/* Append given data to end of buffer, possibly allocating new buffers. */
Dave Barach9b8ffd92016-07-08 08:13:45 -04001452u32
1453vlib_buffer_add_data (vlib_main_t * vm,
1454 u32 free_list_index,
1455 u32 buffer_index, void *data, u32 n_data_bytes)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001456{
1457 u32 n_buffer_bytes, n_left, n_left_this_buffer, bi;
Dave Barach9b8ffd92016-07-08 08:13:45 -04001458 vlib_buffer_t *b;
1459 void *d;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001460
1461 bi = buffer_index;
1462 if (bi == 0
1463 && 1 != vlib_buffer_alloc_from_free_list (vm, &bi, 1, free_list_index))
1464 goto out_of_buffers;
1465
1466 d = data;
1467 n_left = n_data_bytes;
1468 n_buffer_bytes = vlib_buffer_free_list_buffer_size (vm, free_list_index);
Dave Barach9b8ffd92016-07-08 08:13:45 -04001469
Ed Warnickecb9cada2015-12-08 15:45:58 -07001470 b = vlib_get_buffer (vm, bi);
1471 b->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
1472
Dave Barach9b8ffd92016-07-08 08:13:45 -04001473 /* Get to the end of the chain before we try to append data... */
Ed Warnickecb9cada2015-12-08 15:45:58 -07001474 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
1475 b = vlib_get_buffer (vm, b->next_buffer);
1476
1477 while (1)
1478 {
1479 u32 n;
1480
1481 ASSERT (n_buffer_bytes >= b->current_length);
Dave Barach9b8ffd92016-07-08 08:13:45 -04001482 n_left_this_buffer =
1483 n_buffer_bytes - (b->current_data + b->current_length);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001484 n = clib_min (n_left_this_buffer, n_left);
Damjan Marionf1213b82016-03-13 02:22:06 +01001485 clib_memcpy (vlib_buffer_get_current (b) + b->current_length, d, n);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001486 b->current_length += n;
1487 n_left -= n;
1488 if (n_left == 0)
1489 break;
1490
1491 d += n;
Dave Barach9b8ffd92016-07-08 08:13:45 -04001492 if (1 !=
1493 vlib_buffer_alloc_from_free_list (vm, &b->next_buffer, 1,
1494 free_list_index))
Ed Warnickecb9cada2015-12-08 15:45:58 -07001495 goto out_of_buffers;
1496
1497 b->flags |= VLIB_BUFFER_NEXT_PRESENT;
1498
1499 b = vlib_get_buffer (vm, b->next_buffer);
1500 }
1501
1502 return bi;
1503
Dave Barach9b8ffd92016-07-08 08:13:45 -04001504out_of_buffers:
Ed Warnickecb9cada2015-12-08 15:45:58 -07001505 clib_error ("out of buffers");
1506 return bi;
1507}
1508
Pierre Pfister328e99b2016-02-12 13:18:42 +00001509u16
Dave Barach9b8ffd92016-07-08 08:13:45 -04001510vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm,
1511 u32 free_list_index,
1512 vlib_buffer_t * first,
1513 vlib_buffer_t ** last,
1514 void *data, u16 data_len)
1515{
Pierre Pfister328e99b2016-02-12 13:18:42 +00001516 vlib_buffer_t *l = *last;
Dave Barach9b8ffd92016-07-08 08:13:45 -04001517 u32 n_buffer_bytes =
1518 vlib_buffer_free_list_buffer_size (vm, free_list_index);
Pierre Pfister328e99b2016-02-12 13:18:42 +00001519 u16 copied = 0;
Dave Barach9b8ffd92016-07-08 08:13:45 -04001520 ASSERT (n_buffer_bytes >= l->current_length + l->current_data);
1521 while (data_len)
1522 {
1523 u16 max = n_buffer_bytes - l->current_length - l->current_data;
1524 if (max == 0)
1525 {
1526 if (1 !=
1527 vlib_buffer_alloc_from_free_list (vm, &l->next_buffer, 1,
1528 free_list_index))
1529 return copied;
1530 *last = l = vlib_buffer_chain_buffer (vm, first, l, l->next_buffer);
1531 max = n_buffer_bytes - l->current_length - l->current_data;
1532 }
Pierre Pfister328e99b2016-02-12 13:18:42 +00001533
Dave Barach9b8ffd92016-07-08 08:13:45 -04001534 u16 len = (data_len > max) ? max : data_len;
1535 clib_memcpy (vlib_buffer_get_current (l) + l->current_length,
1536 data + copied, len);
1537 vlib_buffer_chain_increase_length (first, l, len);
1538 data_len -= len;
1539 copied += len;
1540 }
Pierre Pfister328e99b2016-02-12 13:18:42 +00001541 return copied;
1542}
1543
Damjan Marion1cd8f3c2016-11-24 02:07:32 +01001544#if DPDK > 0
1545clib_error_t *
1546vlib_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs,
1547 unsigned socket_id)
1548{
1549 vlib_buffer_main_t *bm = vm->buffer_main;
1550 vlib_physmem_main_t *vpm = &vm->physmem_main;
1551 struct rte_mempool *rmp;
1552 int i;
1553
1554 if (!rte_pktmbuf_pool_create)
1555 return clib_error_return (0, "not linked with DPDK");
1556
1557 vec_validate_aligned (bm->pktmbuf_pools, socket_id, CLIB_CACHE_LINE_BYTES);
1558
1559 /* pool already exists, nothing to do */
1560 if (bm->pktmbuf_pools[socket_id])
1561 return 0;
1562
1563 u8 *pool_name = format (0, "mbuf_pool_socket%u%c", socket_id, 0);
1564
1565 rmp = rte_pktmbuf_pool_create ((char *) pool_name, /* pool name */
1566 num_mbufs, /* number of mbufs */
1567 512, /* cache size */
1568 VLIB_BUFFER_HDR_SIZE, /* priv size */
1569 VLIB_BUFFER_PRE_DATA_SIZE + VLIB_BUFFER_DATA_SIZE, /* dataroom size */
1570 socket_id); /* cpu socket */
1571
1572 if (rmp)
1573 {
1574 {
1575 uword this_pool_end;
1576 uword this_pool_start;
1577 uword this_pool_size;
1578 uword save_vpm_start, save_vpm_end, save_vpm_size;
1579 struct rte_mempool_memhdr *memhdr;
1580
1581 this_pool_start = ~0ULL;
1582 this_pool_end = 0LL;
1583
1584 STAILQ_FOREACH (memhdr, &rmp->mem_list, next)
1585 {
1586 if (((uword) (memhdr->addr + memhdr->len)) > this_pool_end)
1587 this_pool_end = (uword) (memhdr->addr + memhdr->len);
1588 if (((uword) memhdr->addr) < this_pool_start)
1589 this_pool_start = (uword) (memhdr->addr);
1590 }
1591 ASSERT (this_pool_start < ~0ULL && this_pool_end > 0);
1592 this_pool_size = this_pool_end - this_pool_start;
1593
1594 if (CLIB_DEBUG > 1)
1595 {
1596 clib_warning ("%s: pool start %llx pool end %llx pool size %lld",
1597 pool_name, this_pool_start, this_pool_end,
1598 this_pool_size);
1599 clib_warning
1600 ("before: virtual.start %llx virtual.end %llx virtual.size %lld",
1601 vpm->virtual.start, vpm->virtual.end, vpm->virtual.size);
1602 }
1603
1604 save_vpm_start = vpm->virtual.start;
1605 save_vpm_end = vpm->virtual.end;
1606 save_vpm_size = vpm->virtual.size;
1607
1608 if ((this_pool_start < vpm->virtual.start) || vpm->virtual.start == 0)
1609 vpm->virtual.start = this_pool_start;
1610 if (this_pool_end > vpm->virtual.end)
1611 vpm->virtual.end = this_pool_end;
1612
1613 vpm->virtual.size = vpm->virtual.end - vpm->virtual.start;
1614
1615 if (CLIB_DEBUG > 1)
1616 {
1617 clib_warning
1618 ("after: virtual.start %llx virtual.end %llx virtual.size %lld",
1619 vpm->virtual.start, vpm->virtual.end, vpm->virtual.size);
1620 }
1621
1622 /* check if fits into buffer index range */
1623 if ((u64) vpm->virtual.size >
1624 ((u64) 1 << (32 + CLIB_LOG2_CACHE_LINE_BYTES)))
1625 {
1626 clib_warning ("physmem: virtual size out of range!");
1627 vpm->virtual.start = save_vpm_start;
1628 vpm->virtual.end = save_vpm_end;
1629 vpm->virtual.size = save_vpm_size;
1630 rmp = 0;
1631 }
1632 }
1633 if (rmp)
1634 {
1635 bm->pktmbuf_pools[socket_id] = rmp;
1636 vec_free (pool_name);
1637 return 0;
1638 }
1639 }
1640
1641 vec_free (pool_name);
1642
1643 /* no usable pool for this socket, try to use pool from another one */
1644 for (i = 0; i < vec_len (bm->pktmbuf_pools); i++)
1645 {
1646 if (bm->pktmbuf_pools[i])
1647 {
1648 clib_warning
1649 ("WARNING: Failed to allocate mempool for CPU socket %u. "
1650 "Threads running on socket %u will use socket %u mempool.",
1651 socket_id, socket_id, i);
1652 bm->pktmbuf_pools[socket_id] = bm->pktmbuf_pools[i];
1653 return 0;
1654 }
1655 }
1656
1657 return clib_error_return (0, "failed to allocate mempool on socket %u",
1658 socket_id);
1659}
1660#endif
1661
Dave Barach9b8ffd92016-07-08 08:13:45 -04001662static void
1663vlib_serialize_tx (serialize_main_header_t * m, serialize_stream_t * s)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001664{
Dave Barach9b8ffd92016-07-08 08:13:45 -04001665 vlib_main_t *vm;
1666 vlib_serialize_buffer_main_t *sm;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001667 uword n, n_bytes_to_write;
Dave Barach9b8ffd92016-07-08 08:13:45 -04001668 vlib_buffer_t *last;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001669
1670 n_bytes_to_write = s->current_buffer_index;
Dave Barach9b8ffd92016-07-08 08:13:45 -04001671 sm =
1672 uword_to_pointer (s->data_function_opaque,
1673 vlib_serialize_buffer_main_t *);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001674 vm = sm->vlib_main;
1675
1676 ASSERT (sm->tx.max_n_data_bytes_per_chain > 0);
1677 if (serialize_stream_is_end_of_stream (s)
Dave Barach9b8ffd92016-07-08 08:13:45 -04001678 || sm->tx.n_total_data_bytes + n_bytes_to_write >
1679 sm->tx.max_n_data_bytes_per_chain)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001680 {
Dave Barach9b8ffd92016-07-08 08:13:45 -04001681 vlib_process_t *p = vlib_get_current_process (vm);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001682
1683 last = vlib_get_buffer (vm, sm->last_buffer);
1684 last->current_length = n_bytes_to_write;
1685
Dave Barach9b8ffd92016-07-08 08:13:45 -04001686 vlib_set_next_frame_buffer (vm, &p->node_runtime, sm->tx.next_index,
1687 sm->first_buffer);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001688
1689 sm->first_buffer = sm->last_buffer = ~0;
1690 sm->tx.n_total_data_bytes = 0;
1691 }
1692
1693 else if (n_bytes_to_write == 0 && s->n_buffer_bytes == 0)
1694 {
1695 ASSERT (sm->first_buffer == ~0);
1696 ASSERT (sm->last_buffer == ~0);
Dave Barach9b8ffd92016-07-08 08:13:45 -04001697 n =
1698 vlib_buffer_alloc_from_free_list (vm, &sm->first_buffer, 1,
1699 sm->tx.free_list_index);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001700 if (n != 1)
Dave Barach9b8ffd92016-07-08 08:13:45 -04001701 serialize_error (m,
1702 clib_error_create
1703 ("vlib_buffer_alloc_from_free_list fails"));
Ed Warnickecb9cada2015-12-08 15:45:58 -07001704 sm->last_buffer = sm->first_buffer;
Dave Barach9b8ffd92016-07-08 08:13:45 -04001705 s->n_buffer_bytes =
1706 vlib_buffer_free_list_buffer_size (vm, sm->tx.free_list_index);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001707 }
1708
1709 if (n_bytes_to_write > 0)
1710 {
Dave Barach9b8ffd92016-07-08 08:13:45 -04001711 vlib_buffer_t *prev = vlib_get_buffer (vm, sm->last_buffer);
1712 n =
1713 vlib_buffer_alloc_from_free_list (vm, &sm->last_buffer, 1,
1714 sm->tx.free_list_index);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001715 if (n != 1)
Dave Barach9b8ffd92016-07-08 08:13:45 -04001716 serialize_error (m,
1717 clib_error_create
1718 ("vlib_buffer_alloc_from_free_list fails"));
Ed Warnickecb9cada2015-12-08 15:45:58 -07001719 sm->tx.n_total_data_bytes += n_bytes_to_write;
1720 prev->current_length = n_bytes_to_write;
1721 prev->next_buffer = sm->last_buffer;
1722 prev->flags |= VLIB_BUFFER_NEXT_PRESENT;
1723 }
1724
1725 if (sm->last_buffer != ~0)
1726 {
1727 last = vlib_get_buffer (vm, sm->last_buffer);
1728 s->buffer = vlib_buffer_get_current (last);
1729 s->current_buffer_index = 0;
1730 ASSERT (last->current_data == s->current_buffer_index);
1731 }
1732}
1733
Dave Barach9b8ffd92016-07-08 08:13:45 -04001734static void
1735vlib_serialize_rx (serialize_main_header_t * m, serialize_stream_t * s)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001736{
Dave Barach9b8ffd92016-07-08 08:13:45 -04001737 vlib_main_t *vm;
1738 vlib_serialize_buffer_main_t *sm;
1739 vlib_buffer_t *last;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001740
Dave Barach9b8ffd92016-07-08 08:13:45 -04001741 sm =
1742 uword_to_pointer (s->data_function_opaque,
1743 vlib_serialize_buffer_main_t *);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001744 vm = sm->vlib_main;
1745
1746 if (serialize_stream_is_end_of_stream (s))
1747 return;
1748
1749 if (sm->last_buffer != ~0)
1750 {
1751 last = vlib_get_buffer (vm, sm->last_buffer);
1752
1753 if (last->flags & VLIB_BUFFER_NEXT_PRESENT)
1754 sm->last_buffer = last->next_buffer;
1755 else
1756 {
1757 vlib_buffer_free (vm, &sm->first_buffer, /* count */ 1);
1758 sm->first_buffer = sm->last_buffer = ~0;
1759 }
1760 }
1761
1762 if (sm->last_buffer == ~0)
1763 {
1764 while (clib_fifo_elts (sm->rx.buffer_fifo) == 0)
1765 {
Dave Barach9b8ffd92016-07-08 08:13:45 -04001766 sm->rx.ready_one_time_event =
1767 vlib_process_create_one_time_event (vm, vlib_current_process (vm),
1768 ~0);
1769 vlib_process_wait_for_one_time_event (vm, /* no event data */ 0,
1770 sm->rx.ready_one_time_event);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001771 }
1772
1773 clib_fifo_sub1 (sm->rx.buffer_fifo, sm->first_buffer);
1774 sm->last_buffer = sm->first_buffer;
1775 }
1776
1777 ASSERT (sm->last_buffer != ~0);
1778
1779 last = vlib_get_buffer (vm, sm->last_buffer);
1780 s->current_buffer_index = 0;
1781 s->buffer = vlib_buffer_get_current (last);
1782 s->n_buffer_bytes = last->current_length;
1783}
1784
1785static void
1786serialize_open_vlib_helper (serialize_main_t * m,
1787 vlib_main_t * vm,
Dave Barach9b8ffd92016-07-08 08:13:45 -04001788 vlib_serialize_buffer_main_t * sm, uword is_read)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001789{
1790 /* Initialize serialize main but save overflow buffer for re-use between calls. */
1791 {
Dave Barach9b8ffd92016-07-08 08:13:45 -04001792 u8 *save = m->stream.overflow_buffer;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001793 memset (m, 0, sizeof (m[0]));
1794 m->stream.overflow_buffer = save;
1795 if (save)
1796 _vec_len (save) = 0;
1797 }
1798
1799 sm->first_buffer = sm->last_buffer = ~0;
1800 if (is_read)
1801 clib_fifo_reset (sm->rx.buffer_fifo);
1802 else
1803 sm->tx.n_total_data_bytes = 0;
1804 sm->vlib_main = vm;
1805 m->header.data_function = is_read ? vlib_serialize_rx : vlib_serialize_tx;
1806 m->stream.data_function_opaque = pointer_to_uword (sm);
1807}
1808
Dave Barach9b8ffd92016-07-08 08:13:45 -04001809void
1810serialize_open_vlib_buffer (serialize_main_t * m, vlib_main_t * vm,
1811 vlib_serialize_buffer_main_t * sm)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001812{
Dave Barach9b8ffd92016-07-08 08:13:45 -04001813 serialize_open_vlib_helper (m, vm, sm, /* is_read */ 0);
1814}
1815
1816void
1817unserialize_open_vlib_buffer (serialize_main_t * m, vlib_main_t * vm,
1818 vlib_serialize_buffer_main_t * sm)
1819{
1820 serialize_open_vlib_helper (m, vm, sm, /* is_read */ 1);
1821}
1822
1823u32
1824serialize_close_vlib_buffer (serialize_main_t * m)
1825{
1826 vlib_serialize_buffer_main_t *sm
1827 = uword_to_pointer (m->stream.data_function_opaque,
1828 vlib_serialize_buffer_main_t *);
1829 vlib_buffer_t *last;
1830 serialize_stream_t *s = &m->stream;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001831
1832 last = vlib_get_buffer (sm->vlib_main, sm->last_buffer);
1833 last->current_length = s->current_buffer_index;
1834
1835 if (vec_len (s->overflow_buffer) > 0)
1836 {
1837 sm->last_buffer
1838 = vlib_buffer_add_data (sm->vlib_main, sm->tx.free_list_index,
1839 sm->last_buffer == ~0 ? 0 : sm->last_buffer,
1840 s->overflow_buffer,
1841 vec_len (s->overflow_buffer));
1842 _vec_len (s->overflow_buffer) = 0;
1843 }
1844
1845 return sm->first_buffer;
1846}
1847
Dave Barach9b8ffd92016-07-08 08:13:45 -04001848void
1849unserialize_close_vlib_buffer (serialize_main_t * m)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001850{
Dave Barach9b8ffd92016-07-08 08:13:45 -04001851 vlib_serialize_buffer_main_t *sm
1852 = uword_to_pointer (m->stream.data_function_opaque,
1853 vlib_serialize_buffer_main_t *);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001854 if (sm->first_buffer != ~0)
1855 vlib_buffer_free_one (sm->vlib_main, sm->first_buffer);
1856 clib_fifo_reset (sm->rx.buffer_fifo);
1857 if (m->stream.overflow_buffer)
1858 _vec_len (m->stream.overflow_buffer) = 0;
1859}
1860
Dave Barach9b8ffd92016-07-08 08:13:45 -04001861static u8 *
1862format_vlib_buffer_free_list (u8 * s, va_list * va)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001863{
Dave Barach9b8ffd92016-07-08 08:13:45 -04001864 vlib_buffer_free_list_t *f = va_arg (*va, vlib_buffer_free_list_t *);
Damjan Marion1cd8f3c2016-11-24 02:07:32 +01001865#if DPDK > 0
1866 u32 threadnum = va_arg (*va, u32);
1867 uword bytes_alloc, bytes_free, n_free, size;
1868
1869 if (!f)
1870 return format (s, "%=7s%=30s%=12s%=12s%=12s%=12s%=12s%=12s",
1871 "Thread", "Name", "Index", "Size", "Alloc", "Free",
1872 "#Alloc", "#Free");
1873
1874 size = sizeof (vlib_buffer_t) + f->n_data_bytes;
1875 n_free = vec_len (f->aligned_buffers) + vec_len (f->unaligned_buffers);
1876 bytes_alloc = size * f->n_alloc;
1877 bytes_free = size * n_free;
1878
1879 s = format (s, "%7d%30s%12d%12d%=12U%=12U%=12d%=12d", threadnum,
1880#else
Ed Warnickecb9cada2015-12-08 15:45:58 -07001881 uword bytes_alloc, bytes_free, n_free, size;
1882
Dave Barach9b8ffd92016-07-08 08:13:45 -04001883 if (!f)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001884 return format (s, "%=30s%=12s%=12s%=12s%=12s%=12s%=12s",
Dave Barach9b8ffd92016-07-08 08:13:45 -04001885 "Name", "Index", "Size", "Alloc", "Free", "#Alloc",
1886 "#Free");
Ed Warnickecb9cada2015-12-08 15:45:58 -07001887
1888 size = sizeof (vlib_buffer_t) + f->n_data_bytes;
1889 n_free = vec_len (f->aligned_buffers) + vec_len (f->unaligned_buffers);
1890 bytes_alloc = size * f->n_alloc;
1891 bytes_free = size * n_free;
1892
1893 s = format (s, "%30s%12d%12d%=12U%=12U%=12d%=12d",
Damjan Marion1cd8f3c2016-11-24 02:07:32 +01001894#endif
Ed Warnickecb9cada2015-12-08 15:45:58 -07001895 f->name, f->index, f->n_data_bytes,
1896 format_memory_size, bytes_alloc,
Dave Barach9b8ffd92016-07-08 08:13:45 -04001897 format_memory_size, bytes_free, f->n_alloc, n_free);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001898
1899 return s;
1900}
1901
1902static clib_error_t *
1903show_buffers (vlib_main_t * vm,
Dave Barach9b8ffd92016-07-08 08:13:45 -04001904 unformat_input_t * input, vlib_cli_command_t * cmd)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001905{
Damjan Marion1cd8f3c2016-11-24 02:07:32 +01001906#if DPDK > 0
1907 vlib_buffer_main_t *bm;
1908 vlib_buffer_free_list_t *f;
1909 vlib_main_t *curr_vm;
1910 u32 vm_index = 0;
1911
1912 vlib_cli_output (vm, "%U", format_vlib_buffer_free_list, 0, 0);
1913
1914 do
1915 {
1916 curr_vm = vec_len (vlib_mains) ? vlib_mains[vm_index] : vm;
1917 bm = curr_vm->buffer_main;
1918
1919 /* *INDENT-OFF* */
1920 pool_foreach (f, bm->buffer_free_list_pool, ({
1921 vlib_cli_output (vm, "%U", format_vlib_buffer_free_list, f, vm_index);
1922 }));
1923 /* *INDENT-ON* */
1924
1925 vm_index++;
1926 }
1927 while (vm_index < vec_len (vlib_mains));
1928
1929#else
Dave Barach9b8ffd92016-07-08 08:13:45 -04001930 vlib_buffer_main_t *bm = vm->buffer_main;
1931 vlib_buffer_free_list_t *f;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001932
1933 vlib_cli_output (vm, "%U", format_vlib_buffer_free_list, 0);
Dave Barach9b8ffd92016-07-08 08:13:45 -04001934 /* *INDENT-OFF* */
Ed Warnickecb9cada2015-12-08 15:45:58 -07001935 pool_foreach (f, bm->buffer_free_list_pool, ({
1936 vlib_cli_output (vm, "%U", format_vlib_buffer_free_list, f);
1937 }));
Dave Barach9b8ffd92016-07-08 08:13:45 -04001938/* *INDENT-ON* */
Ed Warnickecb9cada2015-12-08 15:45:58 -07001939
Damjan Marion1cd8f3c2016-11-24 02:07:32 +01001940#endif
Ed Warnickecb9cada2015-12-08 15:45:58 -07001941 return 0;
1942}
1943
Dave Barach9b8ffd92016-07-08 08:13:45 -04001944/* *INDENT-OFF* */
Ed Warnickecb9cada2015-12-08 15:45:58 -07001945VLIB_CLI_COMMAND (show_buffers_command, static) = {
1946 .path = "show buffers",
1947 .short_help = "Show packet buffer allocation",
1948 .function = show_buffers,
1949};
Dave Barach9b8ffd92016-07-08 08:13:45 -04001950/* *INDENT-ON* */
1951
Damjan Marion1cd8f3c2016-11-24 02:07:32 +01001952#if DPDK > 0
1953#if CLIB_DEBUG > 0
1954
1955u32 *vlib_buffer_state_validation_lock;
1956uword *vlib_buffer_state_validation_hash;
1957void *vlib_buffer_state_heap;
1958
1959static clib_error_t *
1960buffer_state_validation_init (vlib_main_t * vm)
1961{
1962 void *oldheap;
1963
1964 vlib_buffer_state_heap = mheap_alloc (0, 10 << 20);
1965
1966 oldheap = clib_mem_set_heap (vlib_buffer_state_heap);
1967
1968 vlib_buffer_state_validation_hash = hash_create (0, sizeof (uword));
1969 vec_validate_aligned (vlib_buffer_state_validation_lock, 0,
1970 CLIB_CACHE_LINE_BYTES);
1971 clib_mem_set_heap (oldheap);
1972 return 0;
1973}
1974
1975VLIB_INIT_FUNCTION (buffer_state_validation_init);
1976#endif
1977#endif
1978
1979
Chris Luked4024f52016-09-06 09:32:36 -04001980/** @endcond */
Dave Barach9b8ffd92016-07-08 08:13:45 -04001981/*
1982 * fd.io coding-style-patch-verification: ON
1983 *
1984 * Local Variables:
1985 * eval: (c-set-style "gnu")
1986 * End:
1987 */