src/vppinfra/vec.c

/* SPDX-License-Identifier: Apache-2.0
 * Copyright(c) 2022 Cisco Systems, Inc.
 */

#include <vppinfra/vec.h>
#include <vppinfra/mem.h>

#ifndef CLIB_VECTOR_GROW_BY_ONE
#define CLIB_VECTOR_GROW_BY_ONE 0
#endif

__clib_export uword
vec_mem_size (void *v)
{
  return v ? clib_mem_size (v - vec_get_header_size (v)) : 0;
}

__clib_export void *
_vec_alloc_internal (uword n_elts, const vec_attr_t *const attr)
{
  uword req_size, alloc_size, data_offset, align;
  uword elt_sz = attr->elt_sz;
  void *p, *v, *heap = attr->heap;

  /* alignment must be power of 2 */
  align = clib_max (attr->align, VEC_MIN_ALIGN);
  ASSERT (count_set_bits (align) == 1);

  /* calc offset where vector data starts */
  data_offset = attr->hdr_sz + sizeof (vec_header_t);
  data_offset += heap ? sizeof (void *) : 0;
  data_offset = round_pow2 (data_offset, align);

  req_size = data_offset + n_elts * elt_sz;
  p = clib_mem_heap_alloc_aligned (heap, req_size, align);

  /* zero out whole alocation */
  alloc_size = clib_mem_size (p);
  clib_mem_unpoison (p, alloc_size);
  clib_memset_u8 (p, 0, alloc_size);

  /* fill vector header */
  v = p + data_offset;
  _vec_find (v)->len = n_elts;
  _vec_find (v)->hdr_size = data_offset / VEC_MIN_ALIGN;
  _vec_find (v)->log2_align = min_log2 (align);
  if (heap)
    {
      _vec_find (v)->default_heap = 0;
      _vec_heap (v) = heap;
    }
  else
    _vec_find (v)->default_heap = 1;

  /* poison extra space given by allocator */
  clib_mem_poison (p + req_size, alloc_size - req_size);
  _vec_set_grow_elts (v, (alloc_size - req_size) / elt_sz);
  return v;
}

static inline void
_vec_update_len (void *v, uword n_elts, uword elt_sz, uword n_data_bytes,
		 uword unused_bytes)
{
  _vec_find (v)->len = n_elts;
  _vec_set_grow_elts (v, unused_bytes / elt_sz);
  clib_mem_unpoison (v, n_data_bytes);
  clib_mem_poison (v + n_data_bytes, unused_bytes);
}

__clib_export void *
_vec_realloc_internal (void *v, uword n_elts, const vec_attr_t *const attr)
{
  uword old_alloc_sz, new_alloc_sz, new_data_size, n_data_bytes, data_offset;
  uword elt_sz;

  if (PREDICT_FALSE (v == 0))
    return _vec_alloc_internal (n_elts, attr);

  elt_sz = attr->elt_sz;
  n_data_bytes = n_elts * elt_sz;
  data_offset = vec_get_header_size (v);
  new_data_size = data_offset + n_data_bytes;
  new_alloc_sz = old_alloc_sz = clib_mem_size (vec_header (v));

  /* realloc if new size cannot fit into existing allocation */
  if (old_alloc_sz < new_data_size)
    {
      uword n_bytes, req_size = new_data_size;
      void *p = v - data_offset;

      req_size += CLIB_VECTOR_GROW_BY_ONE ? 0 : n_data_bytes / 2;

      p = clib_mem_heap_realloc_aligned (vec_get_heap (v), p, req_size,
					 vec_get_align (v));
      new_alloc_sz = clib_mem_size (p);
      v = p + data_offset;

      /* zero out new allocation */
      n_bytes = new_alloc_sz - old_alloc_sz;
      clib_mem_unpoison (p + old_alloc_sz, n_bytes);
      clib_memset_u8 (p + old_alloc_sz, 0, n_bytes);
    }

  _vec_update_len (v, n_elts, elt_sz, n_data_bytes,
		   new_alloc_sz - new_data_size);
  return v;
}

__clib_export void *
_vec_resize_internal (void *v, uword n_elts, const vec_attr_t *const attr)
{
  uword elt_sz = attr->elt_sz;
  if (PREDICT_TRUE (v != 0))
    {
      uword hs = _vec_find (v)->hdr_size * VEC_MIN_ALIGN;
      uword alloc_sz = clib_mem_size (v - hs);
      uword n_data_bytes = elt_sz * n_elts;
      word unused_bytes = alloc_sz - (n_data_bytes + hs);

      if (PREDICT_TRUE (unused_bytes >= 0))
	{
	  _vec_update_len (v, n_elts, elt_sz, n_data_bytes, unused_bytes);
	  return v;
	}
    }

  /* this shouled emit tail jump and likely avoid stack usasge inside this
   * function */
  return _vec_realloc_internal (v, n_elts, attr);
}

__clib_export u32
vec_len_not_inline (void *v)
{
  return vec_len (v);
}

__clib_export void
vec_free_not_inline (void *v)
{
  vec_free (v);
}