src/vppinfra/mem_bulk.c

/*
 * Copyright (c) 2020 Cisco and/or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <vppinfra/clib.h>
#include <vppinfra/mem.h>
#include <vppinfra/time.h>
#include <vppinfra/format.h>
#include <vppinfra/clib_error.h>

/* while usage of dlmalloc APIs is genrally discouraged, in this particular
 * case there is significant benefit of calling them directly due to
 * smaller memory consuption (no wwp and headroom space) */
#include <vppinfra/dlmalloc.h>

#define CLIB_MEM_BULK_DEFAULT_MIN_ELTS_PER_CHUNK 32

typedef struct clib_mem_bulk_chunk_hdr
{
  u32 freelist;
  u32 n_free;
  struct clib_mem_bulk_chunk_hdr *prev, *next;
} clib_mem_bulk_chunk_hdr_t;

typedef struct
{
  u32 elt_sz;
  u32 chunk_hdr_sz;
  u32 elts_per_chunk;
  u32 align;
  u32 chunk_align;
  void *mspace;
  clib_mem_bulk_chunk_hdr_t *full_chunks, *avail_chunks;
} clib_mem_bulk_t;

static inline uword
bulk_chunk_size (clib_mem_bulk_t *b)
{
  return (uword) b->elts_per_chunk * b->elt_sz + b->chunk_hdr_sz;
}

__clib_export clib_mem_bulk_handle_t
clib_mem_bulk_init (u32 elt_sz, u32 align, u32 min_elts_per_chunk)
{
  clib_mem_heap_t *heap = clib_mem_get_heap ();
  clib_mem_bulk_t *b;
  uword sz;

  if ((b = mspace_memalign (heap->mspace, 16, sizeof (clib_mem_bulk_t))) == 0)
    return 0;

  if (align < 16)
    align = 16;

  if (min_elts_per_chunk == 0)
    min_elts_per_chunk = CLIB_MEM_BULK_DEFAULT_MIN_ELTS_PER_CHUNK;

  CLIB_MEM_UNPOISON (b, sizeof (clib_mem_bulk_t));
  clib_memset (b, 0, sizeof (clib_mem_bulk_t));
  b->mspace = heap->mspace;
  b->align = align;
  b->elt_sz = round_pow2 (elt_sz, align);
  b->chunk_hdr_sz = round_pow2 (sizeof (clib_mem_bulk_chunk_hdr_t), align);
  b->elts_per_chunk = min_elts_per_chunk;
  sz = bulk_chunk_size (b);
  b->chunk_align = max_pow2 (sz);
  b->elts_per_chunk += (b->chunk_align - sz) / b->elt_sz;
  return b;
}

__clib_export void
clib_mem_bulk_destroy (clib_mem_bulk_handle_t h)
{
  clib_mem_bulk_t *b = h;
  clib_mem_bulk_chunk_hdr_t *c, *next;
  void *ms = b->mspace;

  c = b->full_chunks;

again:
  while (c)
    {
      next = c->next;
      CLIB_MEM_POISON (c, bulk_chunk_size (b));
      mspace_free (ms, c);
      c = next;
    }

  if (b->avail_chunks)
    {
      c = b->avail_chunks;
      b->avail_chunks = 0;
      goto again;
    }

  CLIB_MEM_POISON (b, sizeof (clib_mem_bulk_t));
  mspace_free (ms, b);
}

static inline void *
get_chunk_elt_ptr (clib_mem_bulk_t *b, clib_mem_bulk_chunk_hdr_t *c, u32 index)
{
  return (u8 *) c + b->chunk_hdr_sz + index * b->elt_sz;
}

static inline void
add_to_chunk_list (clib_mem_bulk_chunk_hdr_t **first,
		   clib_mem_bulk_chunk_hdr_t *c)
{
  c->next = *first;
  c->prev = 0;
  if (c->next)
    c->next->prev = c;
  *first = c;
}

static inline void
remove_from_chunk_list (clib_mem_bulk_chunk_hdr_t **first,
			clib_mem_bulk_chunk_hdr_t *c)
{
  if (c->next)
    c->next->prev = c->prev;
  if (c->prev)
    c->prev->next = c->next;
  else
    *first = c->next;
}

__clib_export void *
clib_mem_bulk_alloc (clib_mem_bulk_handle_t h)
{
  clib_mem_bulk_t *b = h;
  clib_mem_bulk_chunk_hdr_t *c = b->avail_chunks;
  u32 elt_idx;

  if (b->avail_chunks == 0)
    {
      u32 i, sz = bulk_chunk_size (b);
      c = mspace_memalign (b->mspace, b->chunk_align, sz);
      CLIB_MEM_UNPOISON (c, sz);
      clib_memset (c, 0, sizeof (clib_mem_bulk_chunk_hdr_t));
      b->avail_chunks = c;
      c->n_free = b->elts_per_chunk;

      /* populate freelist */
      for (i = 0; i < b->elts_per_chunk - 1; i++)
	*((u32 *) get_chunk_elt_ptr (b, c, i)) = i + 1;
      *((u32 *) get_chunk_elt_ptr (b, c, i)) = ~0;
    }

  ASSERT (c->freelist != ~0);
  elt_idx = c->freelist;
  c->freelist = *((u32 *) get_chunk_elt_ptr (b, c, elt_idx));
  c->n_free--;

  if (c->n_free == 0)
    {
      /* chunk is full */
      ASSERT (c->freelist == ~0);
      remove_from_chunk_list (&b->avail_chunks, c);
      add_to_chunk_list (&b->full_chunks, c);
    }

  return get_chunk_elt_ptr (b, c, elt_idx);
}

__clib_export void
clib_mem_bulk_free (clib_mem_bulk_handle_t h, void *p)
{
  clib_mem_bulk_t *b = h;
  uword offset = (uword) p & (b->chunk_align - 1);
  clib_mem_bulk_chunk_hdr_t *c = (void *) ((u8 *) p - offset);
  u32 elt_idx = (offset - b->chunk_hdr_sz) / b->elt_sz;

  ASSERT (elt_idx < b->elts_per_chunk);
  ASSERT (get_chunk_elt_ptr (b, c, elt_idx) == p);

  c->n_free++;

  if (c->n_free == b->elts_per_chunk)
    {
      /* chunk is empty - give it back */
      remove_from_chunk_list (&b->avail_chunks, c);
      CLIB_MEM_POISON (c, bulk_chunk_size (b));
      mspace_free (b->mspace, c);
      return;
    }

  if (c->n_free == 1)
    {
      /* move chunk to avail chunks */
      remove_from_chunk_list (&b->full_chunks, c);
      add_to_chunk_list (&b->avail_chunks, c);
    }

  /* add elt to freelist */
  *(u32 *) p = c->freelist;
  c->freelist = elt_idx;
}

__clib_export u8 *
format_clib_mem_bulk (u8 *s, va_list *args)
{
  clib_mem_bulk_t *b = va_arg (*args, clib_mem_bulk_handle_t);
  clib_mem_bulk_chunk_hdr_t *c;
  uword n_chunks = 0, n_free_elts = 0, n_elts, chunk_sz;

  c = b->full_chunks;
  while (c)
    {
      n_chunks++;
      c = c->next;
    }

  c = b->avail_chunks;
  while (c)
    {
      n_chunks++;
      n_free_elts += c->n_free;
      c = c->next;
    }

  n_elts = n_chunks * b->elts_per_chunk;
  chunk_sz = b->chunk_hdr_sz + (uword) b->elts_per_chunk * b->elt_sz;

  s = format (s, "%u bytes/elt, align %u, chunk-align %u, ", b->elt_sz,
	      b->align, b->chunk_align);
  s = format (s, "%u elts-per-chunk, chunk size %lu bytes", b->elts_per_chunk,
	      chunk_sz);

  if (n_chunks == 0)
    return format (s, "\nempty");

  s = format (s, "\n%lu chunks allocated, ", n_chunks);
  s = format (s, "%lu / %lu free elts (%.1f%%), ", n_free_elts, n_elts,
	      (f64) n_free_elts * 100 / n_elts);
  s = format (s, "%lu bytes of memory consumed", n_chunks * chunk_sz);

  return s;
}