session: segment manager refactor
- use valloc as a 'central' segment baseva manager
- use per segment manager segment pools and use rwlocks to guard them
- add session test that exercises segment creation
- embed segment manager properties into application since they're shared
- fix rw locks
Change-Id: I761164c147275d9e8a926f1eda395e090d231f9a
Signed-off-by: Florin Coras <fcoras@cisco.com>
diff --git a/src/svm/ssvm.c b/src/svm/ssvm.c
index 5055219..7077f8b 100644
--- a/src/svm/ssvm.c
+++ b/src/svm/ssvm.c
@@ -19,11 +19,11 @@
typedef void (*delete_fn) (ssvm_private_t *);
static init_fn master_init_fns[SSVM_N_SEGMENT_TYPES] =
- { ssvm_master_init_shm, ssvm_master_init_memfd };
+ { ssvm_master_init_shm, ssvm_master_init_memfd, ssvm_master_init_private };
static init_fn slave_init_fns[SSVM_N_SEGMENT_TYPES] =
- { ssvm_slave_init_shm, ssvm_slave_init_memfd };
+ { ssvm_slave_init_shm, ssvm_slave_init_memfd, ssvm_slave_init_private };
static delete_fn delete_fns[SSVM_N_SEGMENT_TYPES] =
- { ssvm_delete_shm, ssvm_delete_memfd };
+ { ssvm_delete_shm, ssvm_delete_memfd, ssvm_delete_private };
int
ssvm_master_init_shm (ssvm_private_t * ssvm)
@@ -315,6 +315,59 @@
close (memfd->fd);
}
+/**
+ * Initialize segment in a private heap
+ */
+int
+ssvm_master_init_private (ssvm_private_t * ssvm)
+{
+ u32 pagesize = clib_mem_get_page_size ();
+ ssvm_shared_header_t *sh;
+ mheap_t *heap_header;
+ u32 rnd_size = 0;
+ u8 *heap;
+
+ rnd_size = (ssvm->ssvm_size + (pagesize - 1)) & ~pagesize;
+ heap = mheap_alloc (0, rnd_size);
+ if (heap == 0)
+ {
+ clib_unix_warning ("mheap alloc");
+ return -1;
+ }
+ heap_header = mheap_header (heap);
+ heap_header->flags |= MHEAP_FLAG_THREAD_SAFE;
+
+ ssvm->ssvm_size = rnd_size;
+ ssvm->i_am_master = 1;
+ ssvm->my_pid = getpid ();
+ ssvm->requested_va = ~0;
+
+ /* Allocate a [sic] shared memory header, in process memory... */
+ sh = clib_mem_alloc_aligned (sizeof (*sh), CLIB_CACHE_LINE_BYTES);
+ ssvm->sh = sh;
+
+ memset (sh, 0, sizeof (*sh));
+ sh->heap = heap;
+ sh->type = SSVM_SEGMENT_PRIVATE;
+
+ return 0;
+}
+
+int
+ssvm_slave_init_private (ssvm_private_t * ssvm)
+{
+ clib_warning ("BUG: this should not be called!");
+ return -1;
+}
+
+void
+ssvm_delete_private (ssvm_private_t * ssvm)
+{
+ vec_free (ssvm->name);
+ mheap_free (ssvm->sh->heap);
+ clib_mem_free (ssvm->sh);
+}
+
int
ssvm_master_init (ssvm_private_t * ssvm, ssvm_segment_type_t type)
{
diff --git a/src/svm/ssvm.h b/src/svm/ssvm.h
index 9bf009e..09e1707 100644
--- a/src/svm/ssvm.h
+++ b/src/svm/ssvm.h
@@ -49,6 +49,7 @@
{
SSVM_SEGMENT_SHM = 0,
SSVM_SEGMENT_MEMFD,
+ SSVM_SEGMENT_PRIVATE,
SSVM_N_SEGMENT_TYPES /**< Private segments */
} ssvm_segment_type_t;
@@ -195,6 +196,10 @@
int ssvm_slave_init_memfd (ssvm_private_t * memfd);
void ssvm_delete_memfd (ssvm_private_t * memfd);
+int ssvm_master_init_private (ssvm_private_t * ssvm);
+int ssvm_slave_init_private (ssvm_private_t * ssvm);
+void ssvm_delete_private (ssvm_private_t * ssvm);
+
ssvm_segment_type_t ssvm_type (const ssvm_private_t * ssvm);
u8 *ssvm_name (const ssvm_private_t * ssvm);
diff --git a/src/svm/svm_fifo_segment.c b/src/svm/svm_fifo_segment.c
index 4d1833e..459b1d1 100644
--- a/src/svm/svm_fifo_segment.c
+++ b/src/svm/svm_fifo_segment.c
@@ -56,69 +56,75 @@
}
}
-static void
-preallocate_fifo_pairs (svm_fifo_segment_private_t * s,
- svm_fifo_segment_create_args_t * a, u32 protect_size)
+/**
+ * Pre-allocates fifo pairs in fifo segment
+ *
+ * The number of fifos pre-allocated is the minimum of the requested number
+ * of pairs and the maximum number that fit within the segment. If the maximum
+ * is hit, the number of fifo pairs requested is updated by subtracting the
+ * number of fifos that have been successfully allocated.
+ */
+void
+svm_fifo_segment_preallocate_fifo_pairs (svm_fifo_segment_private_t * s,
+ u32 rx_fifo_size, u32 tx_fifo_size,
+ u32 * n_fifo_pairs)
{
- svm_fifo_segment_header_t *fsh = s->h;
- u32 rx_fifo_size, tx_fifo_size, pairs_to_allocate;
u32 rx_rounded_data_size, tx_rounded_data_size, pair_size;
- svm_fifo_t *f;
- u8 *rx_fifo_space, *tx_fifo_space;
+ u32 rx_fifos_size, tx_fifos_size, pairs_to_allocate;
int rx_freelist_index, tx_freelist_index;
+ ssvm_shared_header_t *sh = s->ssvm.sh;
+ svm_fifo_segment_header_t *fsh = s->h;
+ u8 *rx_fifo_space, *tx_fifo_space;
+ uword space_available;
+ void *oldheap;
+ svm_fifo_t *f;
int i;
/* Parameter check */
- if (a->rx_fifo_size == 0 || a->tx_fifo_size == 0
- || a->preallocated_fifo_pairs == 0)
+ if (rx_fifo_size == 0 || tx_fifo_size == 0 || *n_fifo_pairs == 0)
return;
- if (a->rx_fifo_size < FIFO_SEGMENT_MIN_FIFO_SIZE ||
- a->rx_fifo_size > FIFO_SEGMENT_MAX_FIFO_SIZE)
+ if (rx_fifo_size < FIFO_SEGMENT_MIN_FIFO_SIZE ||
+ rx_fifo_size > FIFO_SEGMENT_MAX_FIFO_SIZE)
{
- clib_warning ("rx fifo_size out of range %d", a->rx_fifo_size);
+ clib_warning ("rx fifo_size out of range %d", rx_fifo_size);
return;
}
- if (a->tx_fifo_size < FIFO_SEGMENT_MIN_FIFO_SIZE ||
- a->tx_fifo_size > FIFO_SEGMENT_MAX_FIFO_SIZE)
+ if (tx_fifo_size < FIFO_SEGMENT_MIN_FIFO_SIZE ||
+ tx_fifo_size > FIFO_SEGMENT_MAX_FIFO_SIZE)
{
- clib_warning ("tx fifo_size out of range %d", a->rx_fifo_size);
+ clib_warning ("tx fifo_size out of range %d", rx_fifo_size);
return;
}
- rx_rounded_data_size = (1 << (max_log2 (a->rx_fifo_size)));
-
- rx_freelist_index = max_log2 (a->rx_fifo_size)
+ rx_rounded_data_size = (1 << (max_log2 (rx_fifo_size)));
+ rx_freelist_index = max_log2 (rx_fifo_size)
- max_log2 (FIFO_SEGMENT_MIN_FIFO_SIZE);
-
- tx_rounded_data_size = (1 << (max_log2 (a->rx_fifo_size)));
-
- tx_freelist_index = max_log2 (a->tx_fifo_size)
+ tx_rounded_data_size = (1 << (max_log2 (tx_fifo_size)));
+ tx_freelist_index = max_log2 (tx_fifo_size)
- max_log2 (FIFO_SEGMENT_MIN_FIFO_SIZE);
/* Calculate space requirements */
pair_size = 2 * sizeof (*f) + rx_rounded_data_size + tx_rounded_data_size;
- if (protect_size)
- protect_size += mheap_bytes (s->ssvm.sh->heap);
- pairs_to_allocate =
- clib_min ((s->ssvm.ssvm_size - protect_size) / pair_size,
- a->preallocated_fifo_pairs);
- rx_fifo_size = (sizeof (*f) + rx_rounded_data_size) * pairs_to_allocate;
- tx_fifo_size = (sizeof (*f) + tx_rounded_data_size) * pairs_to_allocate;
+ space_available = s->ssvm.ssvm_size - mheap_bytes (sh->heap);
+ pairs_to_allocate = clib_min (space_available / pair_size, *n_fifo_pairs);
+ rx_fifos_size = (sizeof (*f) + rx_rounded_data_size) * pairs_to_allocate;
+ tx_fifos_size = (sizeof (*f) + tx_rounded_data_size) * pairs_to_allocate;
vec_validate_init_empty (fsh->free_fifos,
clib_max (rx_freelist_index, tx_freelist_index),
0);
+ oldheap = ssvm_push_heap (sh);
/* Allocate rx fifo space. May fail. */
rx_fifo_space = clib_mem_alloc_aligned_at_offset
- (rx_fifo_size, CLIB_CACHE_LINE_BYTES, 0 /* align_offset */ ,
+ (rx_fifos_size, CLIB_CACHE_LINE_BYTES, 0 /* align_offset */ ,
0 /* os_out_of_memory */ );
/* Same for TX */
tx_fifo_space = clib_mem_alloc_aligned_at_offset
- (tx_fifo_size, CLIB_CACHE_LINE_BYTES, 0 /* align_offset */ ,
+ (tx_fifos_size, CLIB_CACHE_LINE_BYTES, 0 /* align_offset */ ,
0 /* os_out_of_memory */ );
/* Make sure it worked. Clean up if it didn't... */
@@ -128,13 +134,14 @@
clib_mem_free (rx_fifo_space);
else
clib_warning ("rx fifo preallocation failure: size %d npairs %d",
- a->rx_fifo_size, a->preallocated_fifo_pairs);
+ rx_fifo_size, *n_fifo_pairs);
if (tx_fifo_space)
clib_mem_free (tx_fifo_space);
else
clib_warning ("tx fifo preallocation failure: size %d nfifos %d",
- a->tx_fifo_size, a->preallocated_fifo_pairs);
+ tx_fifo_size, *n_fifo_pairs);
+ ssvm_pop_heap (oldheap);
return;
}
@@ -160,7 +167,31 @@
}
/* Account for the pairs allocated */
- a->preallocated_fifo_pairs -= pairs_to_allocate;
+ *n_fifo_pairs -= pairs_to_allocate;
+ ssvm_pop_heap (oldheap);
+}
+
+/**
+ * Initialize svm fifo segment shared header
+ */
+int
+svm_fifo_segment_init (svm_fifo_segment_private_t * s)
+{
+ svm_fifo_segment_header_t *fsh;
+ ssvm_shared_header_t *sh;
+ void *oldheap;
+
+ sh = s->ssvm.sh;
+ oldheap = ssvm_push_heap (sh);
+
+ fsh = clib_mem_alloc (sizeof (*fsh));
+ memset (fsh, 0, sizeof (*fsh));
+ s->h = sh->opaque[0] = fsh;
+
+ ssvm_pop_heap (oldheap);
+
+ sh->ready = 1;
+ return (0);
}
/**
@@ -171,9 +202,6 @@
{
svm_fifo_segment_main_t *sm = &svm_fifo_segment_main;
svm_fifo_segment_private_t *s;
- ssvm_shared_header_t *sh;
- svm_fifo_segment_header_t *fsh;
- void *oldheap;
int rv;
/* Allocate a fresh segment */
@@ -195,18 +223,7 @@
/* Note: requested_va updated due to seg base addr randomization */
sm->next_baseva = s->ssvm.sh->ssvm_va + a->segment_size;
- sh = s->ssvm.sh;
- oldheap = ssvm_push_heap (sh);
-
- /* Set up svm_fifo_segment shared header */
- fsh = clib_mem_alloc (sizeof (*fsh));
- memset (fsh, 0, sizeof (*fsh));
- s->h = sh->opaque[0] = fsh;
- preallocate_fifo_pairs (s, a, a->seg_protected_space);
-
- ssvm_pop_heap (oldheap);
-
- sh->ready = 1;
+ svm_fifo_segment_init (s);
vec_add1 (a->new_segment_indices, s - sm->segments);
return (0);
}
@@ -217,78 +234,45 @@
int
svm_fifo_segment_create_process_private (svm_fifo_segment_create_args_t * a)
{
- svm_fifo_segment_private_t *s;
svm_fifo_segment_main_t *sm = &svm_fifo_segment_main;
+ svm_fifo_segment_private_t *s;
ssvm_shared_header_t *sh;
- svm_fifo_segment_header_t *fsh;
- void *oldheap;
- u8 **heaps = 0;
mheap_t *heap_header;
- int segment_count = 1;
u32 rnd_size = 0;
- int i;
+ u8 *heap;
+ u32 pagesize = clib_mem_get_page_size ();
- if (a->private_segment_count)
+ pool_get (sm->segments, s);
+ memset (s, 0, sizeof (*s));
+
+ rnd_size = (a->segment_size + (pagesize - 1)) & ~pagesize;
+
+ heap = mheap_alloc (0, rnd_size);
+ if (heap == 0)
{
- u8 *heap;
- u32 pagesize = clib_mem_get_page_size ();
- rnd_size = (a->segment_size + (pagesize - 1)) & ~pagesize;
-
- for (i = 0; i < a->private_segment_count; i++)
- {
- heap = mheap_alloc (0, rnd_size);
- if (heap == 0)
- {
- clib_unix_warning ("mheap alloc");
- return -1;
- }
- heap_header = mheap_header (heap);
- heap_header->flags |= MHEAP_FLAG_THREAD_SAFE;
- vec_add1 (heaps, heap);
- }
- segment_count = a->private_segment_count;
+ clib_unix_warning ("mheap alloc");
+ pool_put (sm->segments, s);
+ return -1;
}
+ heap_header = mheap_header (heap);
+ heap_header->flags |= MHEAP_FLAG_THREAD_SAFE;
- /* Allocate segments */
- for (i = 0; i < segment_count; i++)
- {
- pool_get (sm->segments, s);
- memset (s, 0, sizeof (*s));
+ s->ssvm.ssvm_size = rnd_size;
+ s->ssvm.i_am_master = 1;
+ s->ssvm.my_pid = getpid ();
+ s->ssvm.name = format (0, "%s%c", a->segment_name, 0);
+ s->ssvm.requested_va = ~0;
- s->ssvm.ssvm_size = rnd_size;
- s->ssvm.i_am_master = 1;
- s->ssvm.my_pid = getpid ();
- s->ssvm.name = format (0, "%s%c", a->segment_name, 0);
- s->ssvm.requested_va = ~0;
+ /* Allocate a [sic] shared memory header, in process memory... */
+ sh = clib_mem_alloc_aligned (sizeof (*sh), CLIB_CACHE_LINE_BYTES);
+ s->ssvm.sh = sh;
- /* Allocate a [sic] shared memory header, in process memory... */
- sh = clib_mem_alloc_aligned (sizeof (*sh), CLIB_CACHE_LINE_BYTES);
- s->ssvm.sh = sh;
+ memset (sh, 0, sizeof (*sh));
+ sh->heap = heap;
- memset (sh, 0, sizeof (*sh));
- sh->heap = a->private_segment_count ? heaps[i] : clib_mem_get_heap ();
+ svm_fifo_segment_init (s);
+ vec_add1 (a->new_segment_indices, s - sm->segments);
- /* Set up svm_fifo_segment shared header */
- fsh = clib_mem_alloc (sizeof (*fsh));
- memset (fsh, 0, sizeof (*fsh));
- fsh->flags = FIFO_SEGMENT_F_IS_PRIVATE;
- s->h = sh->opaque[0] = fsh;
- if (!a->private_segment_count)
- fsh->flags |= FIFO_SEGMENT_F_IS_MAIN_HEAP;
-
- if (a->private_segment_count)
- {
- if (i != 0)
- fsh->flags |= FIFO_SEGMENT_F_IS_PREALLOCATED;
- oldheap = clib_mem_get_heap ();
- clib_mem_set_heap (sh->heap);
- preallocate_fifo_pairs (s, a, i == 0 ? a->seg_protected_space : 0);
- clib_mem_set_heap (oldheap);
- }
- sh->ready = 1;
- vec_add1 (a->new_segment_indices, s - sm->segments);
- }
- vec_free (heaps);
return (0);
}
@@ -333,18 +317,7 @@
{
svm_fifo_segment_main_t *sm = &svm_fifo_segment_main;
- if (s->h->flags & FIFO_SEGMENT_F_IS_PRIVATE)
- {
- /* Don't try to free vpp's heap! */
- if (!(s->h->flags & FIFO_SEGMENT_F_IS_MAIN_HEAP))
- mheap_free (s->ssvm.sh->heap);
- clib_mem_free (s->ssvm.sh);
- clib_mem_free (s->h);
- }
- else
- {
- ssvm_delete (&s->ssvm);
- }
+ ssvm_delete (&s->ssvm);
memset (s, 0xfe, sizeof (*s));
pool_put (sm->segments, s);
}
@@ -378,10 +351,10 @@
- max_log2 (FIFO_SEGMENT_MIN_FIFO_SIZE);
sh = s->ssvm.sh;
- fsh = (svm_fifo_segment_header_t *) sh->opaque[0];
ssvm_lock_non_recursive (sh, 1);
oldheap = ssvm_push_heap (sh);
+ fsh = (svm_fifo_segment_header_t *) sh->opaque[0];
switch (list_index)
{
@@ -508,7 +481,7 @@
}
void
-svm_fifo_segment_init (u64 baseva, u32 timeout_in_seconds)
+svm_fifo_segment_main_init (u64 baseva, u32 timeout_in_seconds)
{
svm_fifo_segment_main_t *sm = &svm_fifo_segment_main;
@@ -595,7 +568,7 @@
svm_fifo_segment_info (svm_fifo_segment_private_t * seg, uword * address,
u64 * size)
{
- if (seg->h->flags & FIFO_SEGMENT_F_IS_PRIVATE)
+ if (ssvm_type (&seg->ssvm) == SSVM_SEGMENT_PRIVATE)
{
mheap_t *heap_header;
@@ -623,12 +596,8 @@
sp = va_arg (*args, svm_fifo_segment_private_t *);
ssvm_segment_type_t st = ssvm_type (&sp->ssvm);
- if ((sp->h->flags & FIFO_SEGMENT_F_IS_PRIVATE)
- && !(sp->h->flags & FIFO_SEGMENT_F_IS_MAIN_HEAP))
+ if (st == SSVM_SEGMENT_PRIVATE)
s = format (s, "%s", "private-heap");
- else if ((sp->h->flags & FIFO_SEGMENT_F_IS_PRIVATE)
- && (sp->h->flags & FIFO_SEGMENT_F_IS_MAIN_HEAP))
- s = format (s, "%s", "main-heap");
else if (st == SSVM_SEGMENT_MEMFD)
s = format (s, "%s", "memfd");
else if (st == SSVM_SEGMENT_SHM)
diff --git a/src/svm/svm_fifo_segment.h b/src/svm/svm_fifo_segment.h
index 0e86c25..bf8d513 100644
--- a/src/svm/svm_fifo_segment.h
+++ b/src/svm/svm_fifo_segment.h
@@ -31,10 +31,8 @@
#define FIFO_SEGMENT_MAX_FIFO_SIZE (8<<20) /* 8mb max fifo size */
#define FIFO_SEGMENT_ALLOC_CHUNK_SIZE 32 /* Allocation quantum */
-#define FIFO_SEGMENT_F_IS_PRIVATE 1 << 0 /* Private segment */
-#define FIFO_SEGMENT_F_IS_MAIN_HEAP 1 << 1 /* Segment is main heap */
-#define FIFO_SEGMENT_F_IS_PREALLOCATED 1 << 2 /* Segment is preallocated */
-#define FIFO_SEGMENT_F_WILL_DELETE 1 << 3 /* Segment will be removed */
+#define FIFO_SEGMENT_F_IS_PREALLOCATED 1 << 0 /* Segment is preallocated */
+#define FIFO_SEGMENT_F_WILL_DELETE 1 << 1 /* Segment will be removed */
typedef struct
{
@@ -69,14 +67,11 @@
char *segment_name;
u32 segment_size;
u32 *new_segment_indices;
- u32 rx_fifo_size;
- u32 tx_fifo_size;
- u32 preallocated_fifo_pairs;
- u32 private_segment_count;
- u32 seg_protected_space;
int memfd_fd;
} svm_fifo_segment_create_args_t;
+#define svm_fifo_segment_flags(_seg) _seg->h->flags
+
static inline svm_fifo_segment_private_t *
svm_fifo_segment_get_segment (u32 segment_index)
{
@@ -96,19 +91,14 @@
return fifo_segment->h->fifos;
}
-#define foreach_ssvm_fifo_segment_api_error \
-_(OUT_OF_SPACE, "Out of space in segment", -200)
-
-typedef enum
-{
-#define _(n,s,c) SSVM_FIFO_SEGMENT_API_ERROR_##n = c,
- foreach_ssvm_fifo_segment_api_error
-#undef _
-} ssvm_fifo_segment_api_error_enum_t;
-
+int svm_fifo_segment_init (svm_fifo_segment_private_t * s);
int svm_fifo_segment_create (svm_fifo_segment_create_args_t * a);
int svm_fifo_segment_create_process_private (svm_fifo_segment_create_args_t
* a);
+void svm_fifo_segment_preallocate_fifo_pairs (svm_fifo_segment_private_t * s,
+ u32 rx_fifo_size,
+ u32 tx_fifo_size,
+ u32 * n_fifo_pairs);
int svm_fifo_segment_attach (svm_fifo_segment_create_args_t * a);
void svm_fifo_segment_delete (svm_fifo_segment_private_t * s);
@@ -118,7 +108,7 @@
void svm_fifo_segment_free_fifo (svm_fifo_segment_private_t * s,
svm_fifo_t * f,
svm_fifo_segment_freelist_t index);
-void svm_fifo_segment_init (u64 baseva, u32 timeout_in_seconds);
+void svm_fifo_segment_main_init (u64 baseva, u32 timeout_in_seconds);
u32 svm_fifo_segment_index (svm_fifo_segment_private_t * s);
u32 svm_fifo_segment_num_fifos (svm_fifo_segment_private_t * fifo_segment);
u32 svm_fifo_segment_num_free_fifos (svm_fifo_segment_private_t *
diff --git a/src/svm/test_svm_fifo1.c b/src/svm/test_svm_fifo1.c
index 3bdca94..d5b2c98 100644
--- a/src/svm/test_svm_fifo1.c
+++ b/src/svm/test_svm_fifo1.c
@@ -30,9 +30,6 @@
a->segment_name = "fifo-test1";
a->segment_size = 256 << 10;
- a->rx_fifo_size = 4096;
- a->tx_fifo_size = 4096;
- a->preallocated_fifo_pairs = 4;
rv = svm_fifo_segment_create (a);
@@ -276,7 +273,7 @@
int verbose = 0;
int test_id = 0;
- svm_fifo_segment_init (0x200000000ULL, 20);
+ svm_fifo_segment_main_init (0x200000000ULL, 20);
while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
{
diff --git a/src/tests/vnet/session/tcp_echo.c b/src/tests/vnet/session/tcp_echo.c
index a5ad35a..d48c5d9 100644
--- a/src/tests/vnet/session/tcp_echo.c
+++ b/src/tests/vnet/session/tcp_echo.c
@@ -1277,7 +1277,7 @@
clib_time_init (&em->clib_time);
init_error_string_table (em);
- svm_fifo_segment_init (0x200000000ULL, 20);
+ svm_fifo_segment_main_init (0x200000000ULL, 20);
unformat_init_command_line (a, argv);
while (unformat_check_input (a) != UNFORMAT_END_OF_INPUT)
diff --git a/src/tests/vnet/session/udp_echo.c b/src/tests/vnet/session/udp_echo.c
index 2838951..e425169 100644
--- a/src/tests/vnet/session/udp_echo.c
+++ b/src/tests/vnet/session/udp_echo.c
@@ -1071,7 +1071,7 @@
clib_time_init (&utm->clib_time);
init_error_string_table (utm);
- svm_fifo_segment_init (0x200000000ULL, 20);
+ svm_fifo_segment_main_init (0x200000000ULL, 20);
unformat_init_command_line (a, argv);
while (unformat_check_input (a) != UNFORMAT_END_OF_INPUT)
diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c
index 167a745..dd7e90e 100644
--- a/src/vcl/vppcom.c
+++ b/src/vcl/vppcom.c
@@ -2304,8 +2304,8 @@
clib_time_init (&vcm->clib_time);
vppcom_init_error_string_table ();
- svm_fifo_segment_init (vcl_cfg->segment_baseva,
- 20 /* timeout in secs */ );
+ svm_fifo_segment_main_init (vcl_cfg->segment_baseva,
+ 20 /* timeout in secs */ );
clib_spinlock_init (&vcm->sessions_lockp);
}
@@ -2722,7 +2722,7 @@
{
session_t *listen_session = 0;
session_t *client_session = 0;
- u32 client_session_index = ~0;
+ u32 client_session_index = ~0, n_fifos;
int rv;
f64 wait_for;
char *cut_thru_str;
@@ -2821,9 +2821,6 @@
format ((u8 *) a->segment_name, "%d:segment%d%c",
getpid (), vcm->unique_segment_index++, 0);
a->segment_size = vcm->cfg.segment_size;
- a->preallocated_fifo_pairs = vcm->cfg.preallocated_fifo_pairs;
- a->rx_fifo_size = vcm->cfg.rx_fifo_size;
- a->tx_fifo_size = vcm->cfg.tx_fifo_size;
rv = svm_fifo_segment_create (a);
if (PREDICT_FALSE (rv))
@@ -2850,6 +2847,14 @@
vec_free (a->new_segment_indices);
seg = svm_fifo_segment_get_segment (client_session->sm_seg_index);
+ if (vcm->cfg.preallocated_fifo_pairs)
+ {
+ n_fifos = vcm->cfg.preallocated_fifo_pairs;
+ svm_fifo_segment_preallocate_fifo_pairs (seg, vcm->cfg.rx_fifo_size,
+ vcm->cfg.tx_fifo_size,
+ &n_fifos);
+ }
+
client_session->server_rx_fifo =
svm_fifo_segment_alloc_fifo (seg, vcm->cfg.rx_fifo_size,
FIFO_SEGMENT_RX_FREELIST);
diff --git a/src/vnet/session-apps/echo_client.c b/src/vnet/session-apps/echo_client.c
index c38b533..27e253a 100644
--- a/src/vnet/session-apps/echo_client.c
+++ b/src/vnet/session-apps/echo_client.c
@@ -413,13 +413,21 @@
return 0;
}
+int
+echo_client_add_segment_callback (u32 client_index, const ssvm_private_t * sp)
+{
+ /* New heaps may be added */
+ return 0;
+}
+
/* *INDENT-OFF* */
static session_cb_vft_t echo_clients = {
.session_reset_callback = echo_clients_session_reset_callback,
.session_connected_callback = echo_clients_session_connected_callback,
.session_accept_callback = echo_clients_session_create_callback,
.session_disconnect_callback = echo_clients_session_disconnect_callback,
- .builtin_server_rx_callback = echo_clients_rx_callback
+ .builtin_server_rx_callback = echo_clients_rx_callback,
+ .add_segment_callback = echo_client_add_segment_callback
};
/* *INDENT-ON* */
@@ -445,6 +453,7 @@
options[APP_OPTIONS_ACCEPT_COOKIE] = 0x12345678;
options[APP_OPTIONS_SEGMENT_SIZE] = segment_size;
+ options[APP_OPTIONS_ADD_SEGMENT_SIZE] = segment_size;
options[APP_OPTIONS_RX_FIFO_SIZE] = ecm->fifo_size;
options[APP_OPTIONS_TX_FIFO_SIZE] = ecm->fifo_size;
options[APP_OPTIONS_PRIVATE_SEGMENT_COUNT] = ecm->private_segment_count;
@@ -625,7 +634,7 @@
else if (unformat (input, "test-bytes"))
ecm->test_bytes = 1;
else
- return clib_error_return (0, "unknown input `%U'",
+ return clib_error_return (0, "failed: unknown input `%U'",
format_unformat_error, input);
}
diff --git a/src/vnet/session-apps/echo_server.c b/src/vnet/session-apps/echo_server.c
index 37a51d5..024ffa6 100644
--- a/src/vnet/session-apps/echo_server.c
+++ b/src/vnet/session-apps/echo_server.c
@@ -96,8 +96,8 @@
int
echo_server_add_segment_callback (u32 client_index, const ssvm_private_t * sp)
{
- clib_warning ("called...");
- return -1;
+ /* New heaps may be added */
+ return 0;
}
int
@@ -290,6 +290,7 @@
a->session_cb_vft = &echo_server_session_cb_vft;
a->options = options;
a->options[APP_OPTIONS_SEGMENT_SIZE] = segment_size;
+ a->options[APP_OPTIONS_ADD_SEGMENT_SIZE] = segment_size;
a->options[APP_OPTIONS_RX_FIFO_SIZE] = esm->fifo_size;
a->options[APP_OPTIONS_TX_FIFO_SIZE] = esm->fifo_size;
a->options[APP_OPTIONS_PRIVATE_SEGMENT_COUNT] = esm->private_segment_count;
diff --git a/src/vnet/session/application.c b/src/vnet/session/application.c
index bd708fc..71fc93f 100644
--- a/src/vnet/session/application.c
+++ b/src/vnet/session/application.c
@@ -146,7 +146,6 @@
void
application_del (application_t * app)
{
- segment_manager_properties_t *props;
vnet_unbind_args_t _a, *a = &_a;
segment_manager_t *sm;
u64 handle, *handles = 0;
@@ -208,8 +207,6 @@
segment_manager_del (sm);
}
}
- props = segment_manager_properties_get (app->sm_properties);
- segment_manager_properties_free (props);
application_table_del (app);
pool_put (app_pool, app);
}
@@ -258,8 +255,8 @@
application_init (application_t * app, u32 api_client_index, u64 * options,
session_cb_vft_t * cb_fns)
{
- ssvm_segment_type_t st = SSVM_SEGMENT_MEMFD;
- u32 app_evt_queue_size, first_seg_size;
+ u32 app_evt_queue_size, first_seg_size, prealloc_fifo_pairs;
+ ssvm_segment_type_t seg_type = SSVM_SEGMENT_MEMFD;
segment_manager_properties_t *props;
vl_api_registration_t *reg;
segment_manager_t *sm;
@@ -268,16 +265,21 @@
/*
* Make sure we support the requested configuration
*/
- reg = vl_api_client_index_to_registration (api_client_index);
- if (!reg)
- return VNET_API_ERROR_APP_UNSUPPORTED_CFG;
- if (options[APP_OPTIONS_FLAGS] & APP_OPTIONS_FLAGS_IS_BUILTIN)
- st = SSVM_N_SEGMENT_TYPES;
- else if (vl_api_registration_file_index (reg) == VL_API_INVALID_FI)
- st = SSVM_SEGMENT_SHM;
+ if (!(options[APP_OPTIONS_FLAGS] & APP_OPTIONS_FLAGS_IS_BUILTIN))
+ {
+ reg = vl_api_client_index_to_registration (api_client_index);
+ if (!reg)
+ return VNET_API_ERROR_APP_UNSUPPORTED_CFG;
+ if (vl_api_registration_file_index (reg) == VL_API_INVALID_FI)
+ seg_type = SSVM_SEGMENT_SHM;
+ }
+ else
+ {
+ seg_type = SSVM_SEGMENT_PRIVATE;
+ }
- if (!application_verify_cfg (st))
+ if (!application_verify_cfg (seg_type))
return VNET_API_ERROR_APP_UNSUPPORTED_CFG;
/*
@@ -285,8 +287,8 @@
*/
sm = segment_manager_new ();
sm->app_index = app->index;
- props = segment_manager_properties_alloc ();
- app->sm_properties = segment_manager_properties_index (props);
+ props = application_segment_manager_properties (app);
+ segment_manager_properties_init (props);
if (options[APP_OPTIONS_ADD_SEGMENT_SIZE])
{
props->add_segment_size = options[APP_OPTIONS_ADD_SEGMENT_SIZE];
@@ -296,18 +298,15 @@
props->rx_fifo_size = options[APP_OPTIONS_RX_FIFO_SIZE];
if (options[APP_OPTIONS_TX_FIFO_SIZE])
props->tx_fifo_size = options[APP_OPTIONS_TX_FIFO_SIZE];
- props->preallocated_fifo_pairs = options[APP_OPTIONS_PREALLOC_FIFO_PAIRS];
- props->private_segment_count = options[APP_OPTIONS_PRIVATE_SEGMENT_COUNT];
- if (options[APP_OPTIONS_FLAGS] & APP_OPTIONS_FLAGS_IS_BUILTIN)
- props->segment_type = SSVM_N_SEGMENT_TYPES;
- else
- props->segment_type = st;
+ props->segment_type = seg_type;
app_evt_queue_size = options[APP_OPTIONS_EVT_QUEUE_SIZE] > 0 ?
options[APP_OPTIONS_EVT_QUEUE_SIZE] : default_app_evt_queue_size;
first_seg_size = options[APP_OPTIONS_SEGMENT_SIZE];
- if ((rv = segment_manager_init (sm, app->sm_properties, first_seg_size,
- app_evt_queue_size)))
+ prealloc_fifo_pairs = options[APP_OPTIONS_PREALLOC_FIFO_PAIRS];
+
+ if ((rv = segment_manager_init (sm, first_seg_size, app_evt_queue_size,
+ prealloc_fifo_pairs)))
return rv;
sm->first_is_protected = 1;
@@ -321,15 +320,13 @@
app->ns_index = options[APP_OPTIONS_NAMESPACE];
app->listeners_table = hash_create (0, sizeof (u64));
app->proxied_transports = options[APP_OPTIONS_PROXY_TRANSPORT];
+ app->event_queue = segment_manager_event_queue (sm);
/* If no scope enabled, default to global */
if (!application_has_global_scope (app)
&& !application_has_local_scope (app))
app->flags |= APP_OPTIONS_FLAGS_USE_GLOBAL_SCOPE;
- /* Allocate app event queue in the first shared-memory segment */
- app->event_queue = segment_manager_alloc_queue (sm, app_evt_queue_size);
-
/* Check that the obvious things are properly set up */
application_verify_cb_fns (cb_fns);
@@ -377,7 +374,7 @@
}
sm = segment_manager_new ();
- sm->properties_index = app->sm_properties;
+ sm->app_index = app->index;
return sm;
}
@@ -520,17 +517,14 @@
return (application_is_proxy (app) && application_is_builtin (app));
}
+/**
+ * Send an API message to the external app, to map new segment
+ */
int
-application_add_segment_notify (u32 app_index, u32 fifo_segment_index)
+application_add_segment_notify (u32 app_index, ssvm_private_t * fs)
{
application_t *app = application_get (app_index);
- svm_fifo_segment_private_t *fs;
-
- /* Send an API message to the external app, to map new segment */
- ASSERT (app->cb_fns.add_segment_callback);
-
- fs = segment_manager_get_segment (fifo_segment_index);
- return app->cb_fns.add_segment_callback (app->api_client_index, &fs->ssvm);
+ return app->cb_fns.add_segment_callback (app->api_client_index, fs);
}
u8
@@ -699,8 +693,6 @@
transport_proto_t tp;
ASSERT (application_is_proxy (app));
- if (application_is_builtin (app))
- return;
/* *INDENT-OFF* */
transport_proto_foreach (tp, ({
@@ -726,6 +718,19 @@
/* *INDENT-ON* */
}
+segment_manager_properties_t *
+application_segment_manager_properties (application_t * app)
+{
+ return &app->sm_properties;
+}
+
+segment_manager_properties_t *
+application_get_segment_manager_properties (u32 app_index)
+{
+ application_t *app = application_get (app_index);
+ return &app->sm_properties;
+}
+
u8 *
format_application_listener (u8 * s, va_list * args)
{
@@ -766,10 +771,10 @@
void
application_format_connects (application_t * app, int verbose)
{
+ svm_fifo_segment_private_t *fifo_segment;
vlib_main_t *vm = vlib_get_main ();
segment_manager_t *sm;
u8 *app_name, *s = 0;
- int j;
/* Header */
if (app == 0)
@@ -790,15 +795,14 @@
/* Across all fifo segments */
sm = segment_manager_get (app->connects_seg_manager);
- for (j = 0; j < vec_len (sm->segment_indices); j++)
- {
- svm_fifo_segment_private_t *fifo_segment;
- svm_fifo_t *fifo;
- u8 *str;
- fifo_segment = svm_fifo_segment_get_segment (sm->segment_indices[j]);
- fifo = svm_fifo_segment_get_fifo_list (fifo_segment);
- while (fifo)
+ /* *INDENT-OFF* */
+ segment_manager_foreach_segment_w_lock (fifo_segment, sm, ({
+ svm_fifo_t *fifo;
+ u8 *str;
+
+ fifo = svm_fifo_segment_get_fifo_list (fifo_segment);
+ while (fifo)
{
u32 session_index, thread_index;
stream_session_t *session;
@@ -821,8 +825,9 @@
fifo = fifo->next;
}
- vec_free (s);
- }
+ vec_free (s);
+ }));
+ /* *INDENT-ON* */
vec_free (app_name);
}
@@ -851,7 +856,7 @@
app_name = app_get_name_from_reg_index (app);
app_ns_name = app_namespace_id_from_index (app->ns_index);
- props = segment_manager_properties_get (app->sm_properties);
+ props = application_segment_manager_properties (app);
if (verbose)
s =
format (s, "%-10d%-20s%-15d%-15d%-15d%-15d%-15d", app->index, app_name,
@@ -894,21 +899,20 @@
if (pool_elts (app_pool))
{
vlib_cli_output (vm, "%U", format_application_listener,
- 0 /* header */ , 0, 0,
- verbose);
+ 0 /* header */ , 0, 0, verbose);
+
/* *INDENT-OFF* */
- pool_foreach (app, app_pool,
- ({
+ pool_foreach (app, app_pool, ({
/* App's listener sessions */
if (hash_elts (app->listeners_table) == 0)
continue;
- hash_foreach (handle, index, app->listeners_table,
- ({
+ hash_foreach (handle, index, app->listeners_table, ({
vlib_cli_output (vm, "%U", format_application_listener, app,
handle, index, verbose);
}));
}));
/* *INDENT-ON* */
+
}
else
vlib_cli_output (vm, "No active server bindings");
diff --git a/src/vnet/session/application.h b/src/vnet/session/application.h
index afe738f..36ae6fc 100644
--- a/src/vnet/session/application.h
+++ b/src/vnet/session/application.h
@@ -97,7 +97,7 @@
u8 first_segment_manager_in_use;
/** Segment manager properties. Shared by all segment managers */
- u32 sm_properties;
+ segment_manager_properties_t sm_properties;
u16 proxied_transports;
} application_t;
@@ -132,7 +132,7 @@
int application_is_proxy (application_t * app);
int application_is_builtin (application_t * app);
int application_is_builtin_proxy (application_t * app);
-int application_add_segment_notify (u32 app_index, u32 fifo_segment_index);
+int application_add_segment_notify (u32 app_index, ssvm_private_t * fs);
u32 application_session_table (application_t * app, u8 fib_proto);
u32 application_local_session_table (application_t * app);
u8 *application_name_from_index (u32 app_index);
@@ -146,6 +146,11 @@
void application_setup_proxy (application_t * app);
void application_remove_proxy (application_t * app);
+segment_manager_properties_t *application_get_segment_manager_properties (u32
+ app_index);
+segment_manager_properties_t
+ * application_segment_manager_properties (application_t * app);
+
#endif /* SRC_VNET_SESSION_APPLICATION_H_ */
/*
diff --git a/src/vnet/session/application_interface.c b/src/vnet/session/application_interface.c
index ad4c28b..efdd3dd 100644
--- a/src/vnet/session/application_interface.c
+++ b/src/vnet/session/application_interface.c
@@ -406,7 +406,7 @@
a->app_event_queue_address = pointer_to_uword (app->event_queue);
sm = segment_manager_get (app->first_segment_manager);
- fs = segment_manager_get_segment (sm->segment_indices[0]);
+ fs = segment_manager_get_segment_w_lock (sm, 0);
if (application_is_proxy (app))
application_setup_proxy (app);
@@ -415,6 +415,8 @@
a->segment = &fs->ssvm;
a->app_index = app->index;
+ segment_manager_segment_reader_unlock (sm);
+
return 0;
}
diff --git a/src/vnet/session/segment_manager.c b/src/vnet/session/segment_manager.c
index 83b8388..f8af3fb 100644
--- a/src/vnet/session/segment_manager.c
+++ b/src/vnet/session/segment_manager.c
@@ -17,229 +17,34 @@
#include <vnet/session/session.h>
#include <vnet/session/application.h>
+segment_manager_main_t segment_manager_main;
+
/**
* Counter used to build segment names
*/
-u32 segment_name_counter = 0;
-
-/**
- * Pool of segment managers
- */
-segment_manager_t *segment_managers = 0;
-
-/*
- * Pool of segment manager properties
- */
-static segment_manager_properties_t *segment_manager_properties_pool;
-
-/**
- * Process private segment index
- */
-u32 *private_segment_indices;
+static u32 segment_name_counter = 0;
/**
* Default fifo and segment size. TODO config.
*/
-u32 default_fifo_size = 1 << 16;
+u32 default_fifo_size = 1 << 12;
u32 default_segment_size = 1 << 20;
segment_manager_properties_t *
-segment_manager_properties_alloc (void)
+segment_manager_properties_get (segment_manager_t * sm)
{
- segment_manager_properties_t *props;
- pool_get (segment_manager_properties_pool, props);
- memset (props, 0, sizeof (*props));
+ return application_get_segment_manager_properties (sm->app_index);
+}
+
+segment_manager_properties_t *
+segment_manager_properties_init (segment_manager_properties_t * props)
+{
props->add_segment_size = default_segment_size;
props->rx_fifo_size = default_fifo_size;
props->tx_fifo_size = default_fifo_size;
return props;
}
-void
-segment_manager_properties_free (segment_manager_properties_t * props)
-{
- pool_put (segment_manager_properties_pool, props);
- memset (props, 0xFB, sizeof (*props));
-}
-
-segment_manager_properties_t *
-segment_manager_properties_get (u32 smp_index)
-{
- if (pool_is_free_index (segment_manager_properties_pool, smp_index))
- return 0;
- return pool_elt_at_index (segment_manager_properties_pool, smp_index);
-}
-
-u32
-segment_manager_properties_index (segment_manager_properties_t * p)
-{
- return p - segment_manager_properties_pool;
-}
-
-svm_fifo_segment_private_t *
-segment_manager_get_segment (u32 segment_index)
-{
- return svm_fifo_segment_get_segment (segment_index);
-}
-
-void
-segment_manager_get_segment_info (u32 index, u8 ** name, u32 * size)
-{
- svm_fifo_segment_private_t *s;
- s = svm_fifo_segment_get_segment (index);
- *name = s->ssvm.name;
- *size = s->ssvm.ssvm_size;
-}
-
-always_inline int
-segment_manager_add_segment_i (segment_manager_t * sm, u32 segment_size,
- u32 protected_space)
-{
- svm_fifo_segment_create_args_t _ca = { 0 }, *ca = &_ca;
- segment_manager_properties_t *props;
-
- props = segment_manager_properties_get (sm->properties_index);
-
- /* Not configured for addition of new segments and not first */
- if (!props->add_segment && !segment_size)
- {
- clib_warning ("cannot allocate new segment");
- return VNET_API_ERROR_INVALID_VALUE;
- }
-
- ca->segment_size = segment_size ? segment_size : props->add_segment_size;
- ca->rx_fifo_size = props->rx_fifo_size;
- ca->tx_fifo_size = props->tx_fifo_size;
- ca->preallocated_fifo_pairs = props->preallocated_fifo_pairs;
- ca->seg_protected_space = protected_space ? protected_space : 0;
-
- if (props->segment_type != SSVM_N_SEGMENT_TYPES)
- {
- ca->segment_name = (char *) format (0, "%d-%d%c", getpid (),
- segment_name_counter++, 0);
- ca->segment_type = props->segment_type;
- if (svm_fifo_segment_create (ca))
- {
- clib_warning ("svm_fifo_segment_create ('%s', %d) failed",
- ca->segment_name, ca->segment_size);
- return VNET_API_ERROR_SVM_SEGMENT_CREATE_FAIL;
- }
- vec_free (ca->segment_name);
- }
- else
- {
- u32 rx_fifo_size, tx_fifo_size;
- u32 rx_rounded_data_size, tx_rounded_data_size;
- u32 approx_segment_count;
- u64 approx_total_size;
-
- ca->segment_name = "process-private-segment";
- ca->private_segment_count = props->private_segment_count;
-
- /* Calculate space requirements */
- rx_rounded_data_size = (1 << (max_log2 (ca->rx_fifo_size)));
- tx_rounded_data_size = (1 << (max_log2 (ca->tx_fifo_size)));
-
- rx_fifo_size = sizeof (svm_fifo_t) + rx_rounded_data_size;
- tx_fifo_size = sizeof (svm_fifo_t) + tx_rounded_data_size;
-
- approx_total_size = (u64) ca->preallocated_fifo_pairs
- * (rx_fifo_size + tx_fifo_size);
- approx_segment_count = (approx_total_size + protected_space
- + (ca->segment_size -
- 1)) / (u64) ca->segment_size;
-
- /* The user asked us to figure it out... */
- if (ca->private_segment_count == 0
- || approx_segment_count < ca->private_segment_count)
- {
- ca->private_segment_count = approx_segment_count;
- }
- /* Follow directions, but issue a warning */
- else if (approx_segment_count < ca->private_segment_count)
- {
- clib_warning ("Honoring segment count %u, calculated count was %u",
- ca->private_segment_count, approx_segment_count);
- }
- else if (approx_segment_count > ca->private_segment_count)
- {
- clib_warning ("Segment count too low %u, calculated %u.",
- ca->private_segment_count, approx_segment_count);
- return VNET_API_ERROR_INVALID_VALUE;
- }
-
- if (svm_fifo_segment_create_process_private (ca))
- clib_warning ("Failed to create process private segment");
-
- ASSERT (vec_len (ca->new_segment_indices));
- }
-
- vec_append (sm->segment_indices, ca->new_segment_indices);
- vec_free (ca->new_segment_indices);
- return 0;
-}
-
-int
-segment_manager_add_segment (segment_manager_t * sm)
-{
- return segment_manager_add_segment_i (sm, 0, 0);
-}
-
-segment_manager_t *
-segment_manager_new ()
-{
- segment_manager_t *sm;
- pool_get (segment_managers, sm);
- memset (sm, 0, sizeof (*sm));
- return sm;
-}
-
-/**
- * Initializes segment manager based on options provided.
- * Returns error if svm segment allocation fails.
- */
-int
-segment_manager_init (segment_manager_t * sm, u32 props_index,
- u32 first_seg_size, u32 evt_q_size)
-{
- u32 protected_space;
- int rv;
-
- sm->properties_index = props_index;
-
- protected_space = max_pow2 (sizeof (svm_queue_t)
- + evt_q_size * sizeof (session_fifo_event_t));
- protected_space = round_pow2_u64 (protected_space, CLIB_CACHE_LINE_BYTES);
- first_seg_size = first_seg_size > 0 ? first_seg_size : default_segment_size;
- rv = segment_manager_add_segment_i (sm, first_seg_size, protected_space);
- if (rv)
- {
- clib_warning ("Failed to allocate segment");
- return rv;
- }
-
- clib_spinlock_init (&sm->lockp);
- return 0;
-}
-
-u8
-segment_manager_has_fifos (segment_manager_t * sm)
-{
- svm_fifo_segment_private_t *segment;
- int i;
-
- for (i = 0; i < vec_len (sm->segment_indices); i++)
- {
- segment = svm_fifo_segment_get_segment (sm->segment_indices[i]);
- if (CLIB_DEBUG && i && !svm_fifo_segment_has_fifos (segment)
- && !(segment->h->flags & FIFO_SEGMENT_F_IS_PREALLOCATED))
- clib_warning ("segment %d has no fifos!", sm->segment_indices[i]);
- if (svm_fifo_segment_has_fifos (segment))
- return 1;
- }
- return 0;
-}
-
static u8
segment_manager_app_detached (segment_manager_t * sm)
{
@@ -252,24 +57,283 @@
sm->app_index = SEGMENT_MANAGER_INVALID_APP_INDEX;
}
-static void
-segment_manager_del_segment (segment_manager_t * sm, u32 segment_index)
+always_inline u32
+segment_manager_segment_index (segment_manager_t * sm,
+ svm_fifo_segment_private_t * seg)
{
- svm_fifo_segment_private_t *fifo_segment;
- u32 svm_segment_index;
- clib_spinlock_lock (&sm->lockp);
- svm_segment_index = vec_elt (sm->segment_indices, segment_index);
- fifo_segment = svm_fifo_segment_get_segment (svm_segment_index);
- if (!fifo_segment
- || ((fifo_segment->h->flags & FIFO_SEGMENT_F_IS_PREALLOCATED)
- && !segment_manager_app_detached (sm)))
+ return (seg - sm->segments);
+}
+
+/**
+ * Remove segment without lock
+ */
+always_inline void
+segment_manager_del_segment (segment_manager_t * sm,
+ svm_fifo_segment_private_t * fs)
+{
+ segment_manager_main_t *smm = &segment_manager_main;
+
+ if (ssvm_type (&fs->ssvm) != SSVM_SEGMENT_PRIVATE)
+ clib_valloc_free (&smm->va_allocator, fs->ssvm.requested_va);
+
+ ssvm_delete (&fs->ssvm);
+
+ if (CLIB_DEBUG)
+ memset (fs, 0xfb, sizeof (*fs));
+ pool_put (sm->segments, fs);
+}
+
+/**
+ * Removes segment after acquiring writer lock
+ */
+always_inline void
+segment_manager_lock_and_del_segment (segment_manager_t * sm, u32 fs_index)
+{
+ svm_fifo_segment_private_t *fs;
+ u8 is_prealloc;
+
+ clib_rwlock_writer_lock (&sm->segments_rwlock);
+ fs = segment_manager_get_segment (sm, fs_index);
+ is_prealloc = svm_fifo_segment_flags (fs) & FIFO_SEGMENT_F_IS_PREALLOCATED;
+ if (is_prealloc && !segment_manager_app_detached (sm))
{
- clib_spinlock_unlock (&sm->lockp);
+ clib_rwlock_writer_unlock (&sm->segments_rwlock);
return;
}
- svm_fifo_segment_delete (fifo_segment);
- vec_del1 (sm->segment_indices, segment_index);
- clib_spinlock_unlock (&sm->lockp);
+
+ segment_manager_del_segment (sm, fs);
+ clib_rwlock_writer_unlock (&sm->segments_rwlock);
+}
+
+/**
+ * Reads a segment from the segment manager's pool without lock
+ */
+svm_fifo_segment_private_t *
+segment_manager_get_segment (segment_manager_t * sm, u32 segment_index)
+{
+ return pool_elt_at_index (sm->segments, segment_index);
+}
+
+/**
+ * Reads a segment from the segment manager's pool and acquires reader lock
+ *
+ * Caller must drop the reader's lock by calling
+ * @ref segment_manager_segment_reader_unlock once it finishes working with
+ * the segment.
+ */
+svm_fifo_segment_private_t *
+segment_manager_get_segment_w_lock (segment_manager_t * sm, u32 segment_index)
+{
+ clib_rwlock_reader_lock (&sm->segments_rwlock);
+ return pool_elt_at_index (sm->segments, segment_index);
+}
+
+void
+segment_manager_segment_reader_unlock (segment_manager_t * sm)
+{
+ clib_rwlock_reader_unlock (&sm->segments_rwlock);
+}
+
+void
+segment_manager_segment_writer_unlock (segment_manager_t * sm)
+{
+ clib_rwlock_writer_unlock (&sm->segments_rwlock);
+}
+
+/**
+ * Adds segment to segment manager's pool
+ *
+ * If needed a writer's lock is acquired before allocating a new segment
+ * to avoid affecting any of the segments pool readers.
+ */
+always_inline int
+segment_manager_add_segment (segment_manager_t * sm, u32 segment_size)
+{
+ segment_manager_main_t *smm = &segment_manager_main;
+ u32 rnd_margin = 128 << 10, seg_index;
+ segment_manager_properties_t *props;
+ uword baseva = (u64) ~ 0, alloc_size;
+ svm_fifo_segment_private_t *seg;
+ u8 *seg_name;
+ int rv;
+
+ props = segment_manager_properties_get (sm);
+
+ /* Not configured for addition of new segments and not first */
+ if (!props->add_segment && !segment_size)
+ {
+ clib_warning ("cannot allocate new segment");
+ return VNET_API_ERROR_INVALID_VALUE;
+ }
+
+ /*
+ * Allocate fifo segment and lock if needed
+ */
+ if (vlib_num_workers ())
+ {
+ clib_rwlock_writer_lock (&sm->segments_rwlock);
+ pool_get (sm->segments, seg);
+ }
+ else
+ {
+ pool_get (sm->segments, seg);
+ }
+ memset (seg, 0, sizeof (*seg));
+
+ /*
+ * Initialize ssvm segment and svm fifo private header
+ */
+ segment_size = segment_size ? segment_size : props->add_segment_size;
+ if (props->segment_type != SSVM_SEGMENT_PRIVATE)
+ {
+ seg_name = format (0, "%d-%d%c", getpid (), segment_name_counter++, 0);
+ alloc_size = segment_size + rnd_margin;
+ baseva = clib_valloc_alloc (&smm->va_allocator, alloc_size, 0);
+ if (!baseva)
+ {
+ clib_warning ("out of space for segments");
+ return -1;
+ }
+ }
+ else
+ seg_name = format (0, "%s%c", "process-private-segment", 0);
+
+ seg->ssvm.ssvm_size = segment_size;
+ seg->ssvm.name = seg_name;
+ seg->ssvm.requested_va = baseva;
+
+ if ((rv = ssvm_master_init (&seg->ssvm, props->segment_type)))
+ {
+ clib_warning ("svm_master_init ('%v', %u) failed", seg_name,
+ segment_size);
+
+ if (props->segment_type != SSVM_SEGMENT_PRIVATE)
+ clib_valloc_free (&smm->va_allocator, baseva);
+ pool_put (sm->segments, seg);
+ return (rv);
+ }
+
+ svm_fifo_segment_init (seg);
+
+ /*
+ * Save segment index before dropping lock, if any held
+ */
+ seg_index = seg - sm->segments;
+
+ if (vlib_num_workers ())
+ clib_rwlock_writer_unlock (&sm->segments_rwlock);
+
+ return seg_index;
+}
+
+segment_manager_t *
+segment_manager_new ()
+{
+ segment_manager_main_t *smm = &segment_manager_main;
+ segment_manager_t *sm;
+ pool_get (smm->segment_managers, sm);
+ memset (sm, 0, sizeof (*sm));
+ clib_rwlock_init (&sm->segments_rwlock);
+ return sm;
+}
+
+/**
+ * Initializes segment manager based on options provided.
+ * Returns error if ssvm segment(s) allocation fails.
+ */
+int
+segment_manager_init (segment_manager_t * sm, u32 first_seg_size,
+ u32 evt_q_size, u32 prealloc_fifo_pairs)
+{
+ u32 rx_fifo_size, tx_fifo_size, pair_size;
+ u32 rx_rounded_data_size, tx_rounded_data_size;
+ u64 approx_total_size, max_seg_size =
+ ((u64) 1 << 32) - clib_mem_get_page_size ();
+ segment_manager_properties_t *props;
+ svm_fifo_segment_private_t *segment;
+ u32 approx_segment_count;
+ int seg_index, i;
+
+ props = segment_manager_properties_get (sm);
+ first_seg_size = clib_max (first_seg_size, default_segment_size);
+
+ if (prealloc_fifo_pairs)
+ {
+ /* Figure out how many segments should be preallocated */
+ rx_rounded_data_size = (1 << (max_log2 (props->rx_fifo_size)));
+ tx_rounded_data_size = (1 << (max_log2 (props->tx_fifo_size)));
+
+ rx_fifo_size = sizeof (svm_fifo_t) + rx_rounded_data_size;
+ tx_fifo_size = sizeof (svm_fifo_t) + tx_rounded_data_size;
+ pair_size = rx_fifo_size + tx_fifo_size;
+
+ approx_total_size = (u64) prealloc_fifo_pairs *pair_size;
+ if (first_seg_size > approx_total_size)
+ max_seg_size = first_seg_size;
+ approx_segment_count = (approx_total_size + (max_seg_size - 1))
+ / max_seg_size;
+
+ /* Allocate the segments */
+ for (i = 0; i < approx_segment_count + 1; i++)
+ {
+ seg_index = segment_manager_add_segment (sm, max_seg_size);
+ if (seg_index < 0)
+ {
+ clib_warning ("Failed to preallocate segment %d", i);
+ return seg_index;
+ }
+
+ if (i == 0)
+ sm->event_queue = segment_manager_alloc_queue (sm, evt_q_size);
+
+ segment = segment_manager_get_segment (sm, seg_index);
+ svm_fifo_segment_preallocate_fifo_pairs (segment,
+ props->rx_fifo_size,
+ props->tx_fifo_size,
+ &prealloc_fifo_pairs);
+ svm_fifo_segment_flags (segment) = FIFO_SEGMENT_F_IS_PREALLOCATED;
+ if (prealloc_fifo_pairs == 0)
+ break;
+ }
+ }
+ else
+ {
+ seg_index = segment_manager_add_segment (sm, first_seg_size);
+ if (seg_index)
+ {
+ clib_warning ("Failed to allocate segment");
+ return seg_index;
+ }
+ sm->event_queue = segment_manager_alloc_queue (sm, evt_q_size);
+ }
+
+ return 0;
+}
+
+u8
+segment_manager_has_fifos (segment_manager_t * sm)
+{
+ svm_fifo_segment_private_t *seg;
+ u8 first = 1;
+
+ /* *INDENT-OFF* */
+ segment_manager_foreach_segment_w_lock (seg, sm, ({
+ if (CLIB_DEBUG && !first && !svm_fifo_segment_has_fifos (seg)
+ && !(svm_fifo_segment_flags (seg) & FIFO_SEGMENT_F_IS_PREALLOCATED))
+ {
+ clib_warning ("segment %d has no fifos!",
+ segment_manager_segment_index (sm, seg));
+ first = 0;
+ }
+ if (svm_fifo_segment_has_fifos (seg))
+ {
+ segment_manager_segment_reader_unlock (sm);
+ return 1;
+ }
+ }));
+ /* *INDENT-ON* */
+
+ return 0;
}
/**
@@ -278,42 +342,35 @@
void
segment_manager_del_sessions (segment_manager_t * sm)
{
- int j;
svm_fifo_segment_private_t *fifo_segment;
+ stream_session_t *session;
svm_fifo_t *fifo;
- ASSERT (vec_len (sm->segment_indices));
+ ASSERT (pool_elts (sm->segments) != 0);
/* Across all fifo segments used by the server */
- for (j = 0; j < vec_len (sm->segment_indices); j++)
- {
- fifo_segment = svm_fifo_segment_get_segment (sm->segment_indices[j]);
- fifo = svm_fifo_segment_get_fifo_list (fifo_segment);
+ /* *INDENT-OFF* */
+ segment_manager_foreach_segment_w_lock (fifo_segment, sm, ({
+ fifo = svm_fifo_segment_get_fifo_list (fifo_segment);
- /*
- * Remove any residual sessions from the session lookup table
- * Don't bother deleting the individual fifos, we're going to
- * throw away the fifo segment in a minute.
- */
- while (fifo)
- {
- u32 session_index, thread_index;
- stream_session_t *session;
+ /*
+ * Remove any residual sessions from the session lookup table
+ * Don't bother deleting the individual fifos, we're going to
+ * throw away the fifo segment in a minute.
+ */
+ while (fifo)
+ {
+ session = session_get (fifo->master_session_index,
+ fifo->master_thread_index);
+ stream_session_disconnect (session);
+ fifo = fifo->next;
+ }
- session_index = fifo->master_session_index;
- thread_index = fifo->master_thread_index;
- session = session_get (session_index, thread_index);
-
- /* Instead of directly removing the session call disconnect */
- if (session->session_state != SESSION_STATE_CLOSED)
- stream_session_disconnect (session);
- fifo = fifo->next;
- }
-
- /* Instead of removing the segment, test when cleaning up disconnected
- * sessions if the segment can be removed.
- */
- }
+ /* Instead of removing the segment, test when cleaning up disconnected
+ * sessions if the segment can be removed.
+ */
+ }));
+ /* *INDENT-ON* */
}
/**
@@ -326,7 +383,8 @@
void
segment_manager_del (segment_manager_t * sm)
{
- int i;
+ segment_manager_main_t *smm = &segment_manager_main;
+ svm_fifo_segment_private_t *fifo_segment;
ASSERT (!segment_manager_has_fifos (sm)
&& segment_manager_app_detached (sm));
@@ -335,20 +393,20 @@
* them now. Apart from that, the first segment in the first segment manager
* is not removed when all fifos are removed. It can only be removed when
* the manager is explicitly deleted/detached by the app. */
- for (i = vec_len (sm->segment_indices) - 1; i >= 0; i--)
- {
- if (CLIB_DEBUG)
- {
- svm_fifo_segment_private_t *segment;
- segment = svm_fifo_segment_get_segment (sm->segment_indices[i]);
- ASSERT (!svm_fifo_segment_has_fifos (segment));
- }
- segment_manager_del_segment (sm, i);
- }
- clib_spinlock_free (&sm->lockp);
+ clib_rwlock_writer_lock (&sm->segments_rwlock);
+
+ /* *INDENT-OFF* */
+ pool_foreach (fifo_segment, sm->segments, ({
+ segment_manager_del_segment (sm, fifo_segment);
+ }));
+ /* *INDENT-ON* */
+
+ clib_rwlock_writer_unlock (&sm->segments_rwlock);
+
+ clib_rwlock_free (&sm->segments_rwlock);
if (CLIB_DEBUG)
memset (sm, 0xfe, sizeof (*sm));
- pool_put (segment_managers, sm);
+ pool_put (smm->segment_managers, sm);
}
void
@@ -364,6 +422,44 @@
}
}
+always_inline int
+segment_try_alloc_fifos (svm_fifo_segment_private_t * fifo_segment,
+ u32 rx_fifo_size, u32 tx_fifo_size,
+ svm_fifo_t ** rx_fifo, svm_fifo_t ** tx_fifo)
+{
+ rx_fifo_size = clib_max (rx_fifo_size, default_fifo_size);
+ *rx_fifo = svm_fifo_segment_alloc_fifo (fifo_segment, rx_fifo_size,
+ FIFO_SEGMENT_RX_FREELIST);
+
+ tx_fifo_size = clib_max (tx_fifo_size, default_fifo_size);
+ *tx_fifo = svm_fifo_segment_alloc_fifo (fifo_segment, tx_fifo_size,
+ FIFO_SEGMENT_TX_FREELIST);
+
+ if (*rx_fifo == 0)
+ {
+ /* This would be very odd, but handle it... */
+ if (*tx_fifo != 0)
+ {
+ svm_fifo_segment_free_fifo (fifo_segment, *tx_fifo,
+ FIFO_SEGMENT_TX_FREELIST);
+ *tx_fifo = 0;
+ }
+ return -1;
+ }
+ if (*tx_fifo == 0)
+ {
+ if (*rx_fifo != 0)
+ {
+ svm_fifo_segment_free_fifo (fifo_segment, *rx_fifo,
+ FIFO_SEGMENT_RX_FREELIST);
+ *rx_fifo = 0;
+ }
+ return -1;
+ }
+
+ return 0;
+}
+
int
segment_manager_alloc_session_fifos (segment_manager_t * sm,
svm_fifo_t ** rx_fifo,
@@ -371,119 +467,93 @@
u32 * fifo_segment_index)
{
svm_fifo_segment_private_t *fifo_segment;
+ int alloc_fail = 1, rv = 0, new_fs_index;
segment_manager_properties_t *props;
- u32 fifo_size, sm_index;
u8 added_a_segment = 0;
- int i;
+ u32 sm_index;
- ASSERT (vec_len (sm->segment_indices));
+ ASSERT (pool_elts (sm->segments) != 0);
+ props = segment_manager_properties_get (sm);
- /* Make sure we don't have multiple threads trying to allocate segments
- * at the same time. */
- clib_spinlock_lock (&sm->lockp);
+ /*
+ * Find the first free segment to allocate the fifos in
+ */
- /* Allocate svm fifos */
- props = segment_manager_properties_get (sm->properties_index);
-again:
- for (i = 0; i < vec_len (sm->segment_indices); i++)
+ /* *INDENT-OFF* */
+ segment_manager_foreach_segment_w_lock (fifo_segment, sm, ({
+ alloc_fail = segment_try_alloc_fifos (fifo_segment, props->rx_fifo_size,
+ props->tx_fifo_size, rx_fifo,
+ tx_fifo);
+ /* Exit with lock held, drop it after notifying app */
+ if (!alloc_fail)
+ goto alloc_success;
+ }));
+ /* *INDENT-ON* */
+
+alloc_check:
+
+ if (!alloc_fail)
{
- *fifo_segment_index = vec_elt (sm->segment_indices, i);
- fifo_segment = svm_fifo_segment_get_segment (*fifo_segment_index);
- fifo_size = props->rx_fifo_size;
- fifo_size = (fifo_size == 0) ? default_fifo_size : fifo_size;
- *rx_fifo = svm_fifo_segment_alloc_fifo (fifo_segment, fifo_size,
- FIFO_SEGMENT_RX_FREELIST);
+ alloc_success:
- fifo_size = props->tx_fifo_size;
- fifo_size = (fifo_size == 0) ? default_fifo_size : fifo_size;
- *tx_fifo = svm_fifo_segment_alloc_fifo (fifo_segment, fifo_size,
- FIFO_SEGMENT_TX_FREELIST);
+ ASSERT (rx_fifo && tx_fifo);
+ sm_index = segment_manager_index (sm);
+ (*tx_fifo)->segment_manager = sm_index;
+ (*rx_fifo)->segment_manager = sm_index;
+ *fifo_segment_index = segment_manager_segment_index (sm, fifo_segment);
- if (*rx_fifo == 0)
- {
- /* This would be very odd, but handle it... */
- if (*tx_fifo != 0)
- {
- svm_fifo_segment_free_fifo (fifo_segment, *tx_fifo,
- FIFO_SEGMENT_TX_FREELIST);
- *tx_fifo = 0;
- }
- continue;
- }
- if (*tx_fifo == 0)
- {
- if (*rx_fifo != 0)
- {
- svm_fifo_segment_free_fifo (fifo_segment, *rx_fifo,
- FIFO_SEGMENT_RX_FREELIST);
- *rx_fifo = 0;
- }
- continue;
- }
- break;
+ if (added_a_segment)
+ rv = application_add_segment_notify (sm->app_index,
+ &fifo_segment->ssvm);
+ /* Drop the lock after app is notified */
+ segment_manager_segment_reader_unlock (sm);
+ return rv;
}
- /* See if we're supposed to create another segment */
- if (*rx_fifo == 0)
+ /*
+ * Allocation failed, see if we can add a new segment
+ */
+ if (props->add_segment)
{
- if (props->add_segment && !props->segment_type)
+ if (added_a_segment)
{
- if (added_a_segment)
- {
- clib_warning ("added a segment, still can't allocate a fifo");
- clib_spinlock_unlock (&sm->lockp);
- return SESSION_ERROR_NEW_SEG_NO_SPACE;
- }
-
- if (segment_manager_add_segment (sm))
- {
- clib_spinlock_unlock (&sm->lockp);
- return VNET_API_ERROR_URI_FIFO_CREATE_FAILED;
- }
-
- added_a_segment = 1;
- goto again;
+ clib_warning ("Added a segment, still can't allocate a fifo");
+ segment_manager_segment_reader_unlock (sm);
+ return SESSION_ERROR_NEW_SEG_NO_SPACE;
}
- else
+ if ((new_fs_index = segment_manager_add_segment (sm, 0)) < 0)
{
- clib_warning ("No space to allocate fifos!");
- clib_spinlock_unlock (&sm->lockp);
- return SESSION_ERROR_NO_SPACE;
+ clib_warning ("Failed to add new segment");
+ return SESSION_ERROR_SEG_CREATE;
}
+ fifo_segment = segment_manager_get_segment_w_lock (sm, new_fs_index);
+ alloc_fail = segment_try_alloc_fifos (fifo_segment, props->rx_fifo_size,
+ props->tx_fifo_size, rx_fifo,
+ tx_fifo);
+ added_a_segment = 1;
+ goto alloc_check;
}
-
- /* Backpointers to segment manager */
- sm_index = segment_manager_index (sm);
- (*tx_fifo)->segment_manager = sm_index;
- (*rx_fifo)->segment_manager = sm_index;
-
- clib_spinlock_unlock (&sm->lockp);
-
- if (added_a_segment)
- return application_add_segment_notify (sm->app_index,
- *fifo_segment_index);
-
- return 0;
+ else
+ {
+ clib_warning ("Can't add new seg and no space to allocate fifos!");
+ return SESSION_ERROR_NO_SPACE;
+ }
}
void
-segment_manager_dealloc_fifos (u32 svm_segment_index, svm_fifo_t * rx_fifo,
+segment_manager_dealloc_fifos (u32 segment_index, svm_fifo_t * rx_fifo,
svm_fifo_t * tx_fifo)
{
- segment_manager_t *sm;
svm_fifo_segment_private_t *fifo_segment;
- u32 i, segment_index = ~0;
- u8 is_first;
-
- sm = segment_manager_get_if_valid (rx_fifo->segment_manager);
+ segment_manager_t *sm;
/* It's possible to have no segment manager if the session was removed
* as result of a detach. */
- if (!sm)
+ if (!(sm = segment_manager_get_if_valid (rx_fifo->segment_manager)))
return;
- fifo_segment = svm_fifo_segment_get_segment (svm_segment_index);
+ fifo_segment = segment_manager_get_segment_w_lock (sm, segment_index);
svm_fifo_segment_free_fifo (fifo_segment, rx_fifo,
FIFO_SEGMENT_RX_FREELIST);
svm_fifo_segment_free_fifo (fifo_segment, tx_fifo,
@@ -497,48 +567,42 @@
*/
if (!svm_fifo_segment_has_fifos (fifo_segment))
{
- is_first = sm->segment_indices[0] == svm_segment_index;
+ segment_manager_segment_reader_unlock (sm);
/* Remove segment if it holds no fifos or first but not protected */
- if (!is_first || !sm->first_is_protected)
- {
- /* Find the segment manager segment index */
- for (i = 0; i < vec_len (sm->segment_indices); i++)
- if (sm->segment_indices[i] == svm_segment_index)
- {
- segment_index = i;
- break;
- }
- ASSERT (segment_index != (u32) ~ 0);
- segment_manager_del_segment (sm, segment_index);
- }
+ if (segment_index != 0 || !sm->first_is_protected)
+ segment_manager_lock_and_del_segment (sm, segment_index);
/* Remove segment manager if no sessions and detached from app */
if (segment_manager_app_detached (sm)
&& !segment_manager_has_fifos (sm))
segment_manager_del (sm);
}
+ else
+ segment_manager_segment_reader_unlock (sm);
}
/**
* Allocates shm queue in the first segment
+ *
+ * Must be called with lock held
*/
svm_queue_t *
segment_manager_alloc_queue (segment_manager_t * sm, u32 queue_size)
{
- ssvm_shared_header_t *sh;
svm_fifo_segment_private_t *segment;
+ ssvm_shared_header_t *sh;
svm_queue_t *q;
void *oldheap;
- ASSERT (sm->segment_indices != 0);
+ ASSERT (!pool_is_free_index (sm->segments, 0));
- segment = svm_fifo_segment_get_segment (sm->segment_indices[0]);
+ segment = segment_manager_get_segment (sm, 0);
sh = segment->ssvm.sh;
oldheap = ssvm_push_heap (sh);
- q = svm_queue_init (queue_size,
- sizeof (session_fifo_event_t), 0 /* consumer pid */ ,
+ q = svm_queue_init (queue_size, sizeof (session_fifo_event_t),
+ 0 /* consumer pid */ ,
0 /* signal when queue non-empty */ );
ssvm_pop_heap (oldheap);
return q;
@@ -550,25 +614,42 @@
void
segment_manager_dealloc_queue (segment_manager_t * sm, svm_queue_t * q)
{
- ssvm_shared_header_t *sh;
svm_fifo_segment_private_t *segment;
+ ssvm_shared_header_t *sh;
void *oldheap;
- ASSERT (sm->segment_indices != 0);
+ ASSERT (!pool_is_free_index (sm->segments, 0));
- segment = svm_fifo_segment_get_segment (sm->segment_indices[0]);
+ segment = segment_manager_get_segment_w_lock (sm, 0);
sh = segment->ssvm.sh;
oldheap = ssvm_push_heap (sh);
svm_queue_free (q);
ssvm_pop_heap (oldheap);
+ segment_manager_segment_reader_unlock (sm);
+}
+
+/*
+ * Init segment vm address allocator
+ */
+void
+segment_manager_main_init (segment_manager_main_init_args_t * a)
+{
+ segment_manager_main_t *sm = &segment_manager_main;
+ clib_valloc_chunk_t _ip, *ip = &_ip;
+
+ ip->baseva = a->baseva;
+ ip->size = a->size;
+
+ clib_valloc_init (&sm->va_allocator, ip, 1 /* lock */ );
}
static clib_error_t *
segment_manager_show_fn (vlib_main_t * vm, unformat_input_t * input,
vlib_cli_command_t * cmd)
{
- svm_fifo_segment_private_t *segments, *seg;
+ segment_manager_main_t *smm = &segment_manager_main;
+ svm_fifo_segment_private_t *seg;
segment_manager_t *sm;
u8 show_segments = 0, verbose = 0;
uword address;
@@ -587,39 +668,38 @@
format_unformat_error, input);
}
vlib_cli_output (vm, "%d segment managers allocated",
- pool_elts (segment_managers));
- if (verbose && pool_elts (segment_managers))
+ pool_elts (smm->segment_managers));
+ if (verbose && pool_elts (smm->segment_managers))
{
vlib_cli_output (vm, "%-10s%=15s%=12s", "Index", "App Index",
"Segments");
/* *INDENT-OFF* */
- pool_foreach (sm, segment_managers, ({
+ pool_foreach (sm, smm->segment_managers, ({
vlib_cli_output (vm, "%-10d%=15d%=12d", segment_manager_index(sm),
- sm->app_index, vec_len (sm->segment_indices));
+ sm->app_index, pool_elts (sm->segments));
}));
/* *INDENT-ON* */
}
if (show_segments)
{
- segments = svm_fifo_segment_segments_pool ();
- vlib_cli_output (vm, "%d svm fifo segments allocated",
- pool_elts (segments));
vlib_cli_output (vm, "%-15s%15s%15s%15s%15s%15s", "Name", "Type",
"HeapSize (M)", "ActiveFifos", "FreeFifos", "Address");
/* *INDENT-OFF* */
- pool_foreach (seg, segments, ({
- svm_fifo_segment_info (seg, &address, &size);
- active_fifos = svm_fifo_segment_num_fifos (seg);
- free_fifos = svm_fifo_segment_num_free_fifos (seg, ~0 /* size */);
- vlib_cli_output (vm, "%-15v%15U%15llu%15u%15u%15llx",
- ssvm_name (&seg->ssvm), format_svm_fifo_segment_type,
- seg, size >> 20ULL, active_fifos, free_fifos,
- address);
- if (verbose)
- vlib_cli_output (vm, "%U", format_svm_fifo_segment, seg, verbose);
+ pool_foreach (sm, smm->segment_managers, ({
+ segment_manager_foreach_segment_w_lock (seg, sm, ({
+ svm_fifo_segment_info (seg, &address, &size);
+ active_fifos = svm_fifo_segment_num_fifos (seg);
+ free_fifos = svm_fifo_segment_num_free_fifos (seg, ~0 /* size */);
+ vlib_cli_output (vm, "%-15v%15U%15llu%15u%15u%15llx",
+ ssvm_name (&seg->ssvm), format_svm_fifo_segment_type,
+ seg, size >> 20ULL, active_fifos, free_fifos,
+ address);
+ if (verbose)
+ vlib_cli_output (vm, "%U", format_svm_fifo_segment, seg, verbose);
+ }));
}));
/* *INDENT-ON* */
diff --git a/src/vnet/session/segment_manager.h b/src/vnet/session/segment_manager.h
index b2a792d..9b1d4cd 100644
--- a/src/vnet/session/segment_manager.h
+++ b/src/vnet/session/segment_manager.h
@@ -20,6 +20,7 @@
#include <svm/queue.h>
#include <vlibmemory/api.h>
#include <vppinfra/lock.h>
+#include <vppinfra/valloc.h>
typedef struct _segment_manager_properties
{
@@ -28,7 +29,7 @@
u32 tx_fifo_size;
/** Preallocated pool sizes */
- u32 preallocated_fifo_pairs;
+// u32 preallocated_fifo_pairs;
/** Configured additional segment size */
u32 add_segment_size;
@@ -40,33 +41,58 @@
ssvm_segment_type_t segment_type;
/** Use one or more private mheaps, instead of the global heap */
- u32 private_segment_count;
+// u32 private_segment_count;
} segment_manager_properties_t;
typedef struct _segment_manager
{
- clib_spinlock_t lockp;
+ /** Pool of segments allocated by this manager */
+ svm_fifo_segment_private_t *segments;
- /** segments mapped by this manager */
- u32 *segment_indices;
+ /** rwlock that protects the segments pool */
+ clib_rwlock_t segments_rwlock;
/** Owner app index */
u32 app_index;
/**
- * Pointer to manager properties. Could be shared among all of
- * an app's segment managers s
- */
- u32 properties_index;
-
- /**
* First segment should not be deleted unless segment manger is deleted.
* This also indicates that the segment manager is the first to have been
* allocated for the app.
*/
u8 first_is_protected;
+
+ /**
+ * App event queue allocated in first segment
+ */
+ svm_queue_t *event_queue;
} segment_manager_t;
+#define segment_manager_foreach_segment_w_lock(VAR, SM, BODY) \
+do { \
+ clib_rwlock_reader_lock (&(SM)->segments_rwlock); \
+ pool_foreach((VAR), ((SM)->segments), (BODY)); \
+ clib_rwlock_reader_unlock (&(SM)->segments_rwlock); \
+} while (0)
+
+typedef struct segment_manager_main_
+{
+ /** Pool of segment managers */
+ segment_manager_t *segment_managers;
+
+ /** Virtual address allocator */
+ clib_valloc_main_t va_allocator;
+
+} segment_manager_main_t;
+
+extern segment_manager_main_t segment_manager_main;
+
+typedef struct segment_manager_main_init_args_
+{
+ u64 baseva;
+ u64 size;
+} segment_manager_main_init_args_t;
+
#define SEGMENT_MANAGER_INVALID_APP_INDEX ((u32) ~0)
/** Pool of segment managers */
@@ -75,31 +101,44 @@
always_inline segment_manager_t *
segment_manager_get (u32 index)
{
- return pool_elt_at_index (segment_managers, index);
+ return pool_elt_at_index (segment_manager_main.segment_managers, index);
}
always_inline segment_manager_t *
segment_manager_get_if_valid (u32 index)
{
- if (pool_is_free_index (segment_managers, index))
+ if (pool_is_free_index (segment_manager_main.segment_managers, index))
return 0;
- return pool_elt_at_index (segment_managers, index);
+ return pool_elt_at_index (segment_manager_main.segment_managers, index);
}
always_inline u32
segment_manager_index (segment_manager_t * sm)
{
- return sm - segment_managers;
+ return sm - segment_manager_main.segment_managers;
+}
+
+always_inline svm_queue_t *
+segment_manager_event_queue (segment_manager_t * sm)
+{
+ return sm->event_queue;
}
segment_manager_t *segment_manager_new ();
-int segment_manager_init (segment_manager_t * sm, u32 props_index,
- u32 seg_size, u32 evt_queue_size);
+int segment_manager_init (segment_manager_t * sm, u32 first_seg_size,
+ u32 evt_q_size, u32 prealloc_fifo_pairs);
-svm_fifo_segment_private_t *segment_manager_get_segment (u32 segment_index);
+svm_fifo_segment_private_t *segment_manager_get_segment (segment_manager_t *
+ sm,
+ u32 segment_index);
+svm_fifo_segment_private_t
+ * segment_manager_get_segment_w_lock (segment_manager_t * sm,
+ u32 segment_index);
+void segment_manager_segment_reader_unlock (segment_manager_t * sm);
+void segment_manager_segment_writer_unlock (segment_manager_t * sm);
+
int segment_manager_add_first_segment (segment_manager_t * sm,
u32 segment_size);
-int segment_manager_add_segment (segment_manager_t * sm);
void segment_manager_del_sessions (segment_manager_t * sm);
void segment_manager_del (segment_manager_t * sm);
void segment_manager_init_del (segment_manager_t * sm);
@@ -117,10 +156,9 @@
void segment_manager_dealloc_queue (segment_manager_t * sm, svm_queue_t * q);
void segment_manager_app_detach (segment_manager_t * sm);
-segment_manager_properties_t *segment_manager_properties_alloc (void);
-void segment_manager_properties_free (segment_manager_properties_t * p);
-segment_manager_properties_t *segment_manager_properties_get (u32 smp_index);
-u32 segment_manager_properties_index (segment_manager_properties_t * p);
+void segment_manager_main_init (segment_manager_main_init_args_t * a);
+segment_manager_properties_t
+ * segment_manager_properties_init (segment_manager_properties_t * sm);
#endif /* SRC_VNET_SESSION_SEGMENT_MANAGER_H_ */
/*
diff --git a/src/vnet/session/session.c b/src/vnet/session/session.c
index ae8c42a..de3cbc5 100644
--- a/src/vnet/session/session.c
+++ b/src/vnet/session/session.c
@@ -1115,6 +1115,7 @@
static clib_error_t *
session_manager_main_enable (vlib_main_t * vm)
{
+ segment_manager_main_init_args_t _sm_args = { 0 }, *sm_args = &_sm_args;
session_manager_main_t *smm = &session_manager_main;
vlib_thread_main_t *vtm = vlib_get_thread_main ();
u32 num_threads;
@@ -1162,8 +1163,9 @@
session_vpp_event_queues_allocate (smm);
/* Initialize fifo segment main baseva and timeout */
- svm_fifo_segment_init (smm->session_baseva + smm->evt_qs_segment_size,
- smm->segment_timeout);
+ sm_args->baseva = smm->session_baseva + smm->evt_qs_segment_size;
+ sm_args->size = smm->session_va_space_size;
+ segment_manager_main_init (sm_args);
/* Preallocate sessions */
if (smm->preallocated_sessions)
@@ -1237,7 +1239,7 @@
{
session_manager_main_t *smm = &session_manager_main;
smm->session_baseva = 0x200000000ULL;
- smm->segment_timeout = 20;
+ smm->session_va_space_size = (u64) 128 << 30;
smm->evt_qs_segment_size = 64 << 20;
smm->is_enabled = 0;
return 0;
diff --git a/src/vnet/session/session.h b/src/vnet/session/session.h
index d0bf95c..ec00e29 100644
--- a/src/vnet/session/session.h
+++ b/src/vnet/session/session.h
@@ -70,7 +70,8 @@
_(EVENT_FIFO_FULL, "Events not sent for lack of event fifo space") \
_(API_QUEUE_FULL, "Sessions not created for lack of API queue space") \
_(NEW_SEG_NO_SPACE, "Created segment, couldn't allocate a fifo pair") \
-_(NO_SPACE, "Couldn't allocate a fifo pair")
+_(NO_SPACE, "Couldn't allocate a fifo pair") \
+_(SEG_CREATE, "Couldn't create a new segment")
typedef enum
{
@@ -171,7 +172,7 @@
/** Session ssvm segment configs*/
uword session_baseva;
- u32 segment_timeout;
+ uword session_va_space_size;
u32 evt_qs_segment_size;
u8 evt_qs_use_memfd_seg;
diff --git a/src/vnet/session/session_api.c b/src/vnet/session/session_api.c
index a1225f8..57af960 100755
--- a/src/vnet/session/session_api.c
+++ b/src/vnet/session/session_api.c
@@ -290,7 +290,7 @@
return -1;
}
- props = segment_manager_properties_get (app->sm_properties);
+ props = application_segment_manager_properties (app);
mp->options[APP_OPTIONS_RX_FIFO_SIZE] = props->rx_fifo_size;
mp->options[APP_OPTIONS_TX_FIFO_SIZE] = props->tx_fifo_size;
diff --git a/src/vnet/session/session_test.c b/src/vnet/session/session_test.c
index a9a902d..febe1b7 100644
--- a/src/vnet/session/session_test.c
+++ b/src/vnet/session/session_test.c
@@ -99,6 +99,44 @@
/* *INDENT-ON* */
static int
+session_create_lookpback (u32 table_id, u32 * sw_if_index,
+ ip4_address_t * intf_addr)
+{
+ u8 intf_mac[6];
+
+ memset (intf_mac, 0, sizeof (intf_mac));
+
+ if (vnet_create_loopback_interface (sw_if_index, intf_mac, 0, 0))
+ {
+ clib_warning ("couldn't create loopback. stopping the test!");
+ return -1;
+ }
+
+ if (table_id != 0)
+ ip_table_bind (FIB_PROTOCOL_IP4, *sw_if_index, table_id, 0);
+
+ vnet_sw_interface_set_flags (vnet_get_main (), *sw_if_index,
+ VNET_SW_INTERFACE_FLAG_ADMIN_UP);
+
+ if (ip4_add_del_interface_address (vlib_get_main (), *sw_if_index,
+ intf_addr, 24, 0))
+ {
+ clib_warning ("couldn't assign loopback ip %U", format_ip4_address,
+ intf_addr);
+ return -1;
+ }
+
+ return 0;
+}
+
+static void
+session_delete_loopback (u32 sw_if_index)
+{
+ /* fails spectacularly */
+ /* vnet_delete_loopback_interface (sw_if_index); */
+}
+
+static int
session_test_basic (vlib_main_t * vm, unformat_input_t * input)
{
session_endpoint_t server_sep = SESSION_ENDPOINT_NULL;
@@ -174,7 +212,7 @@
session_endpoint_t client_sep = SESSION_ENDPOINT_NULL;
session_endpoint_t intf_sep = SESSION_ENDPOINT_NULL;
clib_error_t *error = 0;
- u8 *ns_id = format (0, "appns1"), intf_mac[6];
+ u8 *ns_id = format (0, "appns1");
app_namespace_t *app_ns;
application_t *server;
stream_session_t *s;
@@ -185,7 +223,6 @@
client_sep.is_ip4 = 1;
client_sep.port = dummy_port;
memset (options, 0, sizeof (options));
- memset (intf_mac, 0, sizeof (intf_mac));
options[APP_OPTIONS_FLAGS] = APP_OPTIONS_FLAGS_IS_BUILTIN;
options[APP_OPTIONS_FLAGS] |= APP_OPTIONS_FLAGS_ACCEPT_REDIRECT;
@@ -440,15 +477,7 @@
/*
* Create loopback interface
*/
- if (vnet_create_loopback_interface (&sw_if_index, intf_mac, 0, 0))
- {
- clib_warning ("couldn't create loopback. stopping the test!");
- return 0;
- }
- vnet_sw_interface_set_flags (vnet_get_main (), sw_if_index,
- VNET_SW_INTERFACE_FLAG_ADMIN_UP);
- ip4_add_del_interface_address (vlib_get_main (), sw_if_index, &intf_addr,
- 24, 0);
+ session_create_lookpback (0, &sw_if_index, &intf_addr);
/*
* Update namespace
@@ -492,8 +521,7 @@
* Cleanup
*/
vec_free (ns_id);
- /* fails in multi core scenarions .. */
- /* vnet_delete_loopback_interface (sw_if_index); */
+ session_delete_loopback (sw_if_index);
return 0;
}
@@ -1014,8 +1042,6 @@
sep.ip.ip4.as_u32 -= 1 << 24;
-
-
/*
* Delete masking rule: 1.2.3.4/32 1234 5.6.7.8/32 4321 allow
*/
@@ -1337,7 +1363,7 @@
u32 server_index, app_index;
u32 dummy_server_api_index = ~0, sw_if_index = 0;
clib_error_t *error = 0;
- u8 intf_mac[6], sst, is_filtered = 0;
+ u8 sst, is_filtered = 0;
stream_session_t *s;
transport_connection_t *tc;
u16 lcl_port = 1234, rmt_port = 4321;
@@ -1377,21 +1403,13 @@
/*
* Create loopback interface
*/
- memset (intf_mac, 0, sizeof (intf_mac));
- if (vnet_create_loopback_interface (&sw_if_index, intf_mac, 0, 0))
- {
- clib_warning ("couldn't create loopback. stopping the test!");
- return 0;
- }
- vnet_sw_interface_set_flags (vnet_get_main (), sw_if_index,
- VNET_SW_INTERFACE_FLAG_ADMIN_UP);
- ip4_add_del_interface_address (vlib_get_main (), sw_if_index, &lcl_ip,
- 24, 0);
+ session_create_lookpback (0, &sw_if_index, &lcl_ip);
app_ns = app_namespace_get_default ();
app_ns->sw_if_index = sw_if_index;
memset (options, 0, sizeof (options));
+ options[APP_OPTIONS_FLAGS] = APP_OPTIONS_FLAGS_IS_BUILTIN;
options[APP_OPTIONS_FLAGS] |= APP_OPTIONS_FLAGS_ACCEPT_REDIRECT;
options[APP_OPTIONS_FLAGS] |= APP_OPTIONS_FLAGS_IS_PROXY;
options[APP_OPTIONS_PROXY_TRANSPORT] = 1 << TRANSPORT_PROTO_TCP;
diff --git a/src/vppinfra/lock.h b/src/vppinfra/lock.h
index 0ffb045..dd79c40 100644
--- a/src/vppinfra/lock.h
+++ b/src/vppinfra/lock.h
@@ -109,6 +109,7 @@
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
volatile u32 n_readers;
+ volatile u32 n_readers_lock;
volatile u32 writer_lock;
#if CLIB_DEBUG > 0
pid_t pid;
@@ -137,23 +138,39 @@
always_inline void
clib_rwlock_reader_lock (clib_rwlock_t * p)
{
- if (__sync_fetch_and_add (&(*p)->n_readers, 1) == 0)
+ while (__sync_lock_test_and_set (&(*p)->n_readers_lock, 1))
+ CLIB_PAUSE ();
+
+ (*p)->n_readers += 1;
+ if ((*p)->n_readers == 1)
{
while (__sync_lock_test_and_set (&(*p)->writer_lock, 1))
CLIB_PAUSE ();
}
+ CLIB_MEMORY_BARRIER ();
+ (*p)->n_readers_lock = 0;
+
CLIB_LOCK_DBG (p);
}
always_inline void
clib_rwlock_reader_unlock (clib_rwlock_t * p)
{
+ ASSERT ((*p)->n_readers > 0);
CLIB_LOCK_DBG_CLEAR (p);
- if (__sync_fetch_and_sub (&(*p)->n_readers, 1) == 1)
+
+ while (__sync_lock_test_and_set (&(*p)->n_readers_lock, 1))
+ CLIB_PAUSE ();
+
+ (*p)->n_readers -= 1;
+ if ((*p)->n_readers == 0)
{
CLIB_MEMORY_BARRIER ();
(*p)->writer_lock = 0;
}
+
+ CLIB_MEMORY_BARRIER ();
+ (*p)->n_readers_lock = 0;
}
always_inline void
diff --git a/test/test_session.py b/test/test_session.py
index 7219ffd..80e27c0 100644
--- a/test/test_session.py
+++ b/test/test_session.py
@@ -3,6 +3,7 @@
import unittest
from framework import VppTestCase, VppTestRunner
+from vpp_ip_route import VppIpTable, VppIpRoute, VppRoutePath
class TestSession(VppTestCase):
@@ -15,7 +16,34 @@
def setUp(self):
super(TestSession, self).setUp()
+ self.vapi.session_enable_disable(is_enabled=1)
+ self.create_loopback_interfaces(range(2))
+
+ table_id = 0
+
+ for i in self.lo_interfaces:
+ i.admin_up()
+
+ if table_id != 0:
+ tbl = VppIpTable(self, table_id)
+ tbl.add_vpp_config()
+
+ i.set_table_ip4(table_id)
+ i.config_ip4()
+ table_id += 1
+
+ # Configure namespaces
+ self.vapi.app_namespace_add(namespace_id="0",
+ sw_if_index=self.loop0.sw_if_index)
+ self.vapi.app_namespace_add(namespace_id="1",
+ sw_if_index=self.loop1.sw_if_index)
+
def tearDown(self):
+ for i in self.lo_interfaces:
+ i.unconfig_ip4()
+ i.set_table_ip4(0)
+ i.admin_down()
+
super(TestSession, self).tearDown()
self.vapi.session_enable_disable(is_enabled=1)
@@ -25,7 +53,41 @@
if error:
self.logger.critical(error)
- self.assertEqual(error.find("Failed"), -1)
+ self.assertEqual(error.find("failed"), -1)
+
+ def test_segment_manager_alloc(self):
+ """ Session Segment Manager Multiple Segment Allocation """
+
+ # Add inter-table routes
+ ip_t01 = VppIpRoute(self, self.loop1.local_ip4, 32,
+ [VppRoutePath("0.0.0.0",
+ 0xffffffff,
+ nh_table_id=1)])
+ ip_t10 = VppIpRoute(self, self.loop0.local_ip4, 32,
+ [VppRoutePath("0.0.0.0",
+ 0xffffffff,
+ nh_table_id=0)], table_id=1)
+ ip_t01.add_vpp_config()
+ ip_t10.add_vpp_config()
+
+ # Start builtin server and client with small private segments
+ uri = "tcp://" + self.loop0.local_ip4 + "/1234"
+ error = self.vapi.cli("test echo server appns 0 fifo-size 64 " +
+ "private-segment-size 1m uri " + uri)
+ if error:
+ self.logger.critical(error)
+ self.assertEqual(error.find("failed"), -1)
+
+ error = self.vapi.cli("test echo client nclients 100 appns 1 " +
+ "no-output fifo-size 64 syn-timeout 2 " +
+ "private-segment-size 1m uri " + uri)
+ if error:
+ self.logger.critical(error)
+ self.assertEqual(error.find("failed"), -1)
+
+ # Delete inter-table routes
+ ip_t01.remove_vpp_config()
+ ip_t10.remove_vpp_config()
if __name__ == '__main__':
unittest.main(testRunner=VppTestRunner)