vlib: make runtime_data thread-local
Change-Id: I4aa3e7e42fb81211de1aed07dc7befee87a1e18b
Signed-off-by: Damjan Marion <damarion@cisco.com>
diff --git a/src/vlib/init.h b/src/vlib/init.h
index 4fa5b30..12db3f9 100644
--- a/src/vlib/init.h
+++ b/src/vlib/init.h
@@ -109,6 +109,7 @@
}
#define VLIB_INIT_FUNCTION(x) VLIB_DECLARE_INIT_FUNCTION(x,init)
+#define VLIB_WORKER_INIT_FUNCTION(x) VLIB_DECLARE_INIT_FUNCTION(x,worker_init)
#define VLIB_MAIN_LOOP_ENTER_FUNCTION(x) \
VLIB_DECLARE_INIT_FUNCTION(x,main_loop_enter)
diff --git a/src/vlib/main.h b/src/vlib/main.h
index a6d50b3..98bc823 100644
--- a/src/vlib/main.h
+++ b/src/vlib/main.h
@@ -162,6 +162,7 @@
/* List of init functions to call, setup by constructors */
_vlib_init_function_list_elt_t *init_function_registrations;
+ _vlib_init_function_list_elt_t *worker_init_function_registrations;
_vlib_init_function_list_elt_t *main_loop_enter_function_registrations;
_vlib_init_function_list_elt_t *main_loop_exit_function_registrations;
_vlib_init_function_list_elt_t *api_init_function_registrations;
diff --git a/src/vlib/node.c b/src/vlib/node.c
index c419a13..dc0a4de 100644
--- a/src/vlib/node.c
+++ b/src/vlib/node.c
@@ -434,9 +434,7 @@
rt->errors[i] = vlib_error_set (n->index, i);
STATIC_ASSERT_SIZEOF (vlib_node_runtime_t, 128);
- ASSERT (vec_len (n->runtime_data) <=
- sizeof (vlib_node_runtime_t) -
- STRUCT_OFFSET_OF (vlib_node_runtime_t, runtime_data));
+ ASSERT (vec_len (n->runtime_data) <= VLIB_NODE_RUNTIME_DATA_SIZE);
if (vec_len (n->runtime_data) > 0)
clib_memcpy (rt->runtime_data, n->runtime_data,
diff --git a/src/vlib/node.h b/src/vlib/node.h
index b624e9d..2a532cc 100644
--- a/src/vlib/node.h
+++ b/src/vlib/node.h
@@ -411,65 +411,68 @@
typedef struct vlib_node_runtime_t
{
- CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
- /* Node function to call. */
- vlib_node_function_t *function;
+ CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); /**< cacheline mark */
- /* Vector of errors for this node. */
- vlib_error_t *errors;
+ vlib_node_function_t *function; /**< Node function to call. */
- /* Number of clock cycles. */
- u32 clocks_since_last_overflow;
+ vlib_error_t *errors; /**< Vector of errors for this node. */
- /* Maximum clock cycle for an invocation. */
- u32 max_clock;
+ u32 clocks_since_last_overflow; /**< Number of clock cycles. */
- /* Number of vectors in the recorded max_clock. */
- u32 max_clock_n;
+ u32 max_clock; /**< Maximum clock cycle for an
+ invocation. */
- /* Number of calls. */
- u32 calls_since_last_overflow;
+ u32 max_clock_n; /**< Number of vectors in the recorded
+ max_clock. */
- /* Number of vector elements processed by this node. */
- u32 vectors_since_last_overflow;
+ u32 calls_since_last_overflow; /**< Number of calls. */
- /* Start of next frames for this node. */
- u32 next_frame_index;
+ u32 vectors_since_last_overflow; /**< Number of vector elements
+ processed by this node. */
- /* Node index. */
- u32 node_index;
+ u32 next_frame_index; /**< Start of next frames for this
+ node. */
- /* For input nodes: decremented on each main loop interation until it reaches zero
- and function is called. Allows some input nodes to be called
- more than others. */
- u32 input_main_loops_per_call;
+ u32 node_index; /**< Node index. */
- /* Saved main loop counter of last dispatch of this node. */
- u32 main_loop_count_last_dispatch;
+ u32 input_main_loops_per_call; /**< For input nodes: decremented
+ on each main loop interation until
+ it reaches zero and function is
+ called. Allows some input nodes to
+ be called more than others. */
+
+ u32 main_loop_count_last_dispatch; /**< Saved main loop counter of last
+ dispatch of this node. */
u32 main_loop_vector_stats[2];
- /* Copy of main node flags. */
- u16 flags;
+ u16 flags; /**< Copy of main node flags. */
- /* Input node state. */
- u16 state;
+ u16 state; /**< Input node state. */
u16 n_next_nodes;
- /* Next frame index that vector arguments were last enqueued to
- last time this node ran. Set to zero before first run
- of this node. */
- u16 cached_next_index;
+ u16 cached_next_index; /**< Next frame index that vector
+ arguments were last enqueued to
+ last time this node ran. Set to
+ zero before first run of this
+ node. */
- /* CPU this node runs on */
- u16 cpu_index;
+ u16 cpu_index; /**< CPU this node runs on */
- /* Function dependent node-runtime. */
- u8 runtime_data[0];
+ u8 runtime_data[0]; /**< Function dependent
+ node-runtime data. This data is
+ thread local, and it is not
+ cloned from main thread. It needs
+ to be initialized for each thread
+ before it is used unless
+ runtime_data template exists in
+ vlib_node_t. */
}
vlib_node_runtime_t;
+#define VLIB_NODE_RUNTIME_DATA_SIZE (sizeof (vlib_node_runtime_t) - STRUCT_OFFSET_OF (vlib_node_runtime_t, runtime_data))
+
typedef struct
{
/* Number of allocated frames for this scalar/vector size. */
diff --git a/src/vlib/threads.c b/src/vlib/threads.c
index 07dbff3..3756c3f 100644
--- a/src/vlib/threads.c
+++ b/src/vlib/threads.c
@@ -633,6 +633,8 @@
vm_clone->cpu_index = worker_thread_index;
vm_clone->heap_base = w->thread_mheap;
vm_clone->mbuf_alloc_list = 0;
+ vm_clone->init_functions_called =
+ hash_create (0, /* value bytes */ 0);
memset (&vm_clone->random_buffer, 0,
sizeof (vm_clone->random_buffer));
@@ -674,11 +676,33 @@
}
nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL] =
vec_dup (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]);
+ vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])
+ {
+ vlib_node_t *n = vlib_get_node (vm, rt->node_index);
+ rt->cpu_index = vm_clone->cpu_index;
+ /* copy initial runtime_data from node */
+ if (n->runtime_data_bytes > 0)
+ clib_memcpy (rt->runtime_data, n->runtime_data,
+ VLIB_NODE_RUNTIME_DATA_SIZE);
+ else if (CLIB_DEBUG > 0)
+ memset (rt->runtime_data, 0xfe,
+ VLIB_NODE_RUNTIME_DATA_SIZE);
+ }
nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT] =
vec_dup (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT]);
vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])
+ {
+ vlib_node_t *n = vlib_get_node (vm, rt->node_index);
rt->cpu_index = vm_clone->cpu_index;
+ /* copy initial runtime_data from node */
+ if (n->runtime_data_bytes > 0)
+ clib_memcpy (rt->runtime_data, n->runtime_data,
+ VLIB_NODE_RUNTIME_DATA_SIZE);
+ else if (CLIB_DEBUG > 0)
+ memset (rt->runtime_data, 0xfe,
+ VLIB_NODE_RUNTIME_DATA_SIZE);
+ }
nm_clone->processes = vec_dup (nm->processes);
@@ -926,26 +950,51 @@
clib_mem_free (old_nodes_clone[j]);
vec_free (old_nodes_clone);
- vec_free (nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]);
+ /* re-clone internal nodes */
+ old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL];
nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL] =
vec_dup (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]);
- /* clone input node runtime */
- old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT];
-
- nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT] =
- vec_dup (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT]);
-
- vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])
+ vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL])
{
+ vlib_node_t *n = vlib_get_node (vm, rt->node_index);
rt->cpu_index = vm_clone->cpu_index;
+ /* copy runtime_data, will be overwritten later for existing rt */
+ clib_memcpy (rt->runtime_data, n->runtime_data,
+ VLIB_NODE_RUNTIME_DATA_SIZE);
}
for (j = 0; j < vec_len (old_rt); j++)
{
rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index);
rt->state = old_rt[j].state;
+ clib_memcpy (rt->runtime_data, old_rt[j].runtime_data,
+ VLIB_NODE_RUNTIME_DATA_SIZE);
+ }
+
+ vec_free (old_rt);
+
+ /* re-clone input nodes */
+ old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT];
+ nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT] =
+ vec_dup (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT]);
+
+ vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])
+ {
+ vlib_node_t *n = vlib_get_node (vm, rt->node_index);
+ rt->cpu_index = vm_clone->cpu_index;
+ /* copy runtime_data, will be overwritten later for existing rt */
+ clib_memcpy (rt->runtime_data, n->runtime_data,
+ VLIB_NODE_RUNTIME_DATA_SIZE);
+ }
+
+ for (j = 0; j < vec_len (old_rt); j++)
+ {
+ rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index);
+ rt->state = old_rt[j].state;
+ clib_memcpy (rt->runtime_data, old_rt[j].runtime_data,
+ VLIB_NODE_RUNTIME_DATA_SIZE);
}
vec_free (old_rt);
@@ -1342,6 +1391,7 @@
vlib_worker_thread_t *w = (vlib_worker_thread_t *) arg;
vlib_thread_main_t *tm = vlib_get_thread_main ();
vlib_main_t *vm = vlib_get_main ();
+ clib_error_t *e;
ASSERT (vm->cpu_index == os_get_cpu_number ());
@@ -1349,6 +1399,11 @@
clib_time_init (&vm->clib_time);
clib_mem_set_heap (w->thread_mheap);
+ e = vlib_call_init_exit_functions
+ (vm, vm->worker_init_function_registrations, 1 /* call_once */ );
+ if (e)
+ clib_error_report (e);
+
/* Wait until the dpdk init sequence is complete */
while (tm->extern_thread_mgmt && tm->worker_thread_release == 0)
vlib_worker_thread_barrier_check ();