vcl: separate binary api connections per thread worker
Type: fix
Change-Id: I2d72efc74a3b0a5b9e4da265475b1b01bf361125
Signed-off-by: Florin Coras <fcoras@cisco.com>
diff --git a/src/vcl/vcl_bapi.c b/src/vcl/vcl_bapi.c
index a98b181..194c136 100644
--- a/src/vcl/vcl_bapi.c
+++ b/src/vcl/vcl_bapi.c
@@ -137,7 +137,8 @@
if (mp->n_fds)
{
vec_validate (fds, mp->n_fds);
- if (vl_socket_client_recv_fd_msg (fds, mp->n_fds, 5))
+ if (vl_socket_client_recv_fd_msg2 (&wrk->bapi_sock_ctx, fds, mp->n_fds,
+ 5))
goto failed;
if (mp->fd_flags & SESSION_FD_F_VPP_MQ_SEGMENT)
@@ -217,7 +218,8 @@
if (mp->n_fds)
{
vec_validate (fds, mp->n_fds);
- if (vl_socket_client_recv_fd_msg (fds, mp->n_fds, 5))
+ if (vl_socket_client_recv_fd_msg2 (&wrk->bapi_sock_ctx, fds, mp->n_fds,
+ 5))
goto failed;
if (mp->fd_flags & SESSION_FD_F_VPP_MQ_SEGMENT)
@@ -510,19 +512,25 @@
vppcom_connect_to_vpp (char *app_name)
{
vcl_worker_t *wrk = vcl_worker_get_current ();
- api_main_t *am = vlibapi_get_main ();
vppcom_cfg_t *vcl_cfg = &vcm->cfg;
+ api_main_t *am;
+
+ vlibapi_set_main (&wrk->bapi_api_ctx);
+ vlibapi_set_memory_client_main (&wrk->bapi_shm_ctx);
+ vppcom_api_hookup ();
if (vcl_cfg->vpp_api_socket_name)
{
- if (vl_socket_client_connect ((char *) vcl_cfg->vpp_api_socket_name,
- app_name, 0 /* default rx/tx buffer */ ))
+ if (vl_socket_client_connect2 (&wrk->bapi_sock_ctx,
+ (char *) vcl_cfg->vpp_api_socket_name,
+ app_name, 0 /* default rx/tx buffer */ ))
{
VERR ("app (%s) socket connect failed!", app_name);
return VPPCOM_ECONNREFUSED;
}
- if (vl_socket_client_init_shm (0, 1 /* want_pthread */ ))
+ if (vl_socket_client_init_shm2 (&wrk->bapi_sock_ctx, 0,
+ 1 /* want_pthread */ ))
{
VERR ("app (%s) init shm failed!", app_name);
return VPPCOM_ECONNREFUSED;
@@ -533,6 +541,8 @@
if (!vcl_cfg->vpp_api_filename)
vcl_cfg->vpp_api_filename = format (0, "/vpe-api%c", 0);
+ vl_set_memory_root_path ((char *) vcl_cfg->vpp_api_chroot);
+
VDBG (0, "app (%s) connecting to VPP api (%s)...",
app_name, vcl_cfg->vpp_api_filename);
@@ -542,9 +552,9 @@
VERR ("app (%s) connect failed!", app_name);
return VPPCOM_ECONNREFUSED;
}
-
}
+ am = vlibapi_get_main ();
wrk->vl_input_queue = am->shmem_hdr->vl_input_queue;
wrk->my_client_index = (u32) am->my_client_index;
wrk->wrk_state = STATE_APP_CONN_VPP;
diff --git a/src/vcl/vcl_cfg.c b/src/vcl/vcl_cfg.c
index cce60ba..3be6de0 100644
--- a/src/vcl/vcl_cfg.c
+++ b/src/vcl/vcl_cfg.c
@@ -228,7 +228,7 @@
int fd;
unformat_input_t _input, *input = &_input;
unformat_input_t _line_input, *line_input = &_line_input;
- u8 vc_cfg_input = 0, *chroot_path;
+ u8 vc_cfg_input = 0;
struct stat s;
u32 uid, gid, q_len;
@@ -282,19 +282,12 @@
VCFG_DBG (0, "VCL<%d>: configured max-workers %u", getpid (),
vcl_cfg->max_workers);
}
- else if (unformat (line_input, "api-prefix %s", &chroot_path))
+ else if (unformat (line_input, "api-prefix %s",
+ &vcl_cfg->vpp_api_chroot))
{
- vec_terminate_c_string (chroot_path);
- if (vcl_cfg->vpp_api_filename)
- vec_free (vcl_cfg->vpp_api_filename);
- vcl_cfg->vpp_api_filename = format (0, "/%s-vpe-api%c",
- chroot_path, 0);
- vl_set_memory_root_path ((char *) chroot_path);
-
- VCFG_DBG (0, "VCL<%d>: configured api-prefix (%s) and api "
- "filename (%s)", getpid (), chroot_path,
- vcl_cfg->vpp_api_filename);
- chroot_path = 0; /* Don't vec_free() it! */
+ vec_terminate_c_string (vcl_cfg->vpp_api_chroot);
+ VCFG_DBG (0, "VCL<%d>: configured api-prefix (%s) ", getpid (),
+ vcl_cfg->vpp_api_chroot);
}
else if (unformat (line_input, "api-socket-name %s",
&vcl_cfg->vpp_api_socket_name))
@@ -563,17 +556,15 @@
vppcom_cfg_heapsize (conf_fname);
vppcom_cfg_read_file (conf_fname);
+ /* Regrab cfg after heap initialization */
+ vcl_cfg = &vcm->cfg;
env_var_str = getenv (VPPCOM_ENV_API_PREFIX);
if (env_var_str)
{
- if (vcl_cfg->vpp_api_filename)
- vec_free (vcl_cfg->vpp_api_filename);
- vcl_cfg->vpp_api_filename = format (0, "/%s-vpe-api%c", env_var_str, 0);
- vl_set_memory_root_path ((char *) env_var_str);
-
- VCFG_DBG (0, "VCL<%d>: configured api prefix (%s) and filename (%s) "
- "from " VPPCOM_ENV_API_PREFIX "!", getpid (), env_var_str,
- vcl_cfg->vpp_api_filename);
+ vcl_cfg->vpp_api_chroot = format (0, "%s", env_var_str);
+ vec_terminate_c_string (vcl_cfg->vpp_api_chroot);
+ VCFG_DBG (0, "VCL<%d>: configured api prefix (%s) from "
+ VPPCOM_ENV_API_PREFIX "!", getpid (), env_var_str);
}
env_var_str = getenv (VPPCOM_ENV_APP_NAMESPACE_ID);
if (env_var_str)
diff --git a/src/vcl/vcl_private.h b/src/vcl/vcl_private.h
index 8fdf755..9116111 100644
--- a/src/vcl/vcl_private.h
+++ b/src/vcl/vcl_private.h
@@ -211,6 +211,7 @@
char *event_log_path;
u8 *vpp_api_filename;
u8 *vpp_api_socket_name;
+ u8 *vpp_api_chroot;
u32 tls_engine;
} vppcom_cfg_t;
@@ -298,6 +299,9 @@
u32 forked_child;
+ socket_client_main_t bapi_sock_ctx;
+ memory_client_main_t bapi_shm_ctx;
+ api_main_t bapi_api_ctx;
} vcl_worker_t;
typedef struct vppcom_main_t_
diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c
index 0bbdea5..ce47766 100644
--- a/src/vcl/vppcom.c
+++ b/src/vcl/vppcom.c
@@ -1055,7 +1055,6 @@
vcl_worker_alloc_and_init ();
/* API hookup and connect to VPP */
- vppcom_api_hookup ();
vcl_elog_init (vcm);
vcm->app_state = STATE_APP_START;
rv = vppcom_connect_to_vpp (app_name);
@@ -3590,11 +3589,21 @@
int
vppcom_worker_register (void)
{
+ vcl_worker_t *wrk;
+ u8 *wrk_name = 0;
+ int rv;
+
if (!vcl_worker_alloc_and_init ())
return VPPCOM_EEXIST;
- if (vcl_worker_set_bapi ())
- return VPPCOM_EEXIST;
+ wrk = vcl_worker_get_current ();
+ wrk_name = format (0, "%s-wrk-%u", vcm->app_name, wrk->wrk_index);
+
+ rv = vppcom_connect_to_vpp ((char *) wrk_name);
+ vec_free (wrk_name);
+
+ if (rv)
+ return VPPCOM_EFAULT;
if (vcl_worker_register_with_vpp ())
return VPPCOM_EEXIST;
diff --git a/src/vlibmemory/socket_client.c b/src/vlibmemory/socket_client.c
index 5483110..b7d4f2d 100644
--- a/src/vlibmemory/socket_client.c
+++ b/src/vlibmemory/socket_client.c
@@ -260,10 +260,10 @@
vl_socket_client_enable_disable2 (socket_client_ctx, enable);
}
-clib_error_t *
-vl_sock_api_recv_fd_msg (int socket_fd, int fds[], int n_fds, u32 wait)
+static clib_error_t *
+vl_sock_api_recv_fd_msg_internal (socket_client_main_t * scm, int fds[],
+ int n_fds, u32 wait)
{
- socket_client_main_t *scm = socket_client_ctx;
char msgbuf[16];
char ctl[CMSG_SPACE (sizeof (int) * n_fds)
+ CMSG_SPACE (sizeof (struct ucred))];
@@ -275,8 +275,11 @@
pid_t pid __attribute__ ((unused));
uid_t uid __attribute__ ((unused));
gid_t gid __attribute__ ((unused));
+ int socket_fd;
f64 timeout;
+ socket_fd = scm->client_socket.fd;
+
iov[0].iov_base = msgbuf;
iov[0].iov_len = 5;
mh.msg_iov = iov;
@@ -324,6 +327,26 @@
return 0;
}
+clib_error_t *
+vl_sock_api_recv_fd_msg (int socket_fd, int fds[], int n_fds, u32 wait)
+{
+ return vl_sock_api_recv_fd_msg_internal (socket_client_ctx, fds, n_fds,
+ wait);
+}
+
+clib_error_t *
+vl_sock_api_recv_fd_msg2 (socket_client_main_t * scm, int socket_fd,
+ int fds[], int n_fds, u32 wait)
+{
+ socket_client_main_t *old_ctx;
+ clib_error_t *error;
+
+ old_ctx = vl_socket_client_ctx_push (scm);
+ error = vl_sock_api_recv_fd_msg_internal (scm, fds, n_fds, wait);
+ vl_socket_client_ctx_pop (old_ctx);
+ return error;
+}
+
static void vl_api_sock_init_shm_reply_t_handler
(vl_api_sock_init_shm_reply_t * mp)
{
@@ -562,7 +585,7 @@
{
if (!scm->socket_fd)
return clib_error_return (0, "no socket");
- return vl_sock_api_recv_fd_msg (scm->client_socket.fd, fds, n_fds, wait);
+ return vl_sock_api_recv_fd_msg_internal (scm, fds, n_fds, wait);
}
clib_error_t *