vlib: remove external thread management support
Now DPDK have API to register external threads so we can remove this
mess...
Type: improvement
Change-Id: I71a21f0cd94bd668aa406710c75a0bcc63fdc840
Signed-off-by: Damjan Marion <damarion@cisco.com>
diff --git a/src/plugins/dpdk/CMakeLists.txt b/src/plugins/dpdk/CMakeLists.txt
index 5de75e7..b53f4ad 100644
--- a/src/plugins/dpdk/CMakeLists.txt
+++ b/src/plugins/dpdk/CMakeLists.txt
@@ -131,7 +131,6 @@
SOURCES
buffer.c
main.c
- thread.c
device/cli.c
device/common.c
device/device.c
diff --git a/src/plugins/dpdk/device/cli.c b/src/plugins/dpdk/device/cli.c
index 0f771c6..ba922e2 100644
--- a/src/plugins/dpdk/device/cli.c
+++ b/src/plugins/dpdk/device/cli.c
@@ -373,7 +373,8 @@
* Example of how to display how many DPDK buffer test command has allocated:
* @cliexstart{show dpdk version}
* DPDK Version: DPDK 16.11.0
- * DPDK EAL init args: -c 1 -n 4 --huge-dir /run/vpp/hugepages --file-prefix vpp -w 0000:00:08.0 -w 0000:00:09.0 --master-lcore 0 --socket-mem 256
+ * DPDK EAL init args: --in-memory --no-telemetry --file-prefix vpp
+ * -w 0000:00:08.0 -w 0000:00:09.0
* @cliexend
?*/
/* *INDENT-OFF* */
diff --git a/src/plugins/dpdk/device/dpdk.h b/src/plugins/dpdk/device/dpdk.h
index 51f80a8..cc7305d 100644
--- a/src/plugins/dpdk/device/dpdk.h
+++ b/src/plugins/dpdk/device/dpdk.h
@@ -22,27 +22,12 @@
#include <rte_config.h>
-#include <rte_common.h>
-#include <rte_dev.h>
-#include <rte_memory.h>
#include <rte_eal.h>
-#include <rte_per_lcore.h>
-#include <rte_cycles.h>
-#include <rte_lcore.h>
-#include <rte_per_lcore.h>
-#include <rte_interrupts.h>
-#include <rte_pci.h>
-#include <rte_bus_vmbus.h>
-#include <rte_ether.h>
-#include <rte_ethdev.h>
-#include <rte_ring.h>
-#include <rte_mempool.h>
-#include <rte_mbuf.h>
-#include <rte_version.h>
-#include <rte_sched.h>
-#include <rte_net.h>
#include <rte_bus_pci.h>
-#include <rte_flow.h>
+#include <rte_bus_vmbus.h>
+#include <rte_ethdev.h>
+#include <rte_version.h>
+#include <rte_net.h>
#include <vnet/devices/devices.h>
@@ -313,13 +298,6 @@
#define DPDK_MAX_SIMD_BITWIDTH_256 256
#define DPDK_MAX_SIMD_BITWIDTH_512 512
- /* Required config parameters */
- u8 coremask_set_manually;
- u8 nchannels_set_manually;
- u32 coremask;
- u32 nchannels;
- u32 num_crypto_mbufs;
-
/*
* format interface names ala xxxEthernet%d/%d/%d instead of
* xxxEthernet%x/%x/%x.
diff --git a/src/plugins/dpdk/device/dpdk_priv.h b/src/plugins/dpdk/device/dpdk_priv.h
index 7f595ae..267c0b5 100644
--- a/src/plugins/dpdk/device/dpdk_priv.h
+++ b/src/plugins/dpdk/device/dpdk_priv.h
@@ -33,10 +33,6 @@
_(no-huge) \
_(vmware-tsc-map)
-#define foreach_eal_single_hyphen_mandatory_arg \
-_(coremask, c) \
-_(nchannels, n) \
-
#define foreach_eal_single_hyphen_arg \
_(mem-alloc-request, m) \
_(force-ranks, r)
diff --git a/src/plugins/dpdk/device/init.c b/src/plugins/dpdk/device/init.c
index 3808a5b..04d6f89 100644
--- a/src/plugins/dpdk/device/init.c
+++ b/src/plugins/dpdk/device/init.c
@@ -1504,11 +1504,6 @@
num_whitelisted++;
}
- else if (unformat (input, "num-mem-channels %d", &conf->nchannels))
- conf->nchannels_set_manually = 0;
- else if (unformat (input, "num-crypto-mbufs %d",
- &conf->num_crypto_mbufs))
- ;
else if (unformat (input, "uio-driver %s", &conf->uio_driver_name))
;
else if (unformat (input, "socket-mem %s", &socket_mem))
@@ -1585,17 +1580,6 @@
}
foreach_eal_single_hyphen_arg
#undef _
-#define _(a,b) \
- else if (unformat(input, #a " %s", &s)) \
- { \
- tmp = format (0, "-%s%c", #b, 0); \
- vec_add1 (conf->eal_init_args, tmp); \
- vec_add1 (s, 0); \
- vec_add1 (conf->eal_init_args, s); \
- conf->a##_set_manually = 1; \
- }
- foreach_eal_single_hyphen_mandatory_arg
-#undef _
else if (unformat (input, "default"))
;
@@ -1651,38 +1635,6 @@
if (error)
return error;
- /* I'll bet that -c and -n must be the first and second args... */
- if (!conf->coremask_set_manually)
- {
- vlib_thread_registration_t *tr;
- uword *coremask = 0;
- int i;
-
- /* main thread core */
- coremask = clib_bitmap_set (coremask, tm->main_lcore, 1);
-
- for (i = 0; i < vec_len (tm->registrations); i++)
- {
- tr = tm->registrations[i];
- coremask = clib_bitmap_or (coremask, tr->coremask);
- }
-
- vec_insert (conf->eal_init_args, 2, 1);
- conf->eal_init_args[1] = (u8 *) "-c";
- tmp = format (0, "%U%c", format_bitmap_hex, coremask, 0);
- conf->eal_init_args[2] = tmp;
- clib_bitmap_free (coremask);
- }
-
- if (!conf->nchannels_set_manually)
- {
- vec_insert (conf->eal_init_args, 2, 3);
- conf->eal_init_args[3] = (u8 *) "-n";
- tmp = format (0, "%d", conf->nchannels);
- vec_terminate_c_string (tmp);
- conf->eal_init_args[4] = tmp;
- }
-
if (no_pci == 0 && geteuid () == 0)
dpdk_bind_devices_to_uio (conf);
@@ -1693,7 +1645,6 @@
if (devconf->x == 0 && conf->default_devconf.x > 0) \
devconf->x = conf->default_devconf.x ;
- /* *INDENT-OFF* */
pool_foreach (devconf, conf->dev_confs) {
/* default per-device config items */
@@ -1737,17 +1688,9 @@
vec_add1 (conf->eal_init_args, tmp);
}
}
- /* *INDENT-ON* */
#undef _
- /* set master-lcore */
- tmp = format (0, "--main-lcore%c", 0);
- vec_add1 (conf->eal_init_args, tmp);
- tmp = format (0, "%u%c", tm->main_lcore, 0);
- vec_add1 (conf->eal_init_args, tmp);
-
-
if (socket_mem)
clib_warning ("socket-mem argument is deprecated");
@@ -1999,7 +1942,6 @@
dm->vnet_main = vnet_get_main ();
dm->conf = &dpdk_config_main;
- dm->conf->nchannels = 4;
vec_add1 (dm->conf->eal_init_args, (u8 *) "vnet");
/* Default vlib_buffer_t flags, DISABLES tcp/udp checksumming... */
@@ -2020,10 +1962,13 @@
VLIB_INIT_FUNCTION (dpdk_init);
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
+static clib_error_t *
+dpdk_worker_thread_init (vlib_main_t *vm)
+{
+ if (rte_thread_register () < 0)
+ clib_panic ("dpdk: cannot register thread %u - %s", vm->thread_index,
+ rte_strerror (rte_errno));
+ return 0;
+}
+
+VLIB_WORKER_INIT_FUNCTION (dpdk_worker_thread_init);
diff --git a/src/plugins/dpdk/thread.c b/src/plugins/dpdk/thread.c
deleted file mode 100644
index 3a3fcc6..0000000
--- a/src/plugins/dpdk/thread.c
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Copyright (c) 2017 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <rte_config.h>
-
-#include <rte_common.h>
-#include <rte_log.h>
-#include <rte_memory.h>
-#include <rte_memzone.h>
-#include <rte_tailq.h>
-#include <rte_eal.h>
-#include <rte_per_lcore.h>
-#include <rte_launch.h>
-#include <rte_atomic.h>
-#include <rte_cycles.h>
-#include <rte_prefetch.h>
-#include <rte_lcore.h>
-#include <rte_per_lcore.h>
-#include <rte_branch_prediction.h>
-#include <rte_interrupts.h>
-#include <rte_pci.h>
-#include <rte_random.h>
-#include <rte_debug.h>
-#include <rte_ether.h>
-#include <rte_ethdev.h>
-#include <rte_ring.h>
-#include <rte_mempool.h>
-#include <rte_mbuf.h>
-#include <rte_version.h>
-
-#include <vlib/vlib.h>
-#include <vnet/vnet.h>
-#include <dpdk/device/dpdk.h>
-#include <dpdk/device/dpdk_priv.h>
-
-static clib_error_t *
-dpdk_launch_thread (void *fp, vlib_worker_thread_t * w, unsigned lcore_id)
-{
- int r;
- r = rte_eal_remote_launch (fp, (void *) w, lcore_id);
- if (r)
- return clib_error_return (0, "Failed to launch thread %u", lcore_id);
- return 0;
-}
-
-static clib_error_t *
-dpdk_thread_set_lcore (u32 thread, u16 lcore)
-{
- return 0;
-}
-
-static vlib_thread_callbacks_t callbacks = {
- .vlib_launch_thread_cb = &dpdk_launch_thread,
- .vlib_thread_set_lcore_cb = &dpdk_thread_set_lcore,
-};
-
-static clib_error_t *
-dpdk_thread_init (vlib_main_t * vm)
-{
- vlib_thread_cb_register (vm, &callbacks);
- return 0;
-}
-
-VLIB_INIT_FUNCTION (dpdk_thread_init);
-
-/** @endcond */
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/vlib/threads.c b/src/vlib/threads.c
index 760aa8b..f4d95c4 100644
--- a/src/vlib/threads.c
+++ b/src/vlib/threads.c
@@ -175,6 +175,7 @@
vlib_thread_main_t *tm = &vlib_thread_main;
vlib_worker_thread_t *w;
vlib_thread_registration_t *tr;
+ cpu_set_t cpuset;
u32 n_vlib_mains = 1;
u32 first_index = 1;
u32 i;
@@ -223,17 +224,9 @@
tm->cpu_socket_bitmap = clib_bitmap_set (0, 0, 1);
/* pin main thread to main_lcore */
- if (tm->cb.vlib_thread_set_lcore_cb)
- {
- tm->cb.vlib_thread_set_lcore_cb (0, tm->main_lcore);
- }
- else
- {
- cpu_set_t cpuset;
- CPU_ZERO (&cpuset);
- CPU_SET (tm->main_lcore, &cpuset);
- pthread_setaffinity_np (pthread_self (), sizeof (cpu_set_t), &cpuset);
- }
+ CPU_ZERO (&cpuset);
+ CPU_SET (tm->main_lcore, &cpuset);
+ pthread_setaffinity_np (pthread_self (), sizeof (cpu_set_t), &cpuset);
/* Set up thread 0 */
vec_validate_aligned (vlib_worker_threads, 0, CLIB_CACHE_LINE_BYTES);
@@ -463,6 +456,8 @@
{
clib_mem_main_t *mm = &clib_mem_main;
vlib_thread_main_t *tm = &vlib_thread_main;
+ pthread_t worker;
+ cpu_set_t cpuset;
void *(*fp_arg) (void *) = fp;
void *numa_heap;
@@ -489,12 +484,6 @@
}
}
- if (tm->cb.vlib_launch_thread_cb && !w->registration->use_pthreads)
- return tm->cb.vlib_launch_thread_cb (fp, (void *) w, cpu_id);
- else
- {
- pthread_t worker;
- cpu_set_t cpuset;
CPU_ZERO (&cpuset);
CPU_SET (cpu_id, &cpuset);
@@ -505,7 +494,6 @@
return clib_error_return_unix (0, "pthread_setaffinity_np");
return 0;
- }
}
static clib_error_t *
@@ -1520,7 +1508,6 @@
{
vlib_global_main_t *vgm = vlib_get_global_main ();
vlib_worker_thread_t *w = (vlib_worker_thread_t *) arg;
- vlib_thread_main_t *tm = vlib_get_thread_main ();
vlib_main_t *vm = vlib_get_main ();
clib_error_t *e;
@@ -1540,10 +1527,6 @@
if (e)
clib_error_report (e);
- /* Wait until the dpdk init sequence is complete */
- while (tm->extern_thread_mgmt && tm->worker_thread_release == 0)
- vlib_worker_thread_barrier_check ();
-
vlib_worker_loop (vm);
}
@@ -1586,19 +1569,6 @@
return (fqm - tm->frame_queue_mains);
}
-int
-vlib_thread_cb_register (struct vlib_main_t *vm, vlib_thread_callbacks_t * cb)
-{
- vlib_thread_main_t *tm = vlib_get_thread_main ();
-
- if (tm->extern_thread_mgmt)
- return -1;
-
- tm->cb.vlib_launch_thread_cb = cb->vlib_launch_thread_cb;
- tm->extern_thread_mgmt = 1;
- return 0;
-}
-
void
vlib_process_signal_event_mt_helper (vlib_process_signal_event_mt_args_t *
args)
diff --git a/src/vlib/threads.h b/src/vlib/threads.h
index 91727ba..e406dde 100644
--- a/src/vlib/threads.h
+++ b/src/vlib/threads.h
@@ -235,13 +235,6 @@
typedef struct
{
- clib_error_t *(*vlib_launch_thread_cb) (void *fp, vlib_worker_thread_t * w,
- unsigned cpu_id);
- clib_error_t *(*vlib_thread_set_lcore_cb) (u32 thread, u16 cpu);
-} vlib_thread_callbacks_t;
-
-typedef struct
-{
/* Link list of registrations, built by constructors */
vlib_thread_registration_t *next;
@@ -252,10 +245,6 @@
vlib_worker_thread_t *worker_threads;
- /*
- * Launch all threads as pthreads,
- * not eal_rte_launch (strict affinity) threads
- */
int use_pthreads;
/* Number of vlib_main / vnet_main clones */
@@ -297,10 +286,6 @@
/* scheduling policy priority */
u32 sched_priority;
- /* callbacks */
- vlib_thread_callbacks_t cb;
- int extern_thread_mgmt;
-
/* NUMA-bound heap size */
uword numa_heap_size;
@@ -490,8 +475,6 @@
}
u8 *vlib_thread_stack_init (uword thread_index);
-int vlib_thread_cb_register (struct vlib_main_t *vm,
- vlib_thread_callbacks_t * cb);
extern void *rpc_call_main_thread_cb_fn;
void