Use thread local storage for thread index
This patch deprecates stack-based thread identification,
Also removes requirement that thread stacks are adjacent.
Finally, possibly annoying for some folks, it renames
all occurences of cpu_index and cpu_number with thread
index. Using word "cpu" is misleading here as thread can
be migrated ti different CPU, and also it is not related
to linux cpu index.
Change-Id: I68cdaf661e701d2336fc953dcb9978d10a70f7c1
Signed-off-by: Damjan Marion <damarion@cisco.com>
diff --git a/src/vnet/ipsec/esp.h b/src/vnet/ipsec/esp.h
index 50cac80..799003b 100644
--- a/src/vnet/ipsec/esp.h
+++ b/src/vnet/ipsec/esp.h
@@ -282,8 +282,8 @@
u8 * data, int data_len, u8 * signature, u8 use_esn, u32 seq_hi)
{
esp_main_t *em = &esp_main;
- u32 cpu_index = os_get_cpu_number ();
- HMAC_CTX *ctx = &(em->per_thread_data[cpu_index].hmac_ctx);
+ u32 thread_index = vlib_get_thread_index ();
+ HMAC_CTX *ctx = &(em->per_thread_data[thread_index].hmac_ctx);
const EVP_MD *md = NULL;
unsigned int len;
@@ -292,10 +292,10 @@
if (PREDICT_FALSE (em->esp_integ_algs[alg].md == 0))
return 0;
- if (PREDICT_FALSE (alg != em->per_thread_data[cpu_index].last_integ_alg))
+ if (PREDICT_FALSE (alg != em->per_thread_data[thread_index].last_integ_alg))
{
md = em->esp_integ_algs[alg].md;
- em->per_thread_data[cpu_index].last_integ_alg = alg;
+ em->per_thread_data[thread_index].last_integ_alg = alg;
}
HMAC_Init (ctx, key, key_len, md);
diff --git a/src/vnet/ipsec/esp_decrypt.c b/src/vnet/ipsec/esp_decrypt.c
index 7289b26..925d2b4 100644
--- a/src/vnet/ipsec/esp_decrypt.c
+++ b/src/vnet/ipsec/esp_decrypt.c
@@ -85,8 +85,8 @@
u8 * in, u8 * out, size_t in_len, u8 * key, u8 * iv)
{
esp_main_t *em = &esp_main;
- u32 cpu_index = os_get_cpu_number ();
- EVP_CIPHER_CTX *ctx = &(em->per_thread_data[cpu_index].decrypt_ctx);
+ u32 thread_index = vlib_get_thread_index ();
+ EVP_CIPHER_CTX *ctx = &(em->per_thread_data[thread_index].decrypt_ctx);
const EVP_CIPHER *cipher = NULL;
int out_len;
@@ -95,10 +95,11 @@
if (PREDICT_FALSE (em->esp_crypto_algs[alg].type == 0))
return;
- if (PREDICT_FALSE (alg != em->per_thread_data[cpu_index].last_decrypt_alg))
+ if (PREDICT_FALSE
+ (alg != em->per_thread_data[thread_index].last_decrypt_alg))
{
cipher = em->esp_crypto_algs[alg].type;
- em->per_thread_data[cpu_index].last_decrypt_alg = alg;
+ em->per_thread_data[thread_index].last_decrypt_alg = alg;
}
EVP_DecryptInit_ex (ctx, cipher, NULL, key, iv);
@@ -117,11 +118,11 @@
u32 *recycle = 0;
from = vlib_frame_vector_args (from_frame);
n_left_from = from_frame->n_vectors;
- u32 cpu_index = os_get_cpu_number ();
+ u32 thread_index = vlib_get_thread_index ();
ipsec_alloc_empty_buffers (vm, im);
- u32 *empty_buffers = im->empty_buffers[cpu_index];
+ u32 *empty_buffers = im->empty_buffers[thread_index];
if (PREDICT_FALSE (vec_len (empty_buffers) < n_left_from))
{
diff --git a/src/vnet/ipsec/esp_encrypt.c b/src/vnet/ipsec/esp_encrypt.c
index 44ae229..b2bc4e0 100644
--- a/src/vnet/ipsec/esp_encrypt.c
+++ b/src/vnet/ipsec/esp_encrypt.c
@@ -88,8 +88,8 @@
u8 * in, u8 * out, size_t in_len, u8 * key, u8 * iv)
{
esp_main_t *em = &esp_main;
- u32 cpu_index = os_get_cpu_number ();
- EVP_CIPHER_CTX *ctx = &(em->per_thread_data[cpu_index].encrypt_ctx);
+ u32 thread_index = vlib_get_thread_index ();
+ EVP_CIPHER_CTX *ctx = &(em->per_thread_data[thread_index].encrypt_ctx);
const EVP_CIPHER *cipher = NULL;
int out_len;
@@ -98,10 +98,11 @@
if (PREDICT_FALSE (em->esp_crypto_algs[alg].type == IPSEC_CRYPTO_ALG_NONE))
return;
- if (PREDICT_FALSE (alg != em->per_thread_data[cpu_index].last_encrypt_alg))
+ if (PREDICT_FALSE
+ (alg != em->per_thread_data[thread_index].last_encrypt_alg))
{
cipher = em->esp_crypto_algs[alg].type;
- em->per_thread_data[cpu_index].last_encrypt_alg = alg;
+ em->per_thread_data[thread_index].last_encrypt_alg = alg;
}
EVP_EncryptInit_ex (ctx, cipher, NULL, key, iv);
@@ -119,11 +120,11 @@
n_left_from = from_frame->n_vectors;
ipsec_main_t *im = &ipsec_main;
u32 *recycle = 0;
- u32 cpu_index = os_get_cpu_number ();
+ u32 thread_index = vlib_get_thread_index ();
ipsec_alloc_empty_buffers (vm, im);
- u32 *empty_buffers = im->empty_buffers[cpu_index];
+ u32 *empty_buffers = im->empty_buffers[thread_index];
if (PREDICT_FALSE (vec_len (empty_buffers) < n_left_from))
{
diff --git a/src/vnet/ipsec/ikev2.c b/src/vnet/ipsec/ikev2.c
index 2c1074d..3f9978a 100644
--- a/src/vnet/ipsec/ikev2.c
+++ b/src/vnet/ipsec/ikev2.c
@@ -303,16 +303,16 @@
ikev2_delete_sa (ikev2_sa_t * sa)
{
ikev2_main_t *km = &ikev2_main;
- u32 cpu_index = os_get_cpu_number ();
+ u32 thread_index = vlib_get_thread_index ();
uword *p;
ikev2_sa_free_all_vec (sa);
- p = hash_get (km->per_thread_data[cpu_index].sa_by_rspi, sa->rspi);
+ p = hash_get (km->per_thread_data[thread_index].sa_by_rspi, sa->rspi);
if (p)
{
- hash_unset (km->per_thread_data[cpu_index].sa_by_rspi, sa->rspi);
- pool_put (km->per_thread_data[cpu_index].sas, sa);
+ hash_unset (km->per_thread_data[thread_index].sa_by_rspi, sa->rspi);
+ pool_put (km->per_thread_data[thread_index].sas, sa);
}
}
@@ -776,29 +776,31 @@
ikev2_sa_t *tmp;
u32 i, *delete = 0;
ikev2_child_sa_t *c;
- u32 cpu_index = os_get_cpu_number ();
+ u32 thread_index = vlib_get_thread_index ();
if (!sa->initial_contact)
return;
/* find old IKE SAs with the same authenticated identity */
/* *INDENT-OFF* */
- pool_foreach (tmp, km->per_thread_data[cpu_index].sas, ({
+ pool_foreach (tmp, km->per_thread_data[thread_index].sas, ({
if (tmp->i_id.type != sa->i_id.type ||
vec_len(tmp->i_id.data) != vec_len(sa->i_id.data) ||
memcmp(sa->i_id.data, tmp->i_id.data, vec_len(sa->i_id.data)))
continue;
if (sa->rspi != tmp->rspi)
- vec_add1(delete, tmp - km->per_thread_data[cpu_index].sas);
+ vec_add1(delete, tmp - km->per_thread_data[thread_index].sas);
}));
/* *INDENT-ON* */
for (i = 0; i < vec_len (delete); i++)
{
- tmp = pool_elt_at_index (km->per_thread_data[cpu_index].sas, delete[i]);
- vec_foreach (c, tmp->childs)
- ikev2_delete_tunnel_interface (km->vnet_main, tmp, c);
+ tmp =
+ pool_elt_at_index (km->per_thread_data[thread_index].sas, delete[i]);
+ vec_foreach (c,
+ tmp->childs) ikev2_delete_tunnel_interface (km->vnet_main,
+ tmp, c);
ikev2_delete_sa (tmp);
}
@@ -1922,10 +1924,10 @@
{
ikev2_main_t *km = &ikev2_main;
ikev2_sa_t *sa;
- u32 cpu_index = os_get_cpu_number ();
+ u32 thread_index = vlib_get_thread_index ();
/* *INDENT-OFF* */
- pool_foreach (sa, km->per_thread_data[cpu_index].sas, ({
+ pool_foreach (sa, km->per_thread_data[thread_index].sas, ({
if (sa->ispi == clib_net_to_host_u64(ike->ispi) &&
sa->iaddr.as_u32 == iaddr.as_u32 &&
sa->raddr.as_u32 == raddr.as_u32)
@@ -2036,7 +2038,7 @@
u32 n_left_from, *from, *to_next;
ikev2_next_t next_index;
ikev2_main_t *km = &ikev2_main;
- u32 cpu_index = os_get_cpu_number ();
+ u32 thread_index = vlib_get_thread_index ();
from = vlib_frame_vector_args (frame);
n_left_from = frame->n_vectors;
@@ -2134,11 +2136,14 @@
if (sa0->state == IKEV2_STATE_SA_INIT)
{
/* add SA to the pool */
- pool_get (km->per_thread_data[cpu_index].sas, sa0);
+ pool_get (km->per_thread_data[thread_index].sas,
+ sa0);
clib_memcpy (sa0, &sa, sizeof (*sa0));
- hash_set (km->per_thread_data[cpu_index].sa_by_rspi,
+ hash_set (km->
+ per_thread_data[thread_index].sa_by_rspi,
sa0->rspi,
- sa0 - km->per_thread_data[cpu_index].sas);
+ sa0 -
+ km->per_thread_data[thread_index].sas);
}
else
{
@@ -2169,11 +2174,11 @@
if (sa0->state == IKEV2_STATE_SA_INIT)
{
/* add SA to the pool */
- pool_get (km->per_thread_data[cpu_index].sas, sa0);
+ pool_get (km->per_thread_data[thread_index].sas, sa0);
clib_memcpy (sa0, &sa, sizeof (*sa0));
- hash_set (km->per_thread_data[cpu_index].sa_by_rspi,
+ hash_set (km->per_thread_data[thread_index].sa_by_rspi,
sa0->rspi,
- sa0 - km->per_thread_data[cpu_index].sas);
+ sa0 - km->per_thread_data[thread_index].sas);
}
else
{
@@ -2184,12 +2189,13 @@
else if (ike0->exchange == IKEV2_EXCHANGE_IKE_AUTH)
{
uword *p;
- p = hash_get (km->per_thread_data[cpu_index].sa_by_rspi,
+ p = hash_get (km->per_thread_data[thread_index].sa_by_rspi,
clib_net_to_host_u64 (ike0->rspi));
if (p)
{
- sa0 = pool_elt_at_index (km->per_thread_data[cpu_index].sas,
- p[0]);
+ sa0 =
+ pool_elt_at_index (km->per_thread_data[thread_index].sas,
+ p[0]);
r = ikev2_retransmit_resp (sa0, ike0);
if (r == 1)
@@ -2240,12 +2246,13 @@
else if (ike0->exchange == IKEV2_EXCHANGE_INFORMATIONAL)
{
uword *p;
- p = hash_get (km->per_thread_data[cpu_index].sa_by_rspi,
+ p = hash_get (km->per_thread_data[thread_index].sa_by_rspi,
clib_net_to_host_u64 (ike0->rspi));
if (p)
{
- sa0 = pool_elt_at_index (km->per_thread_data[cpu_index].sas,
- p[0]);
+ sa0 =
+ pool_elt_at_index (km->per_thread_data[thread_index].sas,
+ p[0]);
r = ikev2_retransmit_resp (sa0, ike0);
if (r == 1)
@@ -2305,12 +2312,13 @@
else if (ike0->exchange == IKEV2_EXCHANGE_CREATE_CHILD_SA)
{
uword *p;
- p = hash_get (km->per_thread_data[cpu_index].sa_by_rspi,
+ p = hash_get (km->per_thread_data[thread_index].sa_by_rspi,
clib_net_to_host_u64 (ike0->rspi));
if (p)
{
- sa0 = pool_elt_at_index (km->per_thread_data[cpu_index].sas,
- p[0]);
+ sa0 =
+ pool_elt_at_index (km->per_thread_data[thread_index].sas,
+ p[0]);
r = ikev2_retransmit_resp (sa0, ike0);
if (r == 1)
diff --git a/src/vnet/ipsec/ipsec.h b/src/vnet/ipsec/ipsec.h
index 58f0f14..c884e36 100644
--- a/src/vnet/ipsec/ipsec.h
+++ b/src/vnet/ipsec/ipsec.h
@@ -324,21 +324,21 @@
always_inline void
ipsec_alloc_empty_buffers (vlib_main_t * vm, ipsec_main_t * im)
{
- u32 cpu_index = os_get_cpu_number ();
- uword l = vec_len (im->empty_buffers[cpu_index]);
+ u32 thread_index = vlib_get_thread_index ();
+ uword l = vec_len (im->empty_buffers[thread_index]);
uword n_alloc = 0;
if (PREDICT_FALSE (l < VLIB_FRAME_SIZE))
{
- if (!im->empty_buffers[cpu_index])
+ if (!im->empty_buffers[thread_index])
{
- vec_alloc (im->empty_buffers[cpu_index], 2 * VLIB_FRAME_SIZE);
+ vec_alloc (im->empty_buffers[thread_index], 2 * VLIB_FRAME_SIZE);
}
- n_alloc = vlib_buffer_alloc (vm, im->empty_buffers[cpu_index] + l,
+ n_alloc = vlib_buffer_alloc (vm, im->empty_buffers[thread_index] + l,
2 * VLIB_FRAME_SIZE - l);
- _vec_len (im->empty_buffers[cpu_index]) = l + n_alloc;
+ _vec_len (im->empty_buffers[thread_index]) = l + n_alloc;
}
}
diff --git a/src/vnet/ipsec/ipsec_if.c b/src/vnet/ipsec/ipsec_if.c
index dc88200..ed12489 100644
--- a/src/vnet/ipsec/ipsec_if.c
+++ b/src/vnet/ipsec/ipsec_if.c
@@ -99,7 +99,7 @@
ipsec_add_del_tunnel_if_rpc_callback (ipsec_add_del_tunnel_args_t * a)
{
vnet_main_t *vnm = vnet_get_main ();
- ASSERT (os_get_cpu_number () == 0);
+ ASSERT (vlib_get_thread_index () == 0);
return ipsec_add_del_tunnel_if_internal (vnm, a);
}