ipsec: Layout and prefetching of SA struct
Type: improvement
- collect all DP used variables onto 1st or 2nd cache line
- prefetch the 2nd cache line
- in encrypt prefetch the likely location of the trailer.
Signed-off-by: Neale Ranns <nranns@cisco.com>
Change-Id: I44d58f8d2d469ff71a4f4a71578e7cc1acaeba43
diff --git a/src/vnet/ipsec/esp_decrypt.c b/src/vnet/ipsec/esp_decrypt.c
index 75092cc..ff9fc0c 100644
--- a/src/vnet/ipsec/esp_decrypt.c
+++ b/src/vnet/ipsec/esp_decrypt.c
@@ -1096,6 +1096,9 @@
current_sa_index = vnet_buffer (b[0])->ipsec.sad_index;
sa0 = pool_elt_at_index (im->sad, current_sa_index);
+
+ /* fetch the second cacheline ASAP */
+ CLIB_PREFETCH (sa0->cacheline1, CLIB_CACHE_LINE_BYTES, LOAD);
cpd.icv_sz = sa0->integ_icv_size;
cpd.iv_sz = sa0->crypto_iv_size;
cpd.flags = sa0->flags;
diff --git a/src/vnet/ipsec/esp_encrypt.c b/src/vnet/ipsec/esp_encrypt.c
index c0773e2..591ec26 100644
--- a/src/vnet/ipsec/esp_encrypt.c
+++ b/src/vnet/ipsec/esp_encrypt.c
@@ -624,6 +624,9 @@
CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD);
p -= CLIB_CACHE_LINE_BYTES;
CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD);
+ /* speculate that the trailer goes in the first buffer */
+ CLIB_PREFETCH (vlib_buffer_get_tail (b[1]),
+ CLIB_CACHE_LINE_BYTES, LOAD);
}
if (is_tun)
@@ -646,6 +649,10 @@
current_sa_packets = current_sa_bytes = 0;
sa0 = pool_elt_at_index (im->sad, sa_index0);
+
+ /* fetch the second cacheline ASAP */
+ CLIB_PREFETCH (sa0->cacheline1, CLIB_CACHE_LINE_BYTES, LOAD);
+
current_sa_index = sa_index0;
spi = clib_net_to_host_u32 (sa0->spi);
esp_align = sa0->esp_block_align;
diff --git a/src/vnet/ipsec/ipsec_sa.h b/src/vnet/ipsec/ipsec_sa.h
index 5d65238..02f5eaf 100644
--- a/src/vnet/ipsec/ipsec_sa.h
+++ b/src/vnet/ipsec/ipsec_sa.h
@@ -40,7 +40,7 @@
foreach_ipsec_crypto_alg
#undef _
IPSEC_CRYPTO_N_ALG,
-} ipsec_crypto_alg_t;
+} __clib_packed ipsec_crypto_alg_t;
#define IPSEC_CRYPTO_ALG_IS_GCM(_alg) \
(((_alg == IPSEC_CRYPTO_ALG_AES_GCM_128) || \
@@ -62,13 +62,13 @@
foreach_ipsec_integ_alg
#undef _
IPSEC_INTEG_N_ALG,
-} ipsec_integ_alg_t;
+} __clib_packed ipsec_integ_alg_t;
typedef enum
{
IPSEC_PROTOCOL_AH = 0,
IPSEC_PROTOCOL_ESP = 1
-} ipsec_protocol_t;
+} __clib_packed ipsec_protocol_t;
#define IPSEC_KEY_MAX_LEN 128
typedef struct ipsec_key_t_
@@ -149,9 +149,9 @@
u64 crypto_op_data;
};
- /* data accessed by dataplane code should be above this comment */
CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
+ u64 gcm_iv_counter;
union
{
ip4_header_t ip4_hdr;
@@ -159,31 +159,16 @@
};
udp_header_t udp_hdr;
- fib_node_t node;
- u32 id;
- u32 stat_index;
- ipsec_protocol_t protocol;
-
- ipsec_crypto_alg_t crypto_alg;
- ipsec_key_t crypto_key;
- vnet_crypto_alg_t crypto_calg;
-
- ipsec_integ_alg_t integ_alg;
- ipsec_key_t integ_key;
- vnet_crypto_alg_t integ_calg;
-
- ip46_address_t tunnel_src_addr;
- ip46_address_t tunnel_dst_addr;
-
- fib_node_index_t fib_entry_index;
- u32 sibling;
-
- u32 tx_fib_index;
-
/* Salt used in GCM modes - stored in network byte order */
u32 salt;
- u64 gcm_iv_counter;
+ ipsec_protocol_t protocol;
+ u8 __pad[3];
+
+ /* data accessed by dataplane code should be above this comment */
+ CLIB_CACHE_LINE_ALIGN_MARK (cacheline2);
+
+ /* Elements with u64 size multiples */
union
{
struct
@@ -205,9 +190,32 @@
};
u64 data;
} async_op_data;
+
+ ip46_address_t tunnel_src_addr;
+ ip46_address_t tunnel_dst_addr;
+
+ fib_node_t node;
+
+ /* elements with u32 size */
+ u32 id;
+ u32 stat_index;
+ vnet_crypto_alg_t integ_calg;
+ vnet_crypto_alg_t crypto_calg;
+
+ fib_node_index_t fib_entry_index;
+ u32 sibling;
+ u32 tx_fib_index;
+
+ /* else u8 packed */
+ ipsec_crypto_alg_t crypto_alg;
+ ipsec_integ_alg_t integ_alg;
+
+ ipsec_key_t integ_key;
+ ipsec_key_t crypto_key;
} ipsec_sa_t;
STATIC_ASSERT_OFFSET_OF (ipsec_sa_t, cacheline1, CLIB_CACHE_LINE_BYTES);
+STATIC_ASSERT_OFFSET_OF (ipsec_sa_t, cacheline2, 2 * CLIB_CACHE_LINE_BYTES);
#define _(a,v,s) \
always_inline int \