vlib: move offload flags to 1st cacheline in vlib_buffer_t
Type: improvement
Some tests i.e. ipsec see performance regression when offload flags
are moved to 2nd cacheline. This patch moves them back to 1st cacheline.
Change-Id: I6ead45ff6d2c467b0d248f409e27c2ba31758741
Signed-off-by: Mohsin Kazmi <sykazmi@cisco.com>
diff --git a/src/vnet/buffer.c b/src/vnet/buffer.c
index b41036d..545018a 100644
--- a/src/vnet/buffer.c
+++ b/src/vnet/buffer.c
@@ -22,7 +22,7 @@
vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
#define _(bit, name, ss, v) \
- if (v && (vnet_buffer2 (b)->oflags & VNET_BUFFER_OFFLOAD_F_##name)) \
+ if (v && (vnet_buffer (b)->oflags & VNET_BUFFER_OFFLOAD_F_##name)) \
s = format (s, "%s ", ss);
foreach_vnet_buffer_offload_flag
#undef _
diff --git a/src/vnet/buffer.h b/src/vnet/buffer.h
index 19585e2..0816232 100644
--- a/src/vnet/buffer.h
+++ b/src/vnet/buffer.h
@@ -138,7 +138,8 @@
i16 l3_hdr_offset;
i16 l4_hdr_offset;
u8 feature_arc_index;
- u8 dont_waste_me;
+ /* offload flags */
+ u8 oflags;
union
{
@@ -472,9 +473,6 @@
u16 gso_size;
/* size of L4 prototol header */
u16 gso_l4_hdr_sz;
-
- /* offload flags */
- u32 oflags;
};
struct
@@ -526,12 +524,12 @@
if (b->flags & VNET_BUFFER_F_OFFLOAD)
{
/* add a flag to existing offload */
- vnet_buffer2 (b)->oflags |= oflags;
+ vnet_buffer (b)->oflags |= oflags;
}
else
{
/* no offload yet: reset offload flags to new value */
- vnet_buffer2 (b)->oflags = oflags;
+ vnet_buffer (b)->oflags = oflags;
b->flags |= VNET_BUFFER_F_OFFLOAD;
}
}
@@ -539,8 +537,8 @@
static_always_inline void
vnet_buffer_offload_flags_clear (vlib_buffer_t *b, u32 oflags)
{
- vnet_buffer2 (b)->oflags &= ~oflags;
- if (0 == vnet_buffer2 (b)->oflags)
+ vnet_buffer (b)->oflags &= ~oflags;
+ if (0 == vnet_buffer (b)->oflags)
b->flags &= ~VNET_BUFFER_F_OFFLOAD;
}
diff --git a/src/vnet/devices/virtio/device.c b/src/vnet/devices/virtio/device.c
index 237e3d9..2026478 100644
--- a/src/vnet/devices/virtio/device.c
+++ b/src/vnet/devices/virtio/device.c
@@ -285,7 +285,7 @@
set_checksum_offsets (vlib_buffer_t *b, virtio_net_hdr_v1_t *hdr,
const int is_l2)
{
- u32 oflags = vnet_buffer2 (b)->oflags;
+ u32 oflags = vnet_buffer (b)->oflags;
if (b->flags & VNET_BUFFER_F_IS_IP4)
{
@@ -334,7 +334,7 @@
static void
set_gso_offsets (vlib_buffer_t *b, virtio_net_hdr_v1_t *hdr, const int is_l2)
{
- u32 oflags = vnet_buffer2 (b)->oflags;
+ u32 oflags = vnet_buffer (b)->oflags;
if (b->flags & VNET_BUFFER_F_IS_IP4)
{
diff --git a/src/vnet/devices/virtio/vhost_user_output.c b/src/vnet/devices/virtio/vhost_user_output.c
index 8dbddea..6406fe3 100644
--- a/src/vnet/devices/virtio/vhost_user_output.c
+++ b/src/vnet/devices/virtio/vhost_user_output.c
@@ -225,7 +225,7 @@
generic_header_offset_t gho = { 0 };
int is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4;
int is_ip6 = b->flags & VNET_BUFFER_F_IS_IP6;
- u32 oflags = vnet_buffer2 (b)->oflags;
+ u32 oflags = vnet_buffer (b)->oflags;
ASSERT (!(is_ip4 && is_ip6));
vnet_generic_header_offset_parser (b, &gho, 1 /* l2 */ , is_ip4, is_ip6);
diff --git a/src/vnet/interface_output.h b/src/vnet/interface_output.h
index bfd2066..bb27cdb 100644
--- a/src/vnet/interface_output.h
+++ b/src/vnet/interface_output.h
@@ -85,7 +85,7 @@
ip6_header_t *ip6;
tcp_header_t *th;
udp_header_t *uh;
- u32 oflags = vnet_buffer2 (b)->oflags;
+ u32 oflags;
if (!(b->flags & VNET_BUFFER_F_OFFLOAD))
return;
@@ -96,6 +96,7 @@
ip6 = (ip6_header_t *) (b->data + vnet_buffer (b)->l3_hdr_offset);
th = (tcp_header_t *) (b->data + vnet_buffer (b)->l4_hdr_offset);
uh = (udp_header_t *) (b->data + vnet_buffer (b)->l4_hdr_offset);
+ oflags = vnet_buffer (b)->oflags;
if (is_ip4)
{
diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c
index ddf97d4..da8522d 100644
--- a/src/vnet/ip/ip4_forward.c
+++ b/src/vnet/ip/ip4_forward.c
@@ -1419,7 +1419,7 @@
#define ip4_local_csum_is_offloaded(_b) \
((_b->flags & VNET_BUFFER_F_OFFLOAD) && \
- (vnet_buffer2 (_b)->oflags & \
+ (vnet_buffer (_b)->oflags & \
(VNET_BUFFER_OFFLOAD_F_TCP_CKSUM | VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)))
#define ip4_local_need_csum_check(is_tcp_udp, _b) \
@@ -2067,7 +2067,7 @@
/* Verify checksum. */
ASSERT (ip4_header_checksum_is_valid (ip) ||
- (vnet_buffer2 (b)->oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM));
+ (vnet_buffer (b)->oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM));
}
always_inline uword
diff --git a/src/vnet/ip/ip6_forward.c b/src/vnet/ip/ip6_forward.c
index 0c1bc3b..4f1b148 100644
--- a/src/vnet/ip/ip6_forward.c
+++ b/src/vnet/ip/ip6_forward.c
@@ -1334,8 +1334,8 @@
flags[1] = b[1]->flags;
u32 oflags[2];
- oflags[0] = vnet_buffer2 (b[0])->oflags;
- oflags[1] = vnet_buffer2 (b[1])->oflags;
+ oflags[0] = vnet_buffer (b[0])->oflags;
+ oflags[1] = vnet_buffer (b[1])->oflags;
u32 l4_offload[2];
l4_offload[0] = (flags[0] & VNET_BUFFER_F_OFFLOAD) &&
@@ -1524,7 +1524,7 @@
u32 flags = b[0]->flags;
- u32 oflags = vnet_buffer2 (b[0])->oflags;
+ u32 oflags = vnet_buffer (b[0])->oflags;
u32 l4_offload = (flags & VNET_BUFFER_F_OFFLOAD) &&
(oflags & (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM |
diff --git a/src/vnet/ipsec/ipsec_output.c b/src/vnet/ipsec/ipsec_output.c
index 527aea5..598195f 100644
--- a/src/vnet/ipsec/ipsec_output.c
+++ b/src/vnet/ipsec/ipsec_output.c
@@ -309,7 +309,7 @@
if (PREDICT_FALSE (b0->flags & VNET_BUFFER_F_OFFLOAD))
{
- u32 oflags = vnet_buffer2 (b0)->oflags;
+ u32 oflags = vnet_buffer (b0)->oflags;
/*
* Clearing offload flags before checksum is computed