vlib: refactor checksum offload support
Type: refactor
This patch refactors the offload flags in vlib_buffer_t.
There are two main reasons behind this refactoring.
First, offload flags are insufficient to represent outer
and inner headers offloads. Second, room for these flags
in first cacheline of vlib_buffer_t is also limited.
This patch introduces a generic offload flag in first
cacheline. And detailed offload flags in 2nd cacheline
of the structure for performance optimization.
Change-Id: Icc363a142fb9208ec7113ab5bbfc8230181f6004
Signed-off-by: Mohsin Kazmi <sykazmi@cisco.com>
diff --git a/src/vnet/devices/virtio/device.c b/src/vnet/devices/virtio/device.c
index 3482336..fb996d5 100644
--- a/src/vnet/devices/virtio/device.c
+++ b/src/vnet/devices/virtio/device.c
@@ -282,6 +282,8 @@
set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr,
const int is_l2)
{
+ u32 oflags = vnet_buffer2 (b)->oflags;
+
if (b->flags & VNET_BUFFER_F_IS_IP4)
{
ip4_header_t *ip4;
@@ -290,11 +292,11 @@
0 /* ip6 */ );
hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
hdr->csum_start = gho.l4_hdr_offset; // 0x22;
- if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
{
hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
}
- else if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
+ else if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
{
hdr->csum_offset = STRUCT_OFFSET_OF (udp_header_t, checksum);
}
@@ -305,7 +307,7 @@
*/
ip4 =
(ip4_header_t *) (vlib_buffer_get_current (b) + gho.l3_hdr_offset);
- if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
ip4->checksum = ip4_header_checksum (ip4);
}
else if (b->flags & VNET_BUFFER_F_IS_IP6)
@@ -315,11 +317,11 @@
1 /* ip6 */ );
hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
hdr->csum_start = gho.l4_hdr_offset; // 0x36;
- if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
{
hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
}
- else if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
+ else if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
{
hdr->csum_offset = STRUCT_OFFSET_OF (udp_header_t, checksum);
}
@@ -330,6 +332,8 @@
set_gso_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr,
const int is_l2)
{
+ u32 oflags = vnet_buffer2 (b)->oflags;
+
if (b->flags & VNET_BUFFER_F_IS_IP4)
{
ip4_header_t *ip4;
@@ -348,7 +352,7 @@
* virtio devices do not support IP4 checksum offload. So driver takes care
* of it while doing tx.
*/
- if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
ip4->checksum = ip4_header_checksum (ip4);
}
else if (b->flags & VNET_BUFFER_F_IS_IP6)
@@ -392,8 +396,7 @@
goto done;
}
}
- else if (b->flags & (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
+ else if (b->flags & VNET_BUFFER_F_OFFLOAD)
{
if (csum_offload)
set_checksum_offsets (b, hdr, is_l2);
@@ -584,8 +587,7 @@
goto done;
}
}
- else if (b->flags & (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
+ else if (b->flags & VNET_BUFFER_F_OFFLOAD)
{
if (csum_offload)
set_checksum_offsets (b, hdr, is_l2);
diff --git a/src/vnet/devices/virtio/node.c b/src/vnet/devices/virtio/node.c
index be2405b..98df322 100644
--- a/src/vnet/devices/virtio/node.c
+++ b/src/vnet/devices/virtio/node.c
@@ -235,6 +235,7 @@
if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
{
u16 ethertype = 0, l2hdr_sz = 0;
+ u32 oflags = 0;
if (type == VIRTIO_IF_TYPE_TUN)
{
@@ -280,11 +281,10 @@
(ip4_header_t *) (vlib_buffer_get_current (b0) + l2hdr_sz);
vnet_buffer (b0)->l4_hdr_offset = l2hdr_sz + ip4_header_bytes (ip4);
*l4_proto = ip4->protocol;
+ oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM;
b0->flags |=
- (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM);
- b0->flags |=
- (VNET_BUFFER_F_L2_HDR_OFFSET_VALID
- | VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
+ (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
+ VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
VNET_BUFFER_F_L4_HDR_OFFSET_VALID);
}
else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6))
@@ -301,7 +301,7 @@
}
if (*l4_proto == IP_PROTOCOL_TCP)
{
- b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
tcp_header_t *tcp = (tcp_header_t *) (vlib_buffer_get_current (b0) +
vnet_buffer
(b0)->l4_hdr_offset);
@@ -309,12 +309,14 @@
}
else if (*l4_proto == IP_PROTOCOL_UDP)
{
- b0->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
udp_header_t *udp = (udp_header_t *) (vlib_buffer_get_current (b0) +
vnet_buffer
(b0)->l4_hdr_offset);
*l4_hdr_sz = sizeof (*udp);
}
+ if (oflags)
+ vnet_buffer_offload_flags_set (b0, oflags);
}
}
diff --git a/src/vnet/devices/virtio/vhost_user_input.c b/src/vnet/devices/virtio/vhost_user_input.c
index 62b59f6..739125bb 100644
--- a/src/vnet/devices/virtio/vhost_user_input.c
+++ b/src/vnet/devices/virtio/vhost_user_input.c
@@ -253,6 +253,7 @@
ethernet_header_t *eh = (ethernet_header_t *) b0_data;
u16 ethertype = clib_net_to_host_u16 (eh->type);
u16 l2hdr_sz = sizeof (ethernet_header_t);
+ u32 oflags = 0;
if (ethernet_frame_is_tagged (ethertype))
{
@@ -278,7 +279,8 @@
{
ip4_header_t *ip4 = (ip4_header_t *) (b0_data + l2hdr_sz);
l4_proto = ip4->protocol;
- b0->flags |= VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
+ b0->flags |= VNET_BUFFER_F_IS_IP4;
+ oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM;
}
else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6))
{
@@ -292,12 +294,12 @@
tcp_header_t *tcp = (tcp_header_t *)
(b0_data + vnet_buffer (b0)->l4_hdr_offset);
l4_hdr_sz = tcp_header_bytes (tcp);
- b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
}
else if (l4_proto == IP_PROTOCOL_UDP)
{
l4_hdr_sz = sizeof (udp_header_t);
- b0->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
}
if (hdr->gso_type == VIRTIO_NET_HDR_GSO_UDP)
@@ -318,6 +320,9 @@
vnet_buffer2 (b0)->gso_l4_hdr_sz = l4_hdr_sz;
b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP6);
}
+
+ if (oflags)
+ vnet_buffer_offload_flags_set (b0, oflags);
}
static_always_inline void
diff --git a/src/vnet/devices/virtio/vhost_user_output.c b/src/vnet/devices/virtio/vhost_user_output.c
index 465c0ea..8dbddea 100644
--- a/src/vnet/devices/virtio/vhost_user_output.c
+++ b/src/vnet/devices/virtio/vhost_user_output.c
@@ -225,10 +225,11 @@
generic_header_offset_t gho = { 0 };
int is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4;
int is_ip6 = b->flags & VNET_BUFFER_F_IS_IP6;
+ u32 oflags = vnet_buffer2 (b)->oflags;
ASSERT (!(is_ip4 && is_ip6));
vnet_generic_header_offset_parser (b, &gho, 1 /* l2 */ , is_ip4, is_ip6);
- if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
{
ip4_header_t *ip4;
@@ -238,13 +239,13 @@
}
/* checksum offload */
- if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
{
hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
hdr->csum_start = gho.l4_hdr_offset;
hdr->csum_offset = offsetof (udp_header_t, checksum);
}
- else if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+ else if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
{
hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
hdr->csum_start = gho.l4_hdr_offset;
@@ -254,7 +255,7 @@
/* GSO offload */
if (b->flags & VNET_BUFFER_F_GSO)
{
- if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
{
if (is_ip4 &&
(vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO4)))
@@ -270,7 +271,7 @@
}
}
else if ((vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_UFO)) &&
- (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
+ (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM))
{
hdr->gso_size = vnet_buffer2 (b)->gso_size;
hdr->gso_type = VIRTIO_NET_HDR_GSO_UDP;
@@ -477,9 +478,7 @@
hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
hdr->num_buffers = 1;
- or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) ||
- (b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) ||
- (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM);
+ or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD);
/* Guest supports csum offload and buffer requires checksum offload? */
if (or_flags &&
@@ -813,9 +812,7 @@
hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
hdr->num_buffers = 1; //This is local, no need to check
- or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) ||
- (b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) ||
- (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM);
+ or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD);
/* Guest supports csum offload and buffer requires checksum offload? */
if (or_flags