ip: apply dual loop unrolling in ip4_rewrite
Too many prefetches within loop unrollings induce bottleneck and
performance degradation on some CPUs which have less cache line fill
buffers, e.g, Arm Cortex-A72.
Apply dual loop unrolling and tune prefetches manually to remove
hot-spot with prefetch instructions, to get throughput improvement.
It brings about 7% throughput improvement and saves 28% clocks with
ip4_rewrite nodes on Cortex-A72 CPUs.
Type: feature
Change-Id: I0d35ef19faccbd7a5a4647f50bc369bfcb01a20d
Signed-off-by: Lijian Zhang <Lijian.Zhang@arm.com>
diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c
index ecb3595..3562f50 100644
--- a/src/vnet/ip/ip4_forward.c
+++ b/src/vnet/ip/ip4_forward.c
@@ -2421,6 +2421,7 @@
vlib_get_buffers (vm, from, bufs, n_left_from);
clib_memset_u16 (nexts, IP4_REWRITE_NEXT_DROP, n_left_from);
+#if (CLIB_N_PREFETCHES >= 8)
if (n_left_from >= 6)
{
int i;
@@ -2601,6 +2602,136 @@
b += 2;
n_left_from -= 2;
}
+#elif (CLIB_N_PREFETCHES >= 4)
+ next = nexts;
+ b = bufs;
+ while (n_left_from >= 1)
+ {
+ ip_adjacency_t *adj0;
+ ip4_header_t *ip0;
+ u32 rw_len0, error0, adj_index0;
+ u32 tx_sw_if_index0;
+ u8 *p;
+
+ /* Prefetch next iteration */
+ if (PREDICT_TRUE (n_left_from >= 4))
+ {
+ ip_adjacency_t *adj2;
+ u32 adj_index2;
+
+ vlib_prefetch_buffer_header (b[3], LOAD);
+ vlib_prefetch_buffer_data (b[2], LOAD);
+
+ /* Prefetch adj->rewrite_header */
+ adj_index2 = vnet_buffer (b[2])->ip.adj_index[VLIB_TX];
+ adj2 = adj_get (adj_index2);
+ p = (u8 *) adj2;
+ CLIB_PREFETCH (p + CLIB_CACHE_LINE_BYTES, CLIB_CACHE_LINE_BYTES,
+ LOAD);
+ }
+
+ adj_index0 = vnet_buffer (b[0])->ip.adj_index[VLIB_TX];
+
+ /*
+ * Prefetch the per-adjacency counters
+ */
+ if (do_counters)
+ {
+ vlib_prefetch_combined_counter (&adjacency_counters,
+ thread_index, adj_index0);
+ }
+
+ ip0 = vlib_buffer_get_current (b[0]);
+
+ error0 = IP4_ERROR_NONE;
+
+ ip4_ttl_and_checksum_check (b[0], ip0, next + 0, &error0);
+
+ /* Rewrite packet header and updates lengths. */
+ adj0 = adj_get (adj_index0);
+
+ /* Rewrite header was prefetched. */
+ rw_len0 = adj0[0].rewrite_header.data_bytes;
+ vnet_buffer (b[0])->ip.save_rewrite_length = rw_len0;
+
+ /* Check MTU of outgoing interface. */
+ u16 ip0_len = clib_net_to_host_u16 (ip0->length);
+
+ if (do_gso && (b[0]->flags & VNET_BUFFER_F_GSO))
+ ip0_len = gso_mtu_sz (b[0]);
+
+ ip4_mtu_check (b[0], ip0_len,
+ adj0[0].rewrite_header.max_l3_packet_bytes,
+ ip0->flags_and_fragment_offset &
+ clib_host_to_net_u16 (IP4_HEADER_FLAG_DONT_FRAGMENT),
+ next + 0, &error0);
+
+ if (is_mcast)
+ {
+ error0 = ((adj0[0].rewrite_header.sw_if_index ==
+ vnet_buffer (b[0])->sw_if_index[VLIB_RX]) ?
+ IP4_ERROR_SAME_INTERFACE : error0);
+ }
+
+ /* Don't adjust the buffer for ttl issue; icmp-error node wants
+ * to see the IP header */
+ if (PREDICT_TRUE (error0 == IP4_ERROR_NONE))
+ {
+ u32 next_index = adj0[0].rewrite_header.next_index;
+ vlib_buffer_advance (b[0], -(word) rw_len0);
+ tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
+ vnet_buffer (b[0])->sw_if_index[VLIB_TX] = tx_sw_if_index0;
+
+ if (PREDICT_FALSE
+ (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
+ vnet_feature_arc_start (lm->output_feature_arc_index,
+ tx_sw_if_index0, &next_index, b[0]);
+ next[0] = next_index;
+ }
+ else
+ {
+ b[0]->error = error_node->errors[error0];
+ }
+ if (is_midchain)
+ {
+ calc_checksums (vm, b[0]);
+ }
+ /* Guess we are only writing on simple Ethernet header. */
+ vnet_rewrite_one_header (adj0[0], ip0, sizeof (ethernet_header_t));
+
+ /*
+ * Bump the per-adjacency counters
+ */
+ if (do_counters)
+ {
+ vlib_increment_combined_counter
+ (&adjacency_counters,
+ thread_index,
+ adj_index0, 1, vlib_buffer_length_in_chain (vm, b[0]) + rw_len0);
+ }
+
+ if (is_midchain)
+ {
+ if (adj0->sub_type.midchain.fixup_func)
+ adj0->sub_type.midchain.fixup_func
+ (vm, adj0, b[0], adj0->sub_type.midchain.fixup_data);
+ }
+
+ if (is_mcast)
+ {
+ /*
+ * copy bytes from the IP address into the MAC rewrite
+ */
+ vnet_ip_mcast_fixup_header (IP4_MCAST_ADDR_MASK,
+ adj0->rewrite_header.dst_mcast_offset,
+ &ip0->dst_address.as_u32, (u8 *) ip0);
+ }
+
+ next += 1;
+ b += 1;
+ n_left_from -= 1;
+ }
+#endif
while (n_left_from > 0)
{