fib: midchain adjacency optimisations
Type: improvement
- inline some common encap fixup functions into the midchain
rewrite node so we don't incur the cost of the virtual function call
- change the copy 'guess' from ethernet_header (which will never happen) to an ip4 header
- add adj-midchain-tx to multiarch sources
- don't run adj-midchain-tx as a feature, instead put this node as the
adj's next and at the end of the feature arc.
- cache the feature arc config index (to save the cache miss going to fetch it)
- don't check if features are enabled when taking the arc (since we know they are)
the last two changes will also benefit normal adjacencies taking the arc (i.e. for NAT, ACLs, etc)
for IPSec:
- don't run esp_encrypt as a feature, instead when required insert this
node into the adj's next and into the end of the feature arc. this
implies that encrypt is always 'the last feature' run, which is
symmetric with decrypt always being the first.
- esp_encrpyt for tunnels has adj-midchain-tx as next node
Change-Id: Ida0af56a704302cf2d7797ded5f118a781e8acb7
Signed-off-by: Neale Ranns <nranns@cisco.com>
diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c
index acff66d..ea78d55 100644
--- a/src/vnet/ip/ip4_forward.c
+++ b/src/vnet/ip/ip4_forward.c
@@ -54,6 +54,7 @@
#include <vnet/dpo/load_balance_map.h>
#include <vnet/dpo/classify_dpo.h>
#include <vnet/mfib/mfib_table.h> /* for mFIB table and entry creation */
+#include <vnet/adj/adj_dp.h>
#include <vnet/ip/ip4_forward.h>
#include <vnet/interface_output.h>
@@ -2222,8 +2223,11 @@
if (PREDICT_FALSE
(adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
- vnet_feature_arc_start (lm->output_feature_arc_index,
- tx_sw_if_index0, &next_index, b[0]);
+ vnet_feature_arc_start_w_cfg_index (lm->output_feature_arc_index,
+ tx_sw_if_index0,
+ &next_index, b[0],
+ adj0->ia_cfg_index);
+
next[0] = next_index;
if (is_midchain)
vnet_calc_checksums_inline (vm, b[0], 1 /* is_ip4 */ ,
@@ -2246,8 +2250,10 @@
if (PREDICT_FALSE
(adj1[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
- vnet_feature_arc_start (lm->output_feature_arc_index,
- tx_sw_if_index1, &next_index, b[1]);
+ vnet_feature_arc_start_w_cfg_index (lm->output_feature_arc_index,
+ tx_sw_if_index1,
+ &next_index, b[1],
+ adj1->ia_cfg_index);
next[1] = next_index;
if (is_midchain)
vnet_calc_checksums_inline (vm, b[0], 1 /* is_ip4 */ ,
@@ -2261,9 +2267,14 @@
ip4_ttl_inc (b[1], ip1);
}
- /* Guess we are only writing on simple Ethernet header. */
- vnet_rewrite_two_headers (adj0[0], adj1[0],
- ip0, ip1, sizeof (ethernet_header_t));
+ if (is_midchain)
+ /* Guess we are only writing on ipv4 header. */
+ vnet_rewrite_two_headers (adj0[0], adj1[0],
+ ip0, ip1, sizeof (ip4_header_t));
+ else
+ /* Guess we are only writing on simple Ethernet header. */
+ vnet_rewrite_two_headers (adj0[0], adj1[0],
+ ip0, ip1, sizeof (ethernet_header_t));
if (do_counters)
{
@@ -2284,12 +2295,10 @@
if (is_midchain)
{
- if (error0 == IP4_ERROR_NONE && adj0->sub_type.midchain.fixup_func)
- adj0->sub_type.midchain.fixup_func
- (vm, adj0, b[0], adj0->sub_type.midchain.fixup_data);
- if (error1 == IP4_ERROR_NONE && adj1->sub_type.midchain.fixup_func)
- adj1->sub_type.midchain.fixup_func
- (vm, adj1, b[1], adj1->sub_type.midchain.fixup_data);
+ if (error0 == IP4_ERROR_NONE)
+ adj_midchain_fixup (vm, adj0, b[0]);
+ if (error1 == IP4_ERROR_NONE)
+ adj_midchain_fixup (vm, adj1, b[1]);
}
if (is_mcast)
@@ -2391,17 +2400,25 @@
if (PREDICT_FALSE
(adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
- vnet_feature_arc_start (lm->output_feature_arc_index,
- tx_sw_if_index0, &next_index, b[0]);
+ vnet_feature_arc_start_w_cfg_index (lm->output_feature_arc_index,
+ tx_sw_if_index0,
+ &next_index, b[0],
+ adj0->ia_cfg_index);
next[0] = next_index;
if (is_midchain)
- vnet_calc_checksums_inline (vm, b[0], 1 /* is_ip4 */ ,
- 0 /* is_ip6 */ ,
- 0 /* with gso */ );
+ {
+ vnet_calc_checksums_inline (vm, b[0], 1 /* is_ip4 */ ,
+ 0 /* is_ip6 */ ,
+ 0 /* with gso */ );
- /* Guess we are only writing on simple Ethernet header. */
- vnet_rewrite_one_header (adj0[0], ip0, sizeof (ethernet_header_t));
+ /* Guess we are only writing on ipv4 header. */
+ vnet_rewrite_one_header (adj0[0], ip0, sizeof (ip4_header_t));
+ }
+ else
+ /* Guess we are only writing on simple Ethernet header. */
+ vnet_rewrite_one_header (adj0[0], ip0,
+ sizeof (ethernet_header_t));
/*
* Bump the per-adjacency counters
@@ -2413,9 +2430,8 @@
adj_index0, 1, vlib_buffer_length_in_chain (vm,
b[0]) + rw_len0);
- if (is_midchain && adj0->sub_type.midchain.fixup_func)
- adj0->sub_type.midchain.fixup_func
- (vm, adj0, b[0], adj0->sub_type.midchain.fixup_data);
+ if (is_midchain)
+ adj_midchain_fixup (vm, adj0, b[0]);
if (is_mcast)
/* copy bytes from the IP address into the MAC rewrite */
@@ -2491,18 +2507,26 @@
if (PREDICT_FALSE
(adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
- vnet_feature_arc_start (lm->output_feature_arc_index,
- tx_sw_if_index0, &next_index, b[0]);
+ vnet_feature_arc_start_w_cfg_index (lm->output_feature_arc_index,
+ tx_sw_if_index0,
+ &next_index, b[0],
+ adj0->ia_cfg_index);
next[0] = next_index;
if (is_midchain)
- /* this acts on the packet that is about to be encapped */
- vnet_calc_checksums_inline (vm, b[0], 1 /* is_ip4 */ ,
- 0 /* is_ip6 */ ,
- 0 /* with gso */ );
+ {
+ /* this acts on the packet that is about to be encapped */
+ vnet_calc_checksums_inline (vm, b[0], 1 /* is_ip4 */ ,
+ 0 /* is_ip6 */ ,
+ 0 /* with gso */ );
- /* Guess we are only writing on simple Ethernet header. */
- vnet_rewrite_one_header (adj0[0], ip0, sizeof (ethernet_header_t));
+ /* Guess we are only writing on ipv4 header. */
+ vnet_rewrite_one_header (adj0[0], ip0, sizeof (ip4_header_t));
+ }
+ else
+ /* Guess we are only writing on simple Ethernet header. */
+ vnet_rewrite_one_header (adj0[0], ip0,
+ sizeof (ethernet_header_t));
if (do_counters)
vlib_increment_combined_counter