[qca-nss-sfe] add fast xmit features
call hard start xmit in some case that no vlan tag exist,
won't go to xfrm nor qdisc exist.
Change-Id: I94104b0ce8d0e86ee6e63dc401ec042351ef63c7
Signed-off-by: Ken Zhu <quic_guigenz@quicinc.com>
diff --git a/exports/sfe_api.h b/exports/sfe_api.h
index 5a01604..da9c857 100644
--- a/exports/sfe_api.h
+++ b/exports/sfe_api.h
@@ -45,6 +45,8 @@
#define SFE_RULE_CREATE_FLAG_USE_FLOW_BOTTOM_INTERFACE (1<<8) /**< Use flow interface number instead of top interface. */
#define SFE_RULE_CREATE_FLAG_USE_RETURN_BOTTOM_INTERFACE (1<<9) /**< Use return interface number instead of top interface. */
#define SFE_RULE_CREATE_FLAG_SRC_INTERFACE_CHECK (1<<10) /**< Check source interface. */
+#define SFE_RULE_CREATE_FLAG_FLOW_TRANSMIT_FAST (1<<11) /**< original flow transmit fast. */
+#define SFE_RULE_CREATE_FLAG_RETURN_TRANSMIT_FAST (1<<12) /**< return flow transmit fast. */
/**
* Rule creation validity flags.
diff --git a/sfe.c b/sfe.c
index eca2321..9e24f89 100644
--- a/sfe.c
+++ b/sfe.c
@@ -380,6 +380,46 @@
}
/*
+ * sfe_fast_xmit_check()
+ * Check the fast transmit feasibility.
+ *
+ * This check the per direction's attribute that could not go fast
+ * transmit
+ * xfrm packets, come from a local socket or need sk validation on the skb
+ */
+bool sfe_fast_xmit_check(struct sk_buff *skb, netdev_features_t features)
+{
+
+#ifdef CONFIG_SOCK_VALIDATE_XMIT
+ if (skb->sk && sk_fullsock(skb->sk) && skb->sk->sk_validate_xmit_skb) {
+ DEBUG_INFO("%px:need sk validation\n", skb);
+ return false;
+#ifdef CONFIG_TLS_DEVICE
+ } else if (skb->decrypted) {
+ DEBUG_INFO("%px:SK or decrypted\n", skb);
+ return false;
+#endif
+ }
+#endif
+ if (skb_vlan_tag_present(skb)) {
+ DEBUG_INFO("%px:Vlan is present\n", skb);
+ return false;
+ }
+
+ if (netif_needs_gso(skb, features)) {
+ DEBUG_INFO("%px:Need to be gso\n", skb);
+ return false;
+ }
+
+ if (skb_sec_path(skb)) {
+ DEBUG_INFO("%px:XFRM is present\n", skb);
+ return false;
+ }
+
+ return true;
+}
+
+/*
* sfe_enqueue_msg()
* Queue response message
*
diff --git a/sfe.h b/sfe.h
index 15658e1..c117729 100644
--- a/sfe.h
+++ b/sfe.h
@@ -134,6 +134,11 @@
extern int nf_ct_tcp_no_window_check;
/*
+ * Check the fast transmit feasibility.
+ */
+bool sfe_fast_xmit_check(struct sk_buff *skb, netdev_features_t features);
+
+/*
* This callback will be called in a timer
* at 100 times per second to sync stats back to
* Linux connection track.
diff --git a/sfe_ipv4.c b/sfe_ipv4.c
index bf76295..6315e62 100644
--- a/sfe_ipv4.c
+++ b/sfe_ipv4.c
@@ -308,6 +308,7 @@
stats->connection_flushes64 += s->connection_flushes64;
stats->packets_dropped64 += s->packets_dropped64;
stats->packets_forwarded64 += s->packets_forwarded64;
+ stats->packets_fast_xmited64 += s->packets_fast_xmited64;
stats->packets_not_forwarded64 += s->packets_not_forwarded64;
stats->pppoe_encap_packets_forwarded64 += s->pppoe_encap_packets_forwarded64;
stats->pppoe_decap_packets_forwarded64 += s->pppoe_decap_packets_forwarded64;
@@ -1218,10 +1219,12 @@
original_cm->dscp = msg->dscp_rule.flow_dscp << SFE_IPV4_DSCP_SHIFT;
original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_DSCP_REMARK;
}
-
if (msg->rule_flags & SFE_RULE_CREATE_FLAG_BRIDGE_FLOW) {
original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_BRIDGE_FLOW;
}
+ if (msg->rule_flags & SFE_RULE_CREATE_FLAG_FLOW_TRANSMIT_FAST) {
+ original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION;
+ }
/*
* Add VLAN rule to original_cm
@@ -1415,10 +1418,12 @@
reply_cm->dscp = msg->dscp_rule.return_dscp << SFE_IPV4_DSCP_SHIFT;
reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_DSCP_REMARK;
}
-
if (msg->rule_flags & SFE_RULE_CREATE_FLAG_BRIDGE_FLOW) {
reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_BRIDGE_FLOW;
}
+ if (msg->rule_flags & SFE_RULE_CREATE_FLAG_RETURN_TRANSMIT_FAST) {
+ reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION;
+ }
if ((IPPROTO_GRE == tuple->protocol) && !sfe_ipv4_is_local_ip(si, reply_cm->match_dest_ip)) {
reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_PASSTHROUGH;
@@ -1984,6 +1989,7 @@
u32 packet, byte, original_cm_flags;
u16 pppoe_session_id;
u8 pppoe_remote_mac[ETH_ALEN];
+ u32 original_fast_xmit, reply_fast_xmit;
#ifdef CONFIG_NF_FLOW_COOKIE
int src_flow_cookie, dst_flow_cookie;
#endif
@@ -2024,6 +2030,7 @@
src_rx_packets = original_cm->rx_packet_count64;
src_rx_bytes = original_cm->rx_byte_count64;
src_mark = original_cm->mark;
+ original_fast_xmit = (original_cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT);
dest_dev = c->reply_dev;
dest_ip = c->dest_ip;
dest_ip_xlate = c->dest_ip_xlate;
@@ -2034,6 +2041,7 @@
dest_rx_packets = reply_cm->rx_packet_count64;
dest_rx_bytes = reply_cm->rx_byte_count64;
dest_mark = reply_cm->mark;
+ reply_fast_xmit = (reply_cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT);
last_sync_jiffies = get_jiffies_64() - c->last_sync_jiffies;
original_cm_flags = original_cm->flags;
pppoe_session_id = original_cm->pppoe_session_id;
@@ -2053,12 +2061,14 @@
"src_priority=\"%u\" src_dscp=\"%u\" "
"src_rx_pkts=\"%llu\" src_rx_bytes=\"%llu\" "
"src_mark=\"%08x\" "
+ "src_fast_xmit=\"%s\" "
"dest_dev=\"%s\" "
"dest_ip=\"%pI4\" dest_ip_xlate=\"%pI4\" "
"dest_port=\"%u\" dest_port_xlate=\"%u\" "
"dest_priority=\"%u\" dest_dscp=\"%u\" "
"dest_rx_pkts=\"%llu\" dest_rx_bytes=\"%llu\" "
"dest_mark=\"%08x\" "
+ "reply_fast_xmit=\"%s\" "
#ifdef CONFIG_NF_FLOW_COOKIE
"src_flow_cookie=\"%d\" dst_flow_cookie=\"%d\" "
#endif
@@ -2070,12 +2080,14 @@
src_priority, src_dscp,
src_rx_packets, src_rx_bytes,
src_mark,
+ original_fast_xmit ? "Yes" : "No",
dest_dev->name,
&dest_ip, &dest_ip_xlate,
ntohs(dest_port), ntohs(dest_port_xlate),
dest_priority, dest_dscp,
dest_rx_packets, dest_rx_bytes,
dest_mark,
+ reply_fast_xmit ? "Yes" : "No",
#ifdef CONFIG_NF_FLOW_COOKIE
src_flow_cookie, dst_flow_cookie,
#endif
@@ -2220,6 +2232,7 @@
bytes_read = snprintf(msg, CHAR_DEV_MSG_SIZE, "\t<stats "
"num_connections=\"%u\" "
"pkts_dropped=\"%llu\" "
+ "pkts_fast_xmited=\"%llu\" "
"pkts_forwarded=\"%llu\" pkts_not_forwarded=\"%llu\" "
"create_requests=\"%llu\" create_collisions=\"%llu\" "
"create_failures=\"%llu\" "
@@ -2231,6 +2244,7 @@
"pppoe_bridge_pkts_fwded=\"%llu\" />\n",
num_conn,
stats.packets_dropped64,
+ stats.packets_fast_xmited64,
stats.packets_forwarded64,
stats.packets_not_forwarded64,
stats.connection_create_requests64,
@@ -2458,13 +2472,14 @@
}
return size;
}
+
/*
* sysfs attributes.
*/
static const struct device_attribute sfe_ipv4_cpu_attr =
__ATTR(stats_work_cpu, S_IWUSR | S_IRUGO, sfe_ipv4_get_cpu, sfe_ipv4_set_cpu);
- /*
+/*
* sfe_ipv4_conn_match_hash_init()
* Initialize conn match hash lists
*/
@@ -2656,6 +2671,7 @@
sysfs_remove_file(si->sys_ipv4, &sfe_ipv4_flow_cookie_attr.attr);
#endif /* CONFIG_NF_FLOW_COOKIE */
sysfs_remove_file(si->sys_ipv4, &sfe_ipv4_debug_dev_attr.attr);
+
sysfs_remove_file(si->sys_ipv4, &sfe_ipv4_cpu_attr.attr);
kobject_put(si->sys_ipv4);
diff --git a/sfe_ipv4.h b/sfe_ipv4.h
index 016ee8f..89b5ace 100644
--- a/sfe_ipv4.h
+++ b/sfe_ipv4.h
@@ -72,6 +72,12 @@
/* Source interface check.*/
#define SFE_IPV4_CONNECTION_MATCH_FLAG_PASSTHROUGH (1<<14)
/* passthrough flow: encap/decap to be skipped for this flow */
+#define SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT (1<<15)
+ /* skb go fast xmit */
+#define SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED (1<<16)
+ /* Fast xmit flow checked or not */
+#define SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION (1<<17)
+ /* Fast xmit may be possible for this flow, if SFE check passes */
/*
* IPv4 connection matching structure.
@@ -183,6 +189,11 @@
* Size of all needed L2 headers
*/
u16 l2_hdr_size;
+
+ /*
+ * xmit device's feature
+ */
+ netdev_features_t features;
};
/*
@@ -304,6 +315,7 @@
u64 connection_flushes64; /* Number of IPv4 connection flushes */
u64 packets_dropped64; /* Number of IPv4 packets dropped */
u64 packets_forwarded64; /* Number of IPv4 packets forwarded */
+ u64 packets_fast_xmited64; /* Number of IPv4 packets fast transmited */
u64 packets_not_forwarded64; /* Number of IPv4 packets not forwarded */
u64 exception_events64[SFE_IPV4_EXCEPTION_EVENT_LAST];
u64 pppoe_encap_packets_forwarded64; /* Number of IPv4 PPPoE encap packets forwarded */
diff --git a/sfe_ipv4_gre.c b/sfe_ipv4_gre.c
index c8d424b..084ea3b 100644
--- a/sfe_ipv4_gre.c
+++ b/sfe_ipv4_gre.c
@@ -298,7 +298,7 @@
* Mark outgoing packet.
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_MARK)) {
- skb->mark = cm->connection->mark;
+ skb->mark = cm->mark;
}
this_cpu_inc(si->stats_pcpu->packets_forwarded64);
diff --git a/sfe_ipv4_tcp.c b/sfe_ipv4_tcp.c
index 569dba9..d33778f 100644
--- a/sfe_ipv4_tcp.c
+++ b/sfe_ipv4_tcp.c
@@ -131,6 +131,8 @@
bool ret;
bool hw_csum;
bool bridge_flow;
+ bool fast_xmit;
+ netdev_features_t features;
/*
* Is our packet too short to contain a valid TCP header?
@@ -690,6 +692,20 @@
skb->mark = cm->connection->mark;
}
+ /*
+ * For the first packets, check if it could got fast xmit.
+ */
+ if (unlikely(!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED)
+ && (cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION))){
+ cm->features = netif_skb_features(skb);
+ if (likely(sfe_fast_xmit_check(skb, cm->features))) {
+ cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT;
+ }
+ cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED;
+ }
+ features = cm->features;
+ fast_xmit = !!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT);
+
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->packets_forwarded64);
@@ -701,6 +717,15 @@
prefetch(skb_shinfo(skb));
/*
+ * We do per packet condition check before we could fast xmit the
+ * packet.
+ */
+ if (likely(fast_xmit && dev_fast_xmit(skb, xmit_dev, features))) {
+ this_cpu_inc(si->stats_pcpu->packets_fast_xmited64);
+ return 1;
+ }
+
+ /*
* Mark that this packet has been fast forwarded.
*/
skb->fast_forwarded = 1;
diff --git a/sfe_ipv4_udp.c b/sfe_ipv4_udp.c
index 313d6ff..27e371d 100644
--- a/sfe_ipv4_udp.c
+++ b/sfe_ipv4_udp.c
@@ -132,6 +132,8 @@
int err;
bool bridge_flow;
int ret;
+ bool fast_xmit;
+ netdev_features_t features;
/*
* Is our packet too short to contain a valid UDP header?
@@ -533,6 +535,21 @@
skb->mark = cm->connection->mark;
}
+ /*
+ * For the first packets, check if it could got fast xmit.
+ */
+ if (unlikely(!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED)
+ && (cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION))){
+ cm->features = netif_skb_features(skb);
+ if (likely(sfe_fast_xmit_check(skb, cm->features))) {
+ cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT;
+ }
+ cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED;
+ }
+ features = cm->features;
+
+ fast_xmit = !!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT);
+
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->packets_forwarded64);
@@ -544,6 +561,15 @@
prefetch(skb_shinfo(skb));
/*
+ * We do per packet condition check before we could fast xmit the
+ * packet.
+ */
+ if (likely(fast_xmit && dev_fast_xmit(skb, xmit_dev, features))) {
+ this_cpu_inc(si->stats_pcpu->packets_fast_xmited64);
+ return 1;
+ }
+
+ /*
* Mark that this packet has been fast forwarded.
*/
skb->fast_forwarded = 1;
diff --git a/sfe_ipv6.c b/sfe_ipv6.c
index 4143b14..e5b6658 100644
--- a/sfe_ipv6.c
+++ b/sfe_ipv6.c
@@ -317,6 +317,7 @@
stats->connection_flushes64 += s->connection_flushes64;
stats->packets_dropped64 += s->packets_dropped64;
stats->packets_forwarded64 += s->packets_forwarded64;
+ stats->packets_fast_xmited64 += s->packets_fast_xmited64;
stats->packets_not_forwarded64 += s->packets_not_forwarded64;
stats->pppoe_encap_packets_forwarded64 += s->pppoe_encap_packets_forwarded64;
stats->pppoe_decap_packets_forwarded64 += s->pppoe_decap_packets_forwarded64;
@@ -1212,6 +1213,10 @@
if (msg->rule_flags & SFE_RULE_CREATE_FLAG_BRIDGE_FLOW) {
original_cm->flags |= SFE_IPV6_CONNECTION_MATCH_FLAG_BRIDGE_FLOW;
}
+ if (msg->rule_flags & SFE_RULE_CREATE_FLAG_FLOW_TRANSMIT_FAST) {
+ original_cm->flags |= SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION;
+ }
+
/*
* Add VLAN rule to original_cm
@@ -1385,10 +1390,12 @@
reply_cm->dscp = msg->dscp_rule.return_dscp << SFE_IPV6_DSCP_SHIFT;
reply_cm->flags |= SFE_IPV6_CONNECTION_MATCH_FLAG_DSCP_REMARK;
}
-
if (msg->rule_flags & SFE_RULE_CREATE_FLAG_BRIDGE_FLOW) {
reply_cm->flags |= SFE_IPV6_CONNECTION_MATCH_FLAG_BRIDGE_FLOW;
}
+ if (msg->rule_flags & SFE_RULE_CREATE_FLAG_RETURN_TRANSMIT_FAST) {
+ reply_cm->flags |= SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION;
+ }
if ((IPPROTO_GRE == tuple->protocol) && !sfe_ipv6_is_local_ip(si, (uint8_t *)reply_cm->match_dest_ip)) {
reply_cm->flags |= SFE_IPV6_CONNECTION_MATCH_FLAG_PASSTHROUGH;
@@ -1932,6 +1939,7 @@
u32 packet, byte, original_cm_flags;
u16 pppoe_session_id;
u8 pppoe_remote_mac[ETH_ALEN];
+ u32 original_fast_xmit, reply_fast_xmit;
#ifdef CONFIG_NF_FLOW_COOKIE
int src_flow_cookie, dst_flow_cookie;
#endif
@@ -1972,6 +1980,7 @@
src_rx_packets = original_cm->rx_packet_count64;
src_rx_bytes = original_cm->rx_byte_count64;
src_mark = original_cm->mark;
+ original_fast_xmit = original_cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT;
dest_dev = c->reply_dev;
dest_ip = c->dest_ip[0];
dest_ip_xlate = c->dest_ip_xlate[0];
@@ -1986,6 +1995,7 @@
pppoe_session_id = original_cm->pppoe_session_id;
ether_addr_copy(pppoe_remote_mac, original_cm->pppoe_remote_mac);
dest_mark = reply_cm->mark;
+ reply_fast_xmit = reply_cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT;
#ifdef CONFIG_NF_FLOW_COOKIE
src_flow_cookie = original_cm->flow_cookie;
dst_flow_cookie = reply_cm->flow_cookie;
@@ -2000,12 +2010,14 @@
"src_priority=\"%u\" src_dscp=\"%u\" "
"src_rx_pkts=\"%llu\" src_rx_bytes=\"%llu\" "
"src_mark=\"%08x\" "
+ "src_fast_xmit=\"%s\" "
"dest_dev=\"%s\" "
"dest_ip=\"%pI6\" dest_ip_xlate=\"%pI6\" "
"dest_port=\"%u\" dest_port_xlate=\"%u\" "
"dest_priority=\"%u\" dest_dscp=\"%u\" "
"dest_rx_pkts=\"%llu\" dest_rx_bytes=\"%llu\" "
"dest_mark=\"%08x\" "
+ "reply_fast_xmit=\"%s\" "
#ifdef CONFIG_NF_FLOW_COOKIE
"src_flow_cookie=\"%d\" dst_flow_cookie=\"%d\" "
#endif
@@ -2017,12 +2029,14 @@
src_priority, src_dscp,
src_rx_packets, src_rx_bytes,
src_mark,
+ original_fast_xmit ? "Yes" : "No",
dest_dev->name,
&dest_ip, &dest_ip_xlate,
ntohs(dest_port), ntohs(dest_port_xlate),
dest_priority, dest_dscp,
dest_rx_packets, dest_rx_bytes,
dest_mark,
+ reply_fast_xmit ? "Yes" : "No",
#ifdef CONFIG_NF_FLOW_COOKIE
src_flow_cookie, dst_flow_cookie,
#endif
@@ -2168,6 +2182,7 @@
bytes_read = snprintf(msg, CHAR_DEV_MSG_SIZE, "\t<stats "
"num_connections=\"%u\" "
"pkts_dropped=\"%llu\" "
+ "pkts_fast_xmited=\"%llu\" "
"pkts_forwarded=\"%llu\" pkts_not_forwarded=\"%llu\" "
"create_requests=\"%llu\" create_collisions=\"%llu\" "
"create_failures=\"%llu\" "
@@ -2180,6 +2195,7 @@
num_conn,
stats.packets_dropped64,
+ stats.packets_fast_xmited64,
stats.packets_forwarded64,
stats.packets_not_forwarded64,
stats.connection_create_requests64,
diff --git a/sfe_ipv6.h b/sfe_ipv6.h
index 408ebb5..7359f9b 100644
--- a/sfe_ipv6.h
+++ b/sfe_ipv6.h
@@ -85,6 +85,12 @@
/* Source interface check.*/
#define SFE_IPV6_CONNECTION_MATCH_FLAG_PASSTHROUGH (1<<14)
/* passthrough flow: encap/decap to be skipped for this flow */
+#define SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT (1<<15)
+ /* go fast xmit*/
+#define SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED (1<<16)
+ /* fast xmit checked or not*/
+#define SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION (1<<17)
+ /* Fast xmit may be possible for this flow, if SFE check passes */
/*
* IPv6 connection matching structure.
@@ -194,6 +200,11 @@
* Size of all needed L2 headers
*/
u16 l2_hdr_size;
+
+ /*
+ * xmit device's feature
+ */
+ netdev_features_t features;
};
/*
@@ -322,6 +333,7 @@
u64 connection_flushes64; /* Number of IPv6 connection flushes */
u64 packets_dropped64; /* Number of IPv4 packets dropped */
u64 packets_forwarded64; /* Number of IPv6 packets forwarded */
+ u64 packets_fast_xmited64; /* Number of IPv6 packets fast transmited */
u64 packets_not_forwarded64; /* Number of IPv6 packets not forwarded */
u64 exception_events64[SFE_IPV6_EXCEPTION_EVENT_LAST];
u64 pppoe_encap_packets_forwarded64; /* Number of IPv6 PPPoE encap packets forwarded */
diff --git a/sfe_ipv6_tcp.c b/sfe_ipv6_tcp.c
index b78303c..28cc541 100644
--- a/sfe_ipv6_tcp.c
+++ b/sfe_ipv6_tcp.c
@@ -130,6 +130,8 @@
bool ret;
bool hw_csum;
bool bridge_flow;
+ bool fast_xmit;
+ netdev_features_t features;
/*
* Is our packet too short to contain a valid TCP header?
@@ -695,6 +697,21 @@
skb->mark = cm->mark;
}
+ /*
+ * For the first packets, check if it could got fast xmit.
+ */
+ if (unlikely(!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED)
+ && (cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION))){
+ cm->features = netif_skb_features(skb);
+ if (likely(sfe_fast_xmit_check(skb, cm->features))) {
+ cm->flags |= SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT;
+ }
+ cm->flags |= SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED;
+ }
+ features = cm->features;
+
+ fast_xmit = !!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT);
+
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->packets_forwarded64);
@@ -706,6 +723,15 @@
prefetch(skb_shinfo(skb));
/*
+ * We do per packet condition check before we could fast xmit the
+ * packet.
+ */
+ if (likely(fast_xmit && dev_fast_xmit(skb, xmit_dev, features))) {
+ this_cpu_inc(si->stats_pcpu->packets_fast_xmited64);
+ return 1;
+ }
+
+ /*
* Mark that this packet has been fast forwarded.
*/
skb->fast_forwarded = 1;
diff --git a/sfe_ipv6_udp.c b/sfe_ipv6_udp.c
index 9e32513..5ffefdd 100644
--- a/sfe_ipv6_udp.c
+++ b/sfe_ipv6_udp.c
@@ -137,6 +137,8 @@
int ret;
bool hw_csum;
bool bridge_flow;
+ bool fast_xmit;
+ netdev_features_t features;
DEBUG_TRACE("%px: sfe: sfe_ipv6_recv_udp called.\n", skb);
@@ -527,6 +529,21 @@
skb->mark = cm->mark;
}
+ /*
+ * For the first packets, check if it could got fast xmit.
+ */
+ if (unlikely(!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED)
+ && (cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION))){
+ cm->features = netif_skb_features(skb);
+ if (likely(sfe_fast_xmit_check(skb, cm->features))) {
+ cm->flags |= SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT;
+ }
+ cm->flags |= SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED;
+ }
+ features = cm->features;
+
+ fast_xmit = !!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT);
+
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->packets_forwarded64);
@@ -538,6 +555,15 @@
prefetch(skb_shinfo(skb));
/*
+ * We do per packet condition check before we could fast xmit the
+ * packet.
+ */
+ if (likely(fast_xmit && dev_fast_xmit(skb, xmit_dev, features))) {
+ this_cpu_inc(si->stats_pcpu->packets_fast_xmited64);
+ return 1;
+ }
+
+ /*
* Mark that this packet has been fast forwarded.
*/
skb->fast_forwarded = 1;