[qca-nss-sfe] add fast xmit features

call hard start xmit in some case that no vlan tag exist,
won't go to xfrm nor qdisc exist.

Change-Id: I94104b0ce8d0e86ee6e63dc401ec042351ef63c7
Signed-off-by: Ken Zhu <quic_guigenz@quicinc.com>
diff --git a/exports/sfe_api.h b/exports/sfe_api.h
index 5a01604..da9c857 100644
--- a/exports/sfe_api.h
+++ b/exports/sfe_api.h
@@ -45,6 +45,8 @@
 #define SFE_RULE_CREATE_FLAG_USE_FLOW_BOTTOM_INTERFACE (1<<8) /**< Use flow interface number instead of top interface. */
 #define SFE_RULE_CREATE_FLAG_USE_RETURN_BOTTOM_INTERFACE (1<<9) /**< Use return interface number instead of top interface. */
 #define SFE_RULE_CREATE_FLAG_SRC_INTERFACE_CHECK  (1<<10)  /**< Check source interface. */
+#define SFE_RULE_CREATE_FLAG_FLOW_TRANSMIT_FAST (1<<11) /**< original flow transmit fast. */
+#define SFE_RULE_CREATE_FLAG_RETURN_TRANSMIT_FAST (1<<12) /**< return flow transmit fast. */
 
 /**
  * Rule creation validity flags.
diff --git a/sfe.c b/sfe.c
index eca2321..9e24f89 100644
--- a/sfe.c
+++ b/sfe.c
@@ -380,6 +380,46 @@
 }
 
 /*
+ * sfe_fast_xmit_check()
+ *	Check the fast transmit feasibility.
+ *
+ * This check the per direction's  attribute that could not go fast
+ * transmit
+ * xfrm packets, come from a local socket or need sk validation on the skb
+ */
+bool sfe_fast_xmit_check(struct sk_buff *skb, netdev_features_t features)
+{
+
+#ifdef CONFIG_SOCK_VALIDATE_XMIT
+	if (skb->sk && sk_fullsock(skb->sk) && skb->sk->sk_validate_xmit_skb) {
+		DEBUG_INFO("%px:need sk validation\n", skb);
+		return false;
+#ifdef CONFIG_TLS_DEVICE
+	} else if (skb->decrypted) {
+		DEBUG_INFO("%px:SK or decrypted\n", skb);
+		return false;
+#endif
+	}
+#endif
+	if (skb_vlan_tag_present(skb)) {
+		DEBUG_INFO("%px:Vlan is present\n", skb);
+		return false;
+	}
+
+	if (netif_needs_gso(skb, features)) {
+		DEBUG_INFO("%px:Need to be gso\n", skb);
+		return false;
+	}
+
+	if (skb_sec_path(skb)) {
+		DEBUG_INFO("%px:XFRM is present\n", skb);
+		return false;
+	}
+
+	return true;
+}
+
+/*
  * sfe_enqueue_msg()
  * 	Queue response message
  *
diff --git a/sfe.h b/sfe.h
index 15658e1..c117729 100644
--- a/sfe.h
+++ b/sfe.h
@@ -134,6 +134,11 @@
 extern int nf_ct_tcp_no_window_check;
 
 /*
+ * Check the fast transmit feasibility.
+ */
+bool sfe_fast_xmit_check(struct sk_buff *skb, netdev_features_t features);
+
+/*
  * This callback will be called in a timer
  * at 100 times per second to sync stats back to
  * Linux connection track.
diff --git a/sfe_ipv4.c b/sfe_ipv4.c
index bf76295..6315e62 100644
--- a/sfe_ipv4.c
+++ b/sfe_ipv4.c
@@ -308,6 +308,7 @@
 		stats->connection_flushes64 += s->connection_flushes64;
 		stats->packets_dropped64 += s->packets_dropped64;
 		stats->packets_forwarded64 += s->packets_forwarded64;
+		stats->packets_fast_xmited64 += s->packets_fast_xmited64;
 		stats->packets_not_forwarded64 += s->packets_not_forwarded64;
 		stats->pppoe_encap_packets_forwarded64 += s->pppoe_encap_packets_forwarded64;
 		stats->pppoe_decap_packets_forwarded64 += s->pppoe_decap_packets_forwarded64;
@@ -1218,10 +1219,12 @@
 		original_cm->dscp = msg->dscp_rule.flow_dscp << SFE_IPV4_DSCP_SHIFT;
 		original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_DSCP_REMARK;
 	}
-
 	if (msg->rule_flags & SFE_RULE_CREATE_FLAG_BRIDGE_FLOW) {
 		original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_BRIDGE_FLOW;
 	}
+	if (msg->rule_flags & SFE_RULE_CREATE_FLAG_FLOW_TRANSMIT_FAST) {
+		original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION;
+	}
 
 	/*
 	 * Add VLAN rule to original_cm
@@ -1415,10 +1418,12 @@
 		reply_cm->dscp = msg->dscp_rule.return_dscp << SFE_IPV4_DSCP_SHIFT;
 		reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_DSCP_REMARK;
 	}
-
 	if (msg->rule_flags & SFE_RULE_CREATE_FLAG_BRIDGE_FLOW) {
 		reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_BRIDGE_FLOW;
 	}
+	if (msg->rule_flags & SFE_RULE_CREATE_FLAG_RETURN_TRANSMIT_FAST) {
+		reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION;
+	}
 
 	if ((IPPROTO_GRE == tuple->protocol) && !sfe_ipv4_is_local_ip(si, reply_cm->match_dest_ip)) {
 		reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_PASSTHROUGH;
@@ -1984,6 +1989,7 @@
 	u32 packet, byte, original_cm_flags;
 	u16 pppoe_session_id;
 	u8 pppoe_remote_mac[ETH_ALEN];
+	u32 original_fast_xmit, reply_fast_xmit;
 #ifdef CONFIG_NF_FLOW_COOKIE
 	int src_flow_cookie, dst_flow_cookie;
 #endif
@@ -2024,6 +2030,7 @@
 	src_rx_packets = original_cm->rx_packet_count64;
 	src_rx_bytes = original_cm->rx_byte_count64;
 	src_mark = original_cm->mark;
+	original_fast_xmit = (original_cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT);
 	dest_dev = c->reply_dev;
 	dest_ip = c->dest_ip;
 	dest_ip_xlate = c->dest_ip_xlate;
@@ -2034,6 +2041,7 @@
 	dest_rx_packets = reply_cm->rx_packet_count64;
 	dest_rx_bytes = reply_cm->rx_byte_count64;
 	dest_mark = reply_cm->mark;
+	reply_fast_xmit = (reply_cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT);
 	last_sync_jiffies = get_jiffies_64() - c->last_sync_jiffies;
 	original_cm_flags = original_cm->flags;
 	pppoe_session_id = original_cm->pppoe_session_id;
@@ -2053,12 +2061,14 @@
 				"src_priority=\"%u\" src_dscp=\"%u\" "
 				"src_rx_pkts=\"%llu\" src_rx_bytes=\"%llu\" "
 				"src_mark=\"%08x\" "
+				"src_fast_xmit=\"%s\" "
 				"dest_dev=\"%s\" "
 				"dest_ip=\"%pI4\" dest_ip_xlate=\"%pI4\" "
 				"dest_port=\"%u\" dest_port_xlate=\"%u\" "
 				"dest_priority=\"%u\" dest_dscp=\"%u\" "
 				"dest_rx_pkts=\"%llu\" dest_rx_bytes=\"%llu\" "
 				"dest_mark=\"%08x\" "
+				"reply_fast_xmit=\"%s\" "
 #ifdef CONFIG_NF_FLOW_COOKIE
 				"src_flow_cookie=\"%d\" dst_flow_cookie=\"%d\" "
 #endif
@@ -2070,12 +2080,14 @@
 				src_priority, src_dscp,
 				src_rx_packets, src_rx_bytes,
 				src_mark,
+				original_fast_xmit ? "Yes" : "No",
 				dest_dev->name,
 				&dest_ip, &dest_ip_xlate,
 				ntohs(dest_port), ntohs(dest_port_xlate),
 				dest_priority, dest_dscp,
 				dest_rx_packets, dest_rx_bytes,
 				dest_mark,
+				reply_fast_xmit ? "Yes" : "No",
 #ifdef CONFIG_NF_FLOW_COOKIE
 				src_flow_cookie, dst_flow_cookie,
 #endif
@@ -2220,6 +2232,7 @@
 	bytes_read = snprintf(msg, CHAR_DEV_MSG_SIZE, "\t<stats "
 			      "num_connections=\"%u\" "
 			      "pkts_dropped=\"%llu\" "
+			      "pkts_fast_xmited=\"%llu\" "
 			      "pkts_forwarded=\"%llu\" pkts_not_forwarded=\"%llu\" "
 			      "create_requests=\"%llu\" create_collisions=\"%llu\" "
 			      "create_failures=\"%llu\" "
@@ -2231,6 +2244,7 @@
 			      "pppoe_bridge_pkts_fwded=\"%llu\" />\n",
 				num_conn,
 				stats.packets_dropped64,
+				stats.packets_fast_xmited64,
 				stats.packets_forwarded64,
 				stats.packets_not_forwarded64,
 				stats.connection_create_requests64,
@@ -2458,13 +2472,14 @@
 	}
 	return size;
 }
+
 /*
  * sysfs attributes.
  */
 static const struct device_attribute sfe_ipv4_cpu_attr =
 	__ATTR(stats_work_cpu, S_IWUSR | S_IRUGO, sfe_ipv4_get_cpu, sfe_ipv4_set_cpu);
 
- /*
+/*
  * sfe_ipv4_conn_match_hash_init()
  *	Initialize conn match hash lists
  */
@@ -2656,6 +2671,7 @@
 	sysfs_remove_file(si->sys_ipv4, &sfe_ipv4_flow_cookie_attr.attr);
 #endif /* CONFIG_NF_FLOW_COOKIE */
 	sysfs_remove_file(si->sys_ipv4, &sfe_ipv4_debug_dev_attr.attr);
+
 	sysfs_remove_file(si->sys_ipv4, &sfe_ipv4_cpu_attr.attr);
 
 	kobject_put(si->sys_ipv4);
diff --git a/sfe_ipv4.h b/sfe_ipv4.h
index 016ee8f..89b5ace 100644
--- a/sfe_ipv4.h
+++ b/sfe_ipv4.h
@@ -72,6 +72,12 @@
 					/* Source interface check.*/
 #define SFE_IPV4_CONNECTION_MATCH_FLAG_PASSTHROUGH (1<<14)
 					/* passthrough flow: encap/decap to be skipped for this flow */
+#define SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT (1<<15)
+					/* skb go fast xmit */
+#define SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED (1<<16)
+					/* Fast xmit flow checked or not */
+#define SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION (1<<17)
+					/* Fast xmit may be possible for this flow, if SFE check passes */
 
 /*
  * IPv4 connection matching structure.
@@ -183,6 +189,11 @@
 	 * Size of all needed L2 headers
 	 */
 	u16 l2_hdr_size;
+
+	/*
+	 * xmit device's feature
+	 */
+	netdev_features_t features;
 };
 
 /*
@@ -304,6 +315,7 @@
 	u64 connection_flushes64;		/* Number of IPv4 connection flushes */
 	u64 packets_dropped64;			/* Number of IPv4 packets dropped */
 	u64 packets_forwarded64;		/* Number of IPv4 packets forwarded */
+	u64 packets_fast_xmited64;		/* Number of IPv4 packets fast transmited */
 	u64 packets_not_forwarded64;	/* Number of IPv4 packets not forwarded */
 	u64 exception_events64[SFE_IPV4_EXCEPTION_EVENT_LAST];
 	u64 pppoe_encap_packets_forwarded64;	/* Number of IPv4 PPPoE encap packets forwarded */
diff --git a/sfe_ipv4_gre.c b/sfe_ipv4_gre.c
index c8d424b..084ea3b 100644
--- a/sfe_ipv4_gre.c
+++ b/sfe_ipv4_gre.c
@@ -298,7 +298,7 @@
 	 * Mark outgoing packet.
 	 */
 	if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_MARK)) {
-		skb->mark = cm->connection->mark;
+		skb->mark = cm->mark;
 	}
 
 	this_cpu_inc(si->stats_pcpu->packets_forwarded64);
diff --git a/sfe_ipv4_tcp.c b/sfe_ipv4_tcp.c
index 569dba9..d33778f 100644
--- a/sfe_ipv4_tcp.c
+++ b/sfe_ipv4_tcp.c
@@ -131,6 +131,8 @@
 	bool ret;
 	bool hw_csum;
 	bool bridge_flow;
+	bool fast_xmit;
+	netdev_features_t features;
 
 	/*
 	 * Is our packet too short to contain a valid TCP header?
@@ -690,6 +692,20 @@
 		skb->mark = cm->connection->mark;
 	}
 
+	/*
+	 * For the first packets, check if it could got fast xmit.
+	 */
+	if (unlikely(!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED)
+				&& (cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION))){
+		cm->features = netif_skb_features(skb);
+		if (likely(sfe_fast_xmit_check(skb, cm->features))) {
+			cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT;
+		}
+		cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED;
+	}
+	features = cm->features;
+	fast_xmit = !!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT);
+
 	rcu_read_unlock();
 
 	this_cpu_inc(si->stats_pcpu->packets_forwarded64);
@@ -701,6 +717,15 @@
 	prefetch(skb_shinfo(skb));
 
 	/*
+	 * We do per packet condition check before we could fast xmit the
+	 * packet.
+	 */
+	if (likely(fast_xmit && dev_fast_xmit(skb, xmit_dev, features))) {
+		this_cpu_inc(si->stats_pcpu->packets_fast_xmited64);
+		return 1;
+	}
+
+	/*
 	 * Mark that this packet has been fast forwarded.
 	 */
 	skb->fast_forwarded = 1;
diff --git a/sfe_ipv4_udp.c b/sfe_ipv4_udp.c
index 313d6ff..27e371d 100644
--- a/sfe_ipv4_udp.c
+++ b/sfe_ipv4_udp.c
@@ -132,6 +132,8 @@
 	int err;
 	bool bridge_flow;
 	int ret;
+	bool fast_xmit;
+	netdev_features_t features;
 
 	/*
 	 * Is our packet too short to contain a valid UDP header?
@@ -533,6 +535,21 @@
 		skb->mark = cm->connection->mark;
 	}
 
+	/*
+	 * For the first packets, check if it could got fast xmit.
+	 */
+	if (unlikely(!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED)
+				&& (cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION))){
+		cm->features = netif_skb_features(skb);
+		if (likely(sfe_fast_xmit_check(skb, cm->features))) {
+			cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT;
+		}
+		cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED;
+	}
+	features = cm->features;
+
+	fast_xmit = !!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT);
+
 	rcu_read_unlock();
 
 	this_cpu_inc(si->stats_pcpu->packets_forwarded64);
@@ -544,6 +561,15 @@
 	prefetch(skb_shinfo(skb));
 
 	/*
+	 * We do per packet condition check before we could fast xmit the
+	 * packet.
+	 */
+	if (likely(fast_xmit && dev_fast_xmit(skb, xmit_dev, features))) {
+		this_cpu_inc(si->stats_pcpu->packets_fast_xmited64);
+		return 1;
+	}
+
+	/*
 	 * Mark that this packet has been fast forwarded.
 	 */
 	skb->fast_forwarded = 1;
diff --git a/sfe_ipv6.c b/sfe_ipv6.c
index 4143b14..e5b6658 100644
--- a/sfe_ipv6.c
+++ b/sfe_ipv6.c
@@ -317,6 +317,7 @@
 		stats->connection_flushes64 += s->connection_flushes64;
 		stats->packets_dropped64 += s->packets_dropped64;
 		stats->packets_forwarded64 += s->packets_forwarded64;
+		stats->packets_fast_xmited64 += s->packets_fast_xmited64;
 		stats->packets_not_forwarded64 += s->packets_not_forwarded64;
 		stats->pppoe_encap_packets_forwarded64 += s->pppoe_encap_packets_forwarded64;
 		stats->pppoe_decap_packets_forwarded64 += s->pppoe_decap_packets_forwarded64;
@@ -1212,6 +1213,10 @@
 	if (msg->rule_flags & SFE_RULE_CREATE_FLAG_BRIDGE_FLOW) {
 		original_cm->flags |= SFE_IPV6_CONNECTION_MATCH_FLAG_BRIDGE_FLOW;
 	}
+	if (msg->rule_flags & SFE_RULE_CREATE_FLAG_FLOW_TRANSMIT_FAST) {
+		original_cm->flags |= SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION;
+	}
+
 
 	/*
 	 * Add VLAN rule to original_cm
@@ -1385,10 +1390,12 @@
 		reply_cm->dscp = msg->dscp_rule.return_dscp << SFE_IPV6_DSCP_SHIFT;
 		reply_cm->flags |= SFE_IPV6_CONNECTION_MATCH_FLAG_DSCP_REMARK;
 	}
-
 	if (msg->rule_flags & SFE_RULE_CREATE_FLAG_BRIDGE_FLOW) {
 		reply_cm->flags |= SFE_IPV6_CONNECTION_MATCH_FLAG_BRIDGE_FLOW;
 	}
+	if (msg->rule_flags & SFE_RULE_CREATE_FLAG_RETURN_TRANSMIT_FAST) {
+		reply_cm->flags |= SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION;
+	}
 
 	if ((IPPROTO_GRE == tuple->protocol) && !sfe_ipv6_is_local_ip(si, (uint8_t *)reply_cm->match_dest_ip)) {
 		reply_cm->flags |= SFE_IPV6_CONNECTION_MATCH_FLAG_PASSTHROUGH;
@@ -1932,6 +1939,7 @@
 	u32 packet, byte, original_cm_flags;
 	u16 pppoe_session_id;
 	u8 pppoe_remote_mac[ETH_ALEN];
+	u32 original_fast_xmit, reply_fast_xmit;
 #ifdef CONFIG_NF_FLOW_COOKIE
 	int src_flow_cookie, dst_flow_cookie;
 #endif
@@ -1972,6 +1980,7 @@
 	src_rx_packets = original_cm->rx_packet_count64;
 	src_rx_bytes = original_cm->rx_byte_count64;
 	src_mark = original_cm->mark;
+	original_fast_xmit = original_cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT;
 	dest_dev = c->reply_dev;
 	dest_ip = c->dest_ip[0];
 	dest_ip_xlate = c->dest_ip_xlate[0];
@@ -1986,6 +1995,7 @@
 	pppoe_session_id = original_cm->pppoe_session_id;
 	ether_addr_copy(pppoe_remote_mac, original_cm->pppoe_remote_mac);
 	dest_mark = reply_cm->mark;
+	reply_fast_xmit = reply_cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT;
 #ifdef CONFIG_NF_FLOW_COOKIE
 	src_flow_cookie = original_cm->flow_cookie;
 	dst_flow_cookie = reply_cm->flow_cookie;
@@ -2000,12 +2010,14 @@
 				"src_priority=\"%u\" src_dscp=\"%u\" "
 				"src_rx_pkts=\"%llu\" src_rx_bytes=\"%llu\" "
 				"src_mark=\"%08x\" "
+				"src_fast_xmit=\"%s\" "
 				"dest_dev=\"%s\" "
 				"dest_ip=\"%pI6\" dest_ip_xlate=\"%pI6\" "
 				"dest_port=\"%u\" dest_port_xlate=\"%u\" "
 				"dest_priority=\"%u\" dest_dscp=\"%u\" "
 				"dest_rx_pkts=\"%llu\" dest_rx_bytes=\"%llu\" "
 				"dest_mark=\"%08x\" "
+				"reply_fast_xmit=\"%s\" "
 #ifdef CONFIG_NF_FLOW_COOKIE
 				"src_flow_cookie=\"%d\" dst_flow_cookie=\"%d\" "
 #endif
@@ -2017,12 +2029,14 @@
 				src_priority, src_dscp,
 				src_rx_packets, src_rx_bytes,
 				src_mark,
+				original_fast_xmit ? "Yes" : "No",
 				dest_dev->name,
 				&dest_ip, &dest_ip_xlate,
 				ntohs(dest_port), ntohs(dest_port_xlate),
 				dest_priority, dest_dscp,
 				dest_rx_packets, dest_rx_bytes,
 				dest_mark,
+				reply_fast_xmit ? "Yes" : "No",
 #ifdef CONFIG_NF_FLOW_COOKIE
 				src_flow_cookie, dst_flow_cookie,
 #endif
@@ -2168,6 +2182,7 @@
 	bytes_read = snprintf(msg, CHAR_DEV_MSG_SIZE, "\t<stats "
 			      "num_connections=\"%u\" "
 			      "pkts_dropped=\"%llu\" "
+			      "pkts_fast_xmited=\"%llu\" "
 			      "pkts_forwarded=\"%llu\" pkts_not_forwarded=\"%llu\" "
 			      "create_requests=\"%llu\" create_collisions=\"%llu\" "
 			      "create_failures=\"%llu\" "
@@ -2180,6 +2195,7 @@
 
 				num_conn,
 				stats.packets_dropped64,
+				stats.packets_fast_xmited64,
 				stats.packets_forwarded64,
 				stats.packets_not_forwarded64,
 				stats.connection_create_requests64,
diff --git a/sfe_ipv6.h b/sfe_ipv6.h
index 408ebb5..7359f9b 100644
--- a/sfe_ipv6.h
+++ b/sfe_ipv6.h
@@ -85,6 +85,12 @@
 					/* Source interface check.*/
 #define SFE_IPV6_CONNECTION_MATCH_FLAG_PASSTHROUGH (1<<14)
 					/* passthrough flow: encap/decap to be skipped for this flow */
+#define SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT (1<<15)
+					/* go fast xmit*/
+#define SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED (1<<16)
+					/* fast xmit checked or not*/
+#define SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION (1<<17)
+					/* Fast xmit may be possible for this flow, if SFE check passes */
 
 /*
  * IPv6 connection matching structure.
@@ -194,6 +200,11 @@
 	 * Size of all needed L2 headers
 	 */
 	u16 l2_hdr_size;
+
+	/*
+	* xmit device's feature
+	 */
+	netdev_features_t features;
 };
 
 /*
@@ -322,6 +333,7 @@
 	u64 connection_flushes64;		/* Number of IPv6 connection flushes */
 	u64 packets_dropped64;			/* Number of IPv4 packets dropped */
 	u64 packets_forwarded64;		/* Number of IPv6 packets forwarded */
+	u64 packets_fast_xmited64;           /* Number of IPv6 packets fast transmited */
 	u64 packets_not_forwarded64;	/* Number of IPv6 packets not forwarded */
 	u64 exception_events64[SFE_IPV6_EXCEPTION_EVENT_LAST];
 	u64 pppoe_encap_packets_forwarded64;	/* Number of IPv6 PPPoE encap packets forwarded */
diff --git a/sfe_ipv6_tcp.c b/sfe_ipv6_tcp.c
index b78303c..28cc541 100644
--- a/sfe_ipv6_tcp.c
+++ b/sfe_ipv6_tcp.c
@@ -130,6 +130,8 @@
 	bool ret;
 	bool hw_csum;
 	bool bridge_flow;
+	bool fast_xmit;
+	netdev_features_t features;
 
 	/*
 	 * Is our packet too short to contain a valid TCP header?
@@ -695,6 +697,21 @@
 		skb->mark = cm->mark;
 	}
 
+	/*
+	 * For the first packets, check if it could got fast xmit.
+	 */
+	if (unlikely(!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED)
+				&& (cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION))){
+		cm->features = netif_skb_features(skb);
+		if (likely(sfe_fast_xmit_check(skb, cm->features))) {
+			cm->flags |= SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT;
+		}
+		cm->flags |= SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED;
+	}
+	features = cm->features;
+
+	fast_xmit = !!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT);
+
 	rcu_read_unlock();
 
 	this_cpu_inc(si->stats_pcpu->packets_forwarded64);
@@ -706,6 +723,15 @@
 	prefetch(skb_shinfo(skb));
 
 	/*
+	 * We do per packet condition check before we could fast xmit the
+	 * packet.
+	 */
+	if (likely(fast_xmit && dev_fast_xmit(skb, xmit_dev, features))) {
+		this_cpu_inc(si->stats_pcpu->packets_fast_xmited64);
+		return 1;
+	}
+
+	/*
 	 * Mark that this packet has been fast forwarded.
 	 */
 	skb->fast_forwarded = 1;
diff --git a/sfe_ipv6_udp.c b/sfe_ipv6_udp.c
index 9e32513..5ffefdd 100644
--- a/sfe_ipv6_udp.c
+++ b/sfe_ipv6_udp.c
@@ -137,6 +137,8 @@
 	int ret;
 	bool hw_csum;
 	bool bridge_flow;
+	bool fast_xmit;
+	netdev_features_t features;
 
 	DEBUG_TRACE("%px: sfe: sfe_ipv6_recv_udp called.\n", skb);
 
@@ -527,6 +529,21 @@
 		skb->mark = cm->mark;
 	}
 
+	/*
+	 * For the first packets, check if it could got fast xmit.
+	 */
+	if (unlikely(!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED)
+				&& (cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION))){
+		cm->features = netif_skb_features(skb);
+		if (likely(sfe_fast_xmit_check(skb, cm->features))) {
+			cm->flags |= SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT;
+		}
+		cm->flags |= SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED;
+	}
+	features = cm->features;
+
+	fast_xmit = !!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT);
+
 	rcu_read_unlock();
 
 	this_cpu_inc(si->stats_pcpu->packets_forwarded64);
@@ -538,6 +555,15 @@
 	prefetch(skb_shinfo(skb));
 
 	/*
+	 * We do per packet condition check before we could fast xmit the
+	 * packet.
+	 */
+	if (likely(fast_xmit && dev_fast_xmit(skb, xmit_dev, features))) {
+		this_cpu_inc(si->stats_pcpu->packets_fast_xmited64);
+		return 1;
+	}
+
+	/*
 	 * Mark that this packet has been fast forwarded.
 	 */
 	skb->fast_forwarded = 1;