blob: 14283ea9caac4b8c38896519e63acf861a1c2bdb [file] [log] [blame]
/*
* sfe_ipv4_tcp.c
* Shortcut forwarding engine - IPv4 TCP implementation
*
* Copyright (c) 2013-2016, 2019-2020, The Linux Foundation. All rights reserved.
* Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <linux/skbuff.h>
#include <net/tcp.h>
#include <linux/etherdevice.h>
#include <linux/lockdep.h>
#include "sfe_debug.h"
#include "sfe_api.h"
#include "sfe.h"
#include "sfe_flow_cookie.h"
#include "sfe_ipv4.h"
#include "sfe_pppoe.h"
#include "sfe_vlan.h"
/*
* sfe_ipv4_process_tcp_option_sack()
* Parse TCP SACK option and update ack according
*/
static bool sfe_ipv4_process_tcp_option_sack(const struct tcphdr *th, const u32 data_offs,
u32 *ack)
{
u32 length = sizeof(struct tcphdr);
u8 *ptr = (u8 *)th + length;
/*
* Ignore processing if TCP packet has only TIMESTAMP option.
*/
if (likely(data_offs == length + TCPOLEN_TIMESTAMP + 1 + 1)
&& likely(ptr[0] == TCPOPT_NOP)
&& likely(ptr[1] == TCPOPT_NOP)
&& likely(ptr[2] == TCPOPT_TIMESTAMP)
&& likely(ptr[3] == TCPOLEN_TIMESTAMP)) {
return true;
}
/*
* TCP options. Parse SACK option.
*/
while (length < data_offs) {
u8 size;
u8 kind;
ptr = (u8 *)th + length;
kind = *ptr;
/*
* NOP, for padding
* Not in the switch because to fast escape and to not calculate size
*/
if (kind == TCPOPT_NOP) {
length++;
continue;
}
if (kind == TCPOPT_SACK) {
u32 sack = 0;
u8 re = 1 + 1;
size = *(ptr + 1);
if ((size < (1 + 1 + TCPOLEN_SACK_PERBLOCK))
|| ((size - (1 + 1)) % (TCPOLEN_SACK_PERBLOCK))
|| (size > (data_offs - length))) {
return false;
}
re += 4;
while (re < size) {
u32 sack_re;
u8 *sptr = ptr + re;
sack_re = (sptr[0] << 24) | (sptr[1] << 16) | (sptr[2] << 8) | sptr[3];
if (sack_re > sack) {
sack = sack_re;
}
re += TCPOLEN_SACK_PERBLOCK;
}
if (sack > *ack) {
*ack = sack;
}
length += size;
continue;
}
if (kind == TCPOPT_EOL) {
return true;
}
size = *(ptr + 1);
if (size < 2) {
return false;
}
length += size;
}
return true;
}
/*
* sfe_ipv4_recv_tcp()
* Handle TCP packet receives and forwarding.
*/
int sfe_ipv4_recv_tcp(struct sfe_ipv4 *si, struct sk_buff *skb, struct net_device *dev,
unsigned int len, struct iphdr *iph, unsigned int ihl, bool sync_on_find, struct sfe_l2_info *l2_info)
{
struct tcphdr *tcph;
__be32 src_ip;
__be32 dest_ip;
__be16 src_port;
__be16 dest_port;
struct sfe_ipv4_connection_match *cm;
struct sfe_ipv4_connection_match *counter_cm;
u8 ttl;
u32 flags;
struct net_device *xmit_dev;
bool ret;
bool hw_csum;
bool bridge_flow;
bool fast_xmit;
netdev_features_t features;
/*
* Is our packet too short to contain a valid TCP header?
*/
if (unlikely(!pskb_may_pull(skb, (sizeof(struct tcphdr) + ihl)))) {
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TCP_HEADER_INCOMPLETE);
DEBUG_TRACE("packet too short for TCP header\n");
return 0;
}
/*
* Read the IP address and port information. Read the IP header data first
* because we've almost certainly got that in the cache. We may not yet have
* the TCP header cached though so allow more time for any prefetching.
*/
src_ip = iph->saddr;
dest_ip = iph->daddr;
tcph = (struct tcphdr *)(skb->data + ihl);
src_port = tcph->source;
dest_port = tcph->dest;
flags = tcp_flag_word(tcph);
rcu_read_lock();
/*
* Look for a connection match.
*/
#ifdef CONFIG_NF_FLOW_COOKIE
cm = si->sfe_flow_cookie_table[skb->flow_cookie & SFE_FLOW_COOKIE_MASK].match;
if (unlikely(!cm)) {
cm = sfe_ipv4_find_connection_match_rcu(si, dev, IPPROTO_TCP, src_ip, src_port, dest_ip, dest_port);
}
#else
cm = sfe_ipv4_find_connection_match_rcu(si, dev, IPPROTO_TCP, src_ip, src_port, dest_ip, dest_port);
#endif
if (unlikely(!cm)) {
/*
* We didn't get a connection but as TCP is connection-oriented that
* may be because this is a non-fast connection (not running established).
* For diagnostic purposes we differentiate this here.
*/
if (likely((flags & (TCP_FLAG_SYN | TCP_FLAG_RST | TCP_FLAG_FIN | TCP_FLAG_ACK)) == TCP_FLAG_ACK)) {
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TCP_NO_CONNECTION_FAST_FLAGS);
DEBUG_TRACE("no connection found - fast flags\n");
return 0;
}
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TCP_NO_CONNECTION_SLOW_FLAGS);
DEBUG_TRACE("no connection found - slow flags: 0x%x\n",
flags & (TCP_FLAG_SYN | TCP_FLAG_RST | TCP_FLAG_FIN | TCP_FLAG_ACK));
return 0;
}
/*
* Source interface validate.
*/
if (unlikely((cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_SRC_INTERFACE_CHECK) && (cm->match_dev != dev))) {
struct sfe_ipv4_connection *c = cm->connection;
spin_lock_bh(&si->lock);
ret = sfe_ipv4_remove_connection(si, c);
spin_unlock_bh(&si->lock);
if (ret) {
sfe_ipv4_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_INVALID_SRC_IFACE);
DEBUG_TRACE("flush on wrong source interface check failure\n");
return 0;
}
/*
* If our packet has beern marked as "flush on find" we can't actually
* forward it in the fast path, but now that we've found an associated
* connection we need sync its status before throw it slow path.
*/
if (unlikely(sync_on_find)) {
sfe_ipv4_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TCP_IP_OPTIONS_OR_INITIAL_FRAGMENT);
DEBUG_TRACE("Sync on find\n");
return 0;
}
#ifdef CONFIG_XFRM
/*
* We can't accelerate the flow on this direction, just let it go
* through the slow path.
*/
if (unlikely(!cm->flow_accel)) {
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->packets_not_forwarded64);
return 0;
}
#endif
/*
* Do we expect an ingress VLAN tag for this flow?
*/
if (unlikely(!sfe_vlan_validate_ingress_tag(skb, cm->ingress_vlan_hdr_cnt, cm->ingress_vlan_hdr, l2_info))) {
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_INGRESS_VLAN_TAG_MISMATCH);
DEBUG_TRACE("VLAN tag mismatch. skb=%px\n", skb);
return 0;
}
bridge_flow = !!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_BRIDGE_FLOW);
/*
* Does our TTL allow forwarding?
*/
if (likely(!bridge_flow)) {
ttl = iph->ttl;
if (unlikely(ttl < 2)) {
sfe_ipv4_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TCP_SMALL_TTL);
DEBUG_TRACE("TTL too low\n");
return 0;
}
}
/*
* If our packet is larger than the MTU of the transmit interface then
* we can't forward it easily.
*/
if (unlikely((len > cm->xmit_dev_mtu) && !skb_is_gso(skb))) {
sfe_ipv4_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TCP_IP_OPTIONS_OR_INITIAL_FRAGMENT);
DEBUG_TRACE("Larger than MTU\n");
return 0;
}
/*
* Look at our TCP flags. Anything missing an ACK or that has RST, SYN or FIN
* set is not a fast path packet.
*/
if (unlikely((flags & (TCP_FLAG_SYN | TCP_FLAG_RST | TCP_FLAG_FIN | TCP_FLAG_ACK)) != TCP_FLAG_ACK)) {
struct sfe_ipv4_connection *c = cm->connection;
spin_lock_bh(&si->lock);
ret = sfe_ipv4_remove_connection(si, c);
spin_unlock_bh(&si->lock);
DEBUG_TRACE("TCP flags: 0x%x are not fast\n",
flags & (TCP_FLAG_SYN | TCP_FLAG_RST | TCP_FLAG_FIN | TCP_FLAG_ACK));
if (ret) {
sfe_ipv4_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TCP_FLAGS);
return 0;
}
counter_cm = cm->counter_match;
/*
* Are we doing sequence number checking?
*/
if (likely(!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_NO_SEQ_CHECK))) {
u32 seq;
u32 ack;
u32 sack;
u32 data_offs;
u32 end;
u32 left_edge;
u32 scaled_win;
u32 max_end;
/*
* Is our sequence fully past the right hand edge of the window?
*/
seq = ntohl(tcph->seq);
if (unlikely((s32)(seq - (cm->protocol_state.tcp.max_end + 1)) > 0)) {
struct sfe_ipv4_connection *c = cm->connection;
spin_lock_bh(&si->lock);
ret = sfe_ipv4_remove_connection(si, c);
spin_unlock_bh(&si->lock);
DEBUG_TRACE("seq: %u exceeds right edge: %u\n",
seq, cm->protocol_state.tcp.max_end + 1);
if (ret) {
sfe_ipv4_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TCP_SEQ_EXCEEDS_RIGHT_EDGE);
return 0;
}
/*
* Check that our TCP data offset isn't too short.
*/
data_offs = tcph->doff << 2;
if (unlikely(data_offs < sizeof(struct tcphdr))) {
struct sfe_ipv4_connection *c = cm->connection;
spin_lock_bh(&si->lock);
ret = sfe_ipv4_remove_connection(si, c);
spin_unlock_bh(&si->lock);
DEBUG_TRACE("TCP data offset: %u, too small\n", data_offs);
if (ret) {
sfe_ipv4_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TCP_SMALL_DATA_OFFS);
return 0;
}
/*
* Update ACK according to any SACK option.
*/
ack = ntohl(tcph->ack_seq);
sack = ack;
if (unlikely(!sfe_ipv4_process_tcp_option_sack(tcph, data_offs, &sack))) {
struct sfe_ipv4_connection *c = cm->connection;
spin_lock_bh(&si->lock);
ret = sfe_ipv4_remove_connection(si, c);
spin_unlock_bh(&si->lock);
DEBUG_TRACE("TCP option SACK size is wrong\n");
if (ret) {
sfe_ipv4_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TCP_BAD_SACK);
return 0;
}
/*
* Check that our TCP data offset isn't past the end of the packet.
*/
data_offs += sizeof(struct iphdr);
if (unlikely(len < data_offs)) {
struct sfe_ipv4_connection *c = cm->connection;
spin_lock_bh(&si->lock);
ret = sfe_ipv4_remove_connection(si, c);
spin_unlock_bh(&si->lock);
DEBUG_TRACE("TCP data offset: %u, past end of packet: %u\n",
data_offs, len);
if (ret) {
sfe_ipv4_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TCP_BIG_DATA_OFFS);
return 0;
}
end = seq + len - data_offs;
/*
* Is our sequence fully before the left hand edge of the window?
*/
if (unlikely((s32)(end - (cm->protocol_state.tcp.end
- counter_cm->protocol_state.tcp.max_win - 1)) < 0)) {
struct sfe_ipv4_connection *c = cm->connection;
spin_lock_bh(&si->lock);
ret = sfe_ipv4_remove_connection(si, c);
spin_unlock_bh(&si->lock);
DEBUG_TRACE("seq: %u before left edge: %u\n",
end, cm->protocol_state.tcp.end - counter_cm->protocol_state.tcp.max_win - 1);
if (ret) {
sfe_ipv4_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TCP_SEQ_BEFORE_LEFT_EDGE);
return 0;
}
/*
* Are we acking data that is to the right of what has been sent?
*/
if (unlikely((s32)(sack - (counter_cm->protocol_state.tcp.end + 1)) > 0)) {
struct sfe_ipv4_connection *c = cm->connection;
spin_lock_bh(&si->lock);
ret = sfe_ipv4_remove_connection(si, c);
spin_unlock_bh(&si->lock);
DEBUG_TRACE("ack: %u exceeds right edge: %u\n",
sack, counter_cm->protocol_state.tcp.end + 1);
if (ret) {
sfe_ipv4_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TCP_ACK_EXCEEDS_RIGHT_EDGE);
return 0;
}
/*
* Is our ack too far before the left hand edge of the window?
*/
left_edge = counter_cm->protocol_state.tcp.end
- cm->protocol_state.tcp.max_win
- SFE_IPV4_TCP_MAX_ACK_WINDOW
- 1;
if (unlikely((s32)(sack - left_edge) < 0)) {
struct sfe_ipv4_connection *c = cm->connection;
spin_lock_bh(&si->lock);
ret = sfe_ipv4_remove_connection(si, c);
spin_unlock_bh(&si->lock);
DEBUG_TRACE("ack: %u before left edge: %u\n", sack, left_edge);
if (ret) {
sfe_ipv4_flush_connection(si, c, SFE_SYNC_REASON_FLUSH);
}
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TCP_ACK_BEFORE_LEFT_EDGE);
return 0;
}
/*
* Have we just seen the largest window size yet for this connection? If yes
* then we need to record the new value.
*/
scaled_win = ntohs(tcph->window) << cm->protocol_state.tcp.win_scale;
scaled_win += (sack - ack);
if (unlikely(cm->protocol_state.tcp.max_win < scaled_win)) {
cm->protocol_state.tcp.max_win = scaled_win;
}
/*
* If our sequence and/or ack numbers have advanced then record the new state.
*/
if (likely((s32)(end - cm->protocol_state.tcp.end) >= 0)) {
cm->protocol_state.tcp.end = end;
}
max_end = sack + scaled_win;
if (likely((s32)(max_end - counter_cm->protocol_state.tcp.max_end) >= 0)) {
counter_cm->protocol_state.tcp.max_end = max_end;
}
}
/*
* Check if skb was cloned. If it was, unshare it. Because
* the data area is going to be written in this path and we don't want to
* change the cloned skb's data section.
*/
if (unlikely(skb_cloned(skb))) {
DEBUG_TRACE("%px: skb is a cloned skb\n", skb);
skb = skb_unshare(skb, GFP_ATOMIC);
if (!skb) {
DEBUG_WARN("Failed to unshare the cloned skb\n");
rcu_read_unlock();
return 0;
}
/*
* Update the iph and tcph pointers with the unshared skb's data area.
*/
iph = (struct iphdr *)skb->data;
tcph = (struct tcphdr *)(skb->data + ihl);
}
/*
* For PPPoE packets, match server MAC and session id
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_PPPOE_DECAP)) {
struct ethhdr *eth;
bool pppoe_match;
if (unlikely(!sfe_l2_parse_flag_check(l2_info, SFE_L2_PARSE_FLAGS_PPPOE_INGRESS))) {
rcu_read_unlock();
DEBUG_TRACE("%px: PPPoE header not present in packet for PPPoE rule\n", skb);
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_INCORRECT_PPPOE_PARSING);
return 0;
}
eth = eth_hdr(skb);
pppoe_match = (cm->pppoe_session_id == sfe_l2_pppoe_session_id_get(l2_info)) &&
ether_addr_equal((u8*)cm->pppoe_remote_mac, (u8 *)eth->h_source);
if (unlikely(!pppoe_match)) {
DEBUG_TRACE("%px: PPPoE session ID %d and %d or MAC %pM and %pM did not match\n",
skb, cm->pppoe_session_id, sfe_l2_pppoe_session_id_get(l2_info),
cm->pppoe_remote_mac, eth->h_source);
rcu_read_unlock();
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_INVALID_PPPOE_SESSION);
return 0;
}
skb->protocol = htons(l2_info->protocol);
this_cpu_inc(si->stats_pcpu->pppoe_decap_packets_forwarded64);
} else if (unlikely(sfe_l2_parse_flag_check(l2_info, SFE_L2_PARSE_FLAGS_PPPOE_INGRESS))) {
/*
* If packet contains PPPoE header but CME doesn't contain PPPoE flag yet we are exceptioning
* the packet to linux
*/
if (unlikely(!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_BRIDGE_FLOW))) {
rcu_read_unlock();
DEBUG_TRACE("%px: CME doesn't contain PPPoE flag but packet has PPPoE header\n", skb);
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_PPPOE_NOT_SET_IN_CME);
return 0;
}
/*
* For bridged flows when packet contains PPPoE header, restore the header back and forward
* to xmit interface
*/
__skb_push(skb, (sizeof(struct pppoe_hdr) + sizeof(struct sfe_ppp_hdr)));
this_cpu_inc(si->stats_pcpu->pppoe_bridge_packets_forwarded64);
}
/*
* Check if skb has enough headroom to write L2 headers
*/
if (unlikely(skb_headroom(skb) < cm->l2_hdr_size)) {
rcu_read_unlock();
DEBUG_WARN("%px: Not enough headroom: %u\n", skb, skb_headroom(skb));
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_NO_HEADROOM);
return 0;
}
/*
* From this point on we're good to modify the packet.
*/
/*
* For PPPoE flows, add PPPoE header before L2 header is added.
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_PPPOE_ENCAP)) {
sfe_pppoe_add_header(skb, cm->pppoe_session_id, PPP_IP);
this_cpu_inc(si->stats_pcpu->pppoe_encap_packets_forwarded64);
}
/*
* Update DSCP
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_DSCP_REMARK)) {
iph->tos = (iph->tos & SFE_IPV4_DSCP_MASK) | cm->dscp;
}
/*
* Decrement our TTL.
*/
if (likely(!bridge_flow)) {
iph->ttl = ttl - 1;
}
/*
* Enable HW csum if rx checksum is verified and xmit interface is CSUM offload capable.
* Note: If L4 csum at Rx was found to be incorrect, we (router) should use incremental L4 checksum here
* so that HW does not re-calculate/replace the L4 csum
*/
hw_csum = !!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_CSUM_OFFLOAD) && (skb->ip_summed == CHECKSUM_UNNECESSARY);
/*
* Do we have to perform translations of the source address/port?
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_XLATE_SRC)) {
u16 tcp_csum;
u32 sum;
iph->saddr = cm->xlate_src_ip;
tcph->source = cm->xlate_src_port;
if (unlikely(!hw_csum)) {
tcp_csum = tcph->check;
if (unlikely(skb->ip_summed == CHECKSUM_PARTIAL)) {
sum = tcp_csum + cm->xlate_src_partial_csum_adjustment;
} else {
sum = tcp_csum + cm->xlate_src_csum_adjustment;
}
sum = (sum & 0xffff) + (sum >> 16);
tcph->check = (u16)sum;
}
}
/*
* Do we have to perform translations of the destination address/port?
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_XLATE_DEST)) {
u16 tcp_csum;
u32 sum;
iph->daddr = cm->xlate_dest_ip;
tcph->dest = cm->xlate_dest_port;
if (unlikely(!hw_csum)) {
tcp_csum = tcph->check;
if (unlikely(skb->ip_summed == CHECKSUM_PARTIAL)) {
sum = tcp_csum + cm->xlate_dest_partial_csum_adjustment;
} else {
sum = tcp_csum + cm->xlate_dest_csum_adjustment;
}
sum = (sum & 0xffff) + (sum >> 16);
tcph->check = (u16)sum;
}
}
/*
* If HW checksum offload is not possible, full L3 checksum and incremental L4 checksum
* are used to update the packet. Setting ip_summed to CHECKSUM_UNNECESSARY ensures checksum is
* not recalculated further in packet path.
*/
if (likely(hw_csum)) {
skb->ip_summed = CHECKSUM_PARTIAL;
} else {
iph->check = sfe_ipv4_gen_ip_csum(iph);
skb->ip_summed = CHECKSUM_UNNECESSARY;
}
/*
* Update traffic stats.
*/
atomic_inc(&cm->rx_packet_count);
atomic_add(len, &cm->rx_byte_count);
xmit_dev = cm->xmit_dev;
skb->dev = xmit_dev;
/*
* Check to see if we need to add VLAN tags
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_INSERT_EGRESS_VLAN_TAG)) {
sfe_vlan_add_tag(skb, cm->egress_vlan_hdr_cnt, cm->egress_vlan_hdr);
}
/*
* Check to see if we need to write an Ethernet header.
*/
if (likely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_L2_HDR)) {
if (unlikely(!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_FAST_ETH_HDR))) {
dev_hard_header(skb, xmit_dev, ntohs(skb->protocol),
cm->xmit_dest_mac, cm->xmit_src_mac, len);
} else {
/*
* For the simple case we write this really fast.
*/
struct ethhdr *eth = (struct ethhdr *)__skb_push(skb, ETH_HLEN);
eth->h_proto = skb->protocol;
ether_addr_copy((u8 *)eth->h_dest, (u8 *)cm->xmit_dest_mac);
ether_addr_copy((u8 *)eth->h_source, (u8 *)cm->xmit_src_mac);
}
}
/*
* Update priority of skb.
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_PRIORITY_REMARK)) {
skb->priority = cm->priority;
}
/*
* Mark outgoing packet
*/
if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_MARK)) {
skb->mark = cm->mark;
}
/*
* For the first packets, check if it could got fast xmit.
*/
if (unlikely(!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED)
&& (cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_DEV_ADMISSION))){
cm->features = netif_skb_features(skb);
if (likely(sfe_fast_xmit_check(skb, cm->features))) {
cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT;
}
cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT_FLOW_CHECKED;
}
features = cm->features;
fast_xmit = !!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT);
rcu_read_unlock();
this_cpu_inc(si->stats_pcpu->packets_forwarded64);
/*
* We're going to check for GSO flags when we transmit the packet so
* start fetching the necessary cache line now.
*/
prefetch(skb_shinfo(skb));
/*
* We do per packet condition check before we could fast xmit the
* packet.
*/
if (likely(fast_xmit && dev_fast_xmit(skb, xmit_dev, features))) {
this_cpu_inc(si->stats_pcpu->packets_fast_xmited64);
return 1;
}
/*
* Mark that this packet has been fast forwarded.
*/
skb->fast_forwarded = 1;
/*
* Send the packet on its way.
*/
dev_queue_xmit(skb);
return 1;
}