Merge "[qca-nss-sfe] Accelerate DS-LITE traffic with encap_limit"
diff --git a/Makefile b/Makefile
index e06326a..0f0e9e6 100644
--- a/Makefile
+++ b/Makefile
@@ -5,7 +5,7 @@
KERNELVERSION := $(word 1, $(subst ., ,$(KERNELVERSION))).$(word 2, $(subst ., ,$(KERNELVERSION)))
SFE_BASE_OBJS := sfe.o sfe_init.o
-SFE_IPV4_OBJS := sfe_ipv4.o sfe_ipv4_udp.o sfe_ipv4_tcp.o sfe_ipv4_icmp.o
+SFE_IPV4_OBJS := sfe_ipv4.o sfe_ipv4_udp.o sfe_ipv4_tcp.o sfe_ipv4_icmp.o sfe_ipv4_tun6rd.o
SFE_IPV6_OBJS := sfe_ipv6.o sfe_ipv6_udp.o sfe_ipv6_tcp.o sfe_ipv6_icmp.o sfe_ipv6_tunipip6.o
SFE_PPPOE_OBJS := sfe_pppoe.o
diff --git a/exports/sfe_api.h b/exports/sfe_api.h
index 54fe9f3..9477550 100644
--- a/exports/sfe_api.h
+++ b/exports/sfe_api.h
@@ -165,6 +165,18 @@
};
/**
+ * Connection mark structure.
+ */
+struct sfe_connection_mark {
+ int protocol; /**< Protocol number. */
+ __be32 src_ip[4]; /**< Source IP address. */
+ __be32 dest_ip[4]; /**< Destination IP address. */
+ __be16 src_port; /**< Source port number. */
+ __be16 dest_port; /**< Destination port number. */
+ u32 mark; /**< Mark value to be updated. */
+};
+
+/**
* Common message structure.
*/
struct sfe_cmn_msg {
@@ -768,6 +780,26 @@
bool sfe_is_l2_feature_enabled(void);
/**
+ * Updates mark values of an IPv4 connection.
+ *
+ * @param mark The mark object.
+ *
+ * @return
+ * None.
+ */
+void sfe_ipv4_mark_rule_update(struct sfe_connection_mark *mark);
+
+/**
+ * Updates mark values of an IPv6 connection.
+ *
+ * @param mark The mark object.
+ *
+ * @return
+ * None.
+ */
+void sfe_ipv6_mark_rule_update(struct sfe_connection_mark *mark);
+
+/**
* @}
*/
diff --git a/sfe.c b/sfe.c
index 868e912..81be6d2 100644
--- a/sfe.c
+++ b/sfe.c
@@ -175,12 +175,14 @@
}
/*
- * Does it have an IPv4 address? If it doesn't then it could be MAP-T
- * (or dslite) interface, else we can't do anything interesting here!
+ * Does it have an IPv4 address? If it doesn't then it
+ * could be map-t, dslite or tun6rd interface, otherwise we
+ * can't do anything interesting here!
*/
if (likely(in4_dev->ifa_list || (dev->priv_flags_ext & IFF_EXT_MAPT)
|| (dev->rtnl_link_ops
- && !strcmp(dev->rtnl_link_ops->kind, "ip6tnl")))) {
+ && (!strcmp(dev->rtnl_link_ops->kind, "ip6tnl")
+ || !strcmp(dev->rtnl_link_ops->kind, "sit"))))) {
return true;
}
return false;
@@ -684,6 +686,9 @@
case IPPROTO_GRE:
break;
+ case IPPROTO_IPV6:
+ break;
+
default:
ret = SFE_CMN_RESPONSE_EMSG;
sfe_incr_exceptions(SFE_EXCEPTION_PROTOCOL_NOT_SUPPORT);
diff --git a/sfe.h b/sfe.h
index 40f29aa..e246b53 100644
--- a/sfe.h
+++ b/sfe.h
@@ -127,18 +127,6 @@
};
/*
- * connection mark structure
- */
-struct sfe_connection_mark {
- int protocol;
- sfe_ip_addr_t src_ip;
- sfe_ip_addr_t dest_ip;
- __be16 src_port;
- __be16 dest_port;
- u32 mark;
-};
-
-/*
* Expose the hook for the receive processing.
*/
extern int (*athrs_fast_nat_recv)(struct sk_buff *skb);
diff --git a/sfe_ipv4.c b/sfe_ipv4.c
index ee9e356..0e12441 100644
--- a/sfe_ipv4.c
+++ b/sfe_ipv4.c
@@ -46,6 +46,7 @@
#include "sfe_ipv4_icmp.h"
#include "sfe_pppoe.h"
#include "sfe_ipv4_gre.h"
+#include "sfe_ipv4_tun6rd.h"
static char *sfe_ipv4_exception_events_string[SFE_IPV4_EXCEPTION_EVENT_LAST] = {
"UDP_HEADER_INCOMPLETE",
@@ -91,6 +92,9 @@
"PPPOE_NOT_SET_IN_CME",
"INGRESS_VLAN_TAG_MISMATCH",
"INVALID_SOURCE_INTERFACE",
+ "TUN6RD_NO_CONNECTION",
+ "TUN6RD_NEEDS_FRAGMENTATION",
+ "TUN6RD_SYNC_ON_FIND",
"GRE_HEADER_INCOMPLETE",
"GRE_NO_CONNECTION",
"GRE_IP_OPTIONS_OR_INITIAL_FRAGMENT",
@@ -526,6 +530,13 @@
}
/*
+ * dereference the decap direction top_interface_dev
+ */
+ if (c->reply_match->top_interface_dev) {
+ dev_put(c->reply_match->top_interface_dev);
+ }
+
+ /*
* Remove the connection match objects.
*/
sfe_ipv4_remove_connection_match(si, c->reply_match);
@@ -896,6 +907,9 @@
return sfe_ipv4_recv_gre(si, skb, dev, len, iph, ihl, sync_on_find, l2_info, tun_outer);
}
#endif
+ if (IPPROTO_IPV6 == protocol) {
+ return sfe_ipv4_recv_tun6rd(si, skb, dev, len, iph, ihl, sync_on_find, l2_info, true);
+ }
sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_UNHANDLED_PROTOCOL);
@@ -1034,6 +1048,33 @@
}
/*
+ * sfe_ipv4_mark_rule_update()
+ * Updates the mark values of match entries.
+ */
+void sfe_ipv4_mark_rule_update(struct sfe_connection_mark *mark)
+{
+ struct sfe_ipv4_connection *c;
+ struct sfe_ipv4 *si = &__si;
+
+ spin_lock_bh(&si->lock);
+ c = sfe_ipv4_find_connection(si, mark->protocol,
+ mark->src_ip[0],
+ mark->src_port,
+ mark->dest_ip[0],
+ mark->dest_port);
+ if (!c) {
+ spin_unlock_bh(&si->lock);
+ DEBUG_WARN("%px: connection not found for mark update\n", mark);
+ return;
+ }
+ c->original_match->mark = mark->mark;
+ c->reply_match->mark = mark->mark;
+ spin_unlock_bh(&si->lock);
+ DEBUG_TRACE("%px: connection mark updated with %d\n", mark, mark->mark);
+}
+EXPORT_SYMBOL(sfe_ipv4_mark_rule_update);
+
+/*
* sfe_ipv4_xmit_eth_type_check()
* Checking if MAC header has to be written.
*/
@@ -1155,7 +1196,7 @@
/*
* Allocate the various connection tracking objects.
*/
- c = (struct sfe_ipv4_connection *)kmalloc(sizeof(struct sfe_ipv4_connection), GFP_ATOMIC);
+ c = (struct sfe_ipv4_connection *)kzalloc(sizeof(struct sfe_ipv4_connection), GFP_ATOMIC);
if (unlikely(!c)) {
DEBUG_WARN("%px: memory allocation of connection entry failed\n", msg);
this_cpu_inc(si->stats_pcpu->connection_create_failures64);
@@ -1164,7 +1205,7 @@
return -ENOMEM;
}
- original_cm = (struct sfe_ipv4_connection_match *)kmalloc(sizeof(struct sfe_ipv4_connection_match), GFP_ATOMIC);
+ original_cm = (struct sfe_ipv4_connection_match *)kzalloc(sizeof(struct sfe_ipv4_connection_match), GFP_ATOMIC);
if (unlikely(!original_cm)) {
DEBUG_WARN("%px: memory allocation of connection match entry failed\n", msg);
this_cpu_inc(si->stats_pcpu->connection_create_failures64);
@@ -1174,7 +1215,7 @@
return -ENOMEM;
}
- reply_cm = (struct sfe_ipv4_connection_match *)kmalloc(sizeof(struct sfe_ipv4_connection_match), GFP_ATOMIC);
+ reply_cm = (struct sfe_ipv4_connection_match *)kzalloc(sizeof(struct sfe_ipv4_connection_match), GFP_ATOMIC);
if (unlikely(!reply_cm)) {
DEBUG_WARN("%px: memory allocation of connection match entry failed\n", msg);
this_cpu_inc(si->stats_pcpu->connection_create_failures64);
@@ -1255,18 +1296,11 @@
original_cm->xlate_src_port = 0;
}
- atomic_set(&original_cm->rx_packet_count, 0);
- original_cm->rx_packet_count64 = 0;
- atomic_set(&original_cm->rx_byte_count, 0);
- original_cm->rx_byte_count64 = 0;
-
original_cm->xmit_dev = dest_dev;
original_cm->xmit_dev_mtu = msg->conn_rule.return_mtu;
original_cm->connection = c;
original_cm->counter_match = reply_cm;
- original_cm->l2_hdr_size = 0;
- original_cm->flags = 0;
/*
* UDP Socket is valid only in decap direction.
@@ -1336,6 +1370,7 @@
original_cm->flow_accel = 1;
}
#endif
+
/*
* If l2_features are disabled and flow uses l2 features such as macvlan/bridge/pppoe/vlan,
* bottom interfaces are expected to be disabled in the flow rule and always top interfaces
@@ -1355,13 +1390,10 @@
}
}
- reply_cm->l2_hdr_size = 0;
if (msg->rule_flags & SFE_RULE_CREATE_FLAG_SRC_INTERFACE_CHECK) {
original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_SRC_INTERFACE_CHECK;
}
- reply_cm->flags = 0;
-
/*
* Adding PPPoE parameters to original and reply entries based on the direction where
* PPPoE header is valid in ECM rule.
@@ -1471,11 +1503,6 @@
reply_cm->xlate_src_port = 0;
}
- atomic_set(&reply_cm->rx_packet_count, 0);
- reply_cm->rx_packet_count64 = 0;
- atomic_set(&reply_cm->rx_byte_count, 0);
- reply_cm->rx_byte_count64 = 0;
-
reply_cm->xmit_dev = src_dev;
reply_cm->xmit_dev_mtu = msg->conn_rule.flow_mtu;
@@ -1550,6 +1577,7 @@
#else
if (!refcount_inc_not_zero(&sk->sk_refcnt)) {
#endif
+ this_cpu_inc(si->stats_pcpu->connection_create_failures64);
spin_unlock_bh(&si->lock);
kfree(reply_cm);
kfree(original_cm);
@@ -1604,6 +1632,8 @@
*/
original_cm->proto = NULL;
reply_cm->proto = NULL;
+ original_cm->top_interface_dev = NULL;
+ reply_cm->top_interface_dev = NULL;
#ifdef SFE_GRE_TUN_ENABLE
if ((IPPROTO_GRE == tuple->protocol) && !(reply_cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_PASSTHROUGH)) {
@@ -1612,6 +1642,8 @@
rcu_read_unlock();
if (unlikely(!reply_cm->proto)) {
+ this_cpu_inc(si->stats_pcpu->connection_create_failures64);
+ spin_unlock_bh(&si->lock);
kfree(reply_cm);
kfree(original_cm);
kfree(c);
@@ -1623,6 +1655,27 @@
}
#endif
+ if (IPPROTO_IPV6 == tuple->protocol) {
+ original_cm->proto = NULL;
+ rcu_read_lock();
+ reply_cm->proto = rcu_dereference(inet_protos[IPPROTO_IPV6]);
+ rcu_read_unlock();
+ reply_cm->top_interface_dev = dev_get_by_index(&init_net, msg->conn_rule.return_top_interface_num);
+
+ if (unlikely(!reply_cm->top_interface_dev)) {
+ DEBUG_WARN("%px: Unable to find top_interface_dev corresponding to %d\n", msg,
+ msg->conn_rule.return_top_interface_num);
+ this_cpu_inc(si->stats_pcpu->connection_create_failures64);
+ spin_unlock_bh(&si->lock);
+ kfree(reply_cm);
+ kfree(original_cm);
+ kfree(c);
+ dev_put(src_dev);
+ dev_put(dest_dev);
+ return -EINVAL;
+ }
+ }
+
#ifdef CONFIG_NF_FLOW_COOKIE
reply_cm->flow_cookie = 0;
#endif
diff --git a/sfe_ipv4.h b/sfe_ipv4.h
index 2f363ca..f19adb1 100644
--- a/sfe_ipv4.h
+++ b/sfe_ipv4.h
@@ -23,6 +23,7 @@
#define SFE_IPV4_DSCP_MASK 0x3
#define SFE_IPV4_DSCP_SHIFT 2
+#include <linux/version.h>
/*
* Specifies the lower bound on ACK numbers carried in the TCP header
@@ -102,7 +103,11 @@
__be16 match_dest_port; /* Destination port/connection ident */
struct udp_sock *up; /* Stores UDP sock information; valid only in decap path */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0))
+ const struct net_protocol *proto; /* stores protocol handler; valid only in decap path */
+#else
struct net_protocol *proto; /* stores protocol handler; valid only in decap path */
+#endif
/*
* Control the operations of the match.
@@ -185,6 +190,8 @@
u16 pppoe_session_id;
u8 pppoe_remote_mac[ETH_ALEN];
+ struct net_device *top_interface_dev; /* Used by tun6rd to store decap VLAN netdevice.*/
+
/*
* Size of all needed L2 headers
*/
@@ -282,6 +289,9 @@
SFE_IPV4_EXCEPTION_EVENT_PPPOE_NOT_SET_IN_CME,
SFE_IPV4_EXCEPTION_EVENT_INGRESS_VLAN_TAG_MISMATCH,
SFE_IPV4_EXCEPTION_EVENT_INVALID_SRC_IFACE,
+ SFE_IPV4_EXCEPTION_EVENT_TUN6RD_NO_CONNECTION,
+ SFE_IPV4_EXCEPTION_EVENT_TUN6RD_NEEDS_FRAGMENTATION,
+ SFE_IPV4_EXCEPTION_EVENT_TUN6RD_SYNC_ON_FIND,
SFE_IPV4_EXCEPTION_EVENT_GRE_HEADER_INCOMPLETE,
SFE_IPV4_EXCEPTION_EVENT_GRE_NO_CONNECTION,
SFE_IPV4_EXCEPTION_EVENT_GRE_IP_OPTIONS_OR_INITIAL_FRAGMENT,
diff --git a/sfe_ipv4_tcp.c b/sfe_ipv4_tcp.c
index fdb73e3..22071ee 100644
--- a/sfe_ipv4_tcp.c
+++ b/sfe_ipv4_tcp.c
@@ -733,9 +733,16 @@
* We do per packet condition check before we could fast xmit the
* packet.
*/
- if (likely(fast_xmit && dev_fast_xmit(skb, xmit_dev, features))) {
- this_cpu_inc(si->stats_pcpu->packets_fast_xmited64);
- return 1;
+ if (likely(fast_xmit)) {
+ if (likely(!skb_is_gso(skb))) {
+ if (likely(dev_fast_xmit(skb, xmit_dev, features))) {
+ this_cpu_inc(si->stats_pcpu->packets_fast_xmited64);
+ return 1;
+ }
+ } else {
+ cm->flags &= ~SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT;
+ DEBUG_TRACE("%px: fast xmit disabled for xmit dev %s", skb, xmit_dev->name);
+ }
}
/*
diff --git a/sfe_ipv4_tun6rd.c b/sfe_ipv4_tun6rd.c
new file mode 100644
index 0000000..3f33ee9
--- /dev/null
+++ b/sfe_ipv4_tun6rd.c
@@ -0,0 +1,235 @@
+/*
+ * sfe_ipv4_tun6rd.c
+ * Shortcut forwarding engine file for IPv4 TUN6RD
+ *
+ * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <linux/skbuff.h>
+#include <linux/etherdevice.h>
+#include <linux/version.h>
+#include <net/protocol.h>
+#include <net/ip.h>
+
+#include "sfe_debug.h"
+#include "sfe_api.h"
+#include "sfe.h"
+#include "sfe_flow_cookie.h"
+#include "sfe_ipv4.h"
+#include "sfe_vlan.h"
+
+/*
+ * sfe_ipv4_recv_tun6rd()
+ * Handle TUN6RD packet receives and forwarding.
+ */
+int sfe_ipv4_recv_tun6rd(struct sfe_ipv4 *si, struct sk_buff *skb, struct net_device *dev,
+ unsigned int len, struct iphdr *iph, unsigned int ihl,
+ bool sync_on_find, struct sfe_l2_info *l2_info, bool tun_outer)
+{
+ __be32 src_ip;
+ __be32 dest_ip;
+ __be16 src_port = 0;
+ __be16 dest_port = 0;
+ struct sfe_ipv4_connection_match *cm;
+
+ DEBUG_TRACE("%px: sfe: sfe_ipv4_recv_tun6rd called.\n", skb);
+
+ /*
+ * Read the IP address information. Read the IP header data first
+ * because we've almost certainly got that in the cache.
+ */
+ src_ip = iph->saddr;
+ dest_ip = iph->daddr;
+
+ rcu_read_lock();
+
+ /*
+ * Look for a connection match.
+ */
+#ifdef CONFIG_NF_FLOW_COOKIE
+ cm = si->sfe_flow_cookie_table[skb->flow_cookie & SFE_FLOW_COOKIE_MASK].match;
+ if (unlikely(!cm)) {
+ cm = sfe_ipv4_find_connection_match_rcu(si, dev, IPPROTO_IPV6, src_ip, src_port, dest_ip, dest_port);
+ }
+#else
+ cm = sfe_ipv4_find_connection_match_rcu(si, dev, IPPROTO_IPV6, src_ip, src_port, dest_ip, dest_port);
+#endif
+ if (unlikely(!cm)) {
+ rcu_read_unlock();
+ sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TUN6RD_NO_CONNECTION);
+ DEBUG_TRACE("%px: no tun6rd connection found\n", skb);
+ return 0;
+ }
+
+ /*
+ * If our packet has been marked as "sync on find" we will sync the status
+ * and forward it to slowpath.
+ */
+ if (unlikely(sync_on_find)) {
+ sfe_ipv4_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
+ rcu_read_unlock();
+ sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TUN6RD_SYNC_ON_FIND);
+ DEBUG_TRACE("%px: Sync on find\n", skb);
+
+ return 0;
+ }
+
+ /*
+ * If cm->proto is set, it means the decap path.
+ * Otherwise we forward the packet in encap path.
+ */
+ if(cm->proto) {
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0))
+ const struct net_protocol *ipprot = cm->proto;
+#else
+ struct net_protocol *ipprot = cm->proto;
+#endif
+
+ /*
+ * Do we expect an ingress VLAN tag for this flow?
+ * Note: We will only have ingress tag check in decap direction.
+ * Here, no modification is needed, we only check tag match between
+ * vlan hdr stored in cm and l2_info.
+ */
+ if (unlikely(!sfe_vlan_validate_ingress_tag(skb, cm->ingress_vlan_hdr_cnt, cm->ingress_vlan_hdr, l2_info))) {
+ rcu_read_unlock();
+ sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_INGRESS_VLAN_TAG_MISMATCH);
+ DEBUG_TRACE("VLAN tag mismatch. skb=%px\n"
+ "cm: %u [0]=%x/%x [1]=%x/%x\n"
+ "l2_info+: %u [0]=%x/%x [1]=%x/%x\n", skb,
+ cm->ingress_vlan_hdr_cnt,
+ htons(cm->ingress_vlan_hdr[0].tpid), cm->ingress_vlan_hdr[0].tci,
+ htons(cm->ingress_vlan_hdr[1].tpid), cm->ingress_vlan_hdr[1].tci,
+ l2_info->vlan_hdr_cnt,
+ htons(l2_info->vlan_hdr[0].tpid), l2_info->vlan_hdr[0].tci,
+ htons(l2_info->vlan_hdr[1].tpid), l2_info->vlan_hdr[1].tci);
+ return 0;
+ }
+ skb_reset_network_header(skb);
+ skb_pull(skb, ihl);
+ skb_reset_transport_header(skb);
+
+ /*
+ * ipprot->handler(skb) will always return 0;
+ * There is no way to tell whether the packet is dropped later in linux or not.
+ * Hence here inc the byte/packet count always.
+ */
+ atomic_inc(&cm->rx_packet_count);
+ atomic_add(len, &cm->rx_byte_count);
+ rcu_read_unlock();
+ this_cpu_inc(si->stats_pcpu->packets_forwarded64);
+ DEBUG_TRACE("%px: %s decap done \n", skb, __func__);
+
+ /*
+ * Update top interface for tunnel searching.
+ */
+ skb->dev = cm->top_interface_dev;
+ ipprot->handler(skb);
+ return 1;
+
+ }
+
+ /*
+ * If our packet is larger than the MTU of the transmit interface then
+ * we can't forward it easily.
+ */
+ if (unlikely(len > cm->xmit_dev_mtu)) {
+ sfe_ipv4_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
+ rcu_read_unlock();
+
+ sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_TUN6RD_NEEDS_FRAGMENTATION);
+ DEBUG_TRACE("%px: Larger than mtu\n", skb);
+ return 0;
+ }
+
+ /*
+ * Update DSCP
+ */
+ if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_DSCP_REMARK)) {
+ iph->tos = (iph->tos & SFE_IPV4_DSCP_MASK) | cm->dscp;
+ }
+
+ /*
+ * Update traffic stats.
+ */
+ atomic_inc(&cm->rx_packet_count);
+ atomic_add(len, &cm->rx_byte_count);
+
+ skb->dev = cm->xmit_dev;
+
+ /*
+ * Check to see if we need to add VLAN tags
+ */
+ if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_INSERT_EGRESS_VLAN_TAG)) {
+
+ /*
+ * Check if skb has enough headroom to write L2 headers
+ */
+ if (unlikely(skb_headroom(skb) < cm->l2_hdr_size)) {
+ rcu_read_unlock();
+ DEBUG_WARN("%px: Not enough headroom: %u\n", skb, skb_headroom(skb));
+ sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_NO_HEADROOM);
+ return 0;
+ }
+ sfe_vlan_add_tag(skb, cm->egress_vlan_hdr_cnt, cm->egress_vlan_hdr);
+ }
+
+ /*
+ * Check to see if we need to write a header.
+ */
+ if (likely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_L2_HDR)) {
+ if (unlikely(!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_FAST_ETH_HDR))) {
+ dev_hard_header(skb, cm->xmit_dev, ntohs(skb->protocol),
+ cm->xmit_dest_mac, cm->xmit_src_mac, len);
+ } else {
+ struct ethhdr *eth = (struct ethhdr *)__skb_push(skb, ETH_HLEN);
+ eth->h_proto = skb->protocol;
+ ether_addr_copy((u8 *)eth->h_dest, (u8 *)cm->xmit_dest_mac);
+ ether_addr_copy((u8 *)eth->h_source, (u8 *)cm->xmit_src_mac);
+ }
+ }
+
+ /*
+ * Update priority of skb.
+ */
+ if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_PRIORITY_REMARK)) {
+ skb->priority = cm->priority;
+ }
+
+ /*
+ * Mark outgoing packet.
+ */
+ if (unlikely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_MARK)) {
+ skb->mark = cm->mark;
+ }
+
+ rcu_read_unlock();
+
+ this_cpu_inc(si->stats_pcpu->packets_forwarded64);
+
+ /*
+ * We're going to check for GSO flags when we transmit the packet so
+ * start fetching the necessary cache line now.
+ */
+ prefetch(skb_shinfo(skb));
+
+ /*
+ * Mark that this packet has been fast forwarded and send it on its way.
+ */
+ skb->fast_forwarded = 1;
+ dev_queue_xmit(skb);
+
+ return 1;
+}
diff --git a/sfe_ipv4_tun6rd.h b/sfe_ipv4_tun6rd.h
new file mode 100644
index 0000000..bfa7cc0
--- /dev/null
+++ b/sfe_ipv4_tun6rd.h
@@ -0,0 +1,22 @@
+/*
+ * sfe_ipv4_tun6rd.h
+ * Shortcut forwarding engine header file for IPv4 TUN6RD
+ *
+ * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+int sfe_ipv4_recv_tun6rd(struct sfe_ipv4 *si, struct sk_buff *skb, struct net_device *dev,
+ unsigned int len, struct iphdr *iph, unsigned int ihl,
+ bool sync_on_find, struct sfe_l2_info *l2_info, bool tun_outer);
diff --git a/sfe_ipv6.c b/sfe_ipv6.c
index 73d8181..01183af 100644
--- a/sfe_ipv6.c
+++ b/sfe_ipv6.c
@@ -1083,6 +1083,33 @@
}
/*
+ * sfe_ipv6_mark_rule_update()
+ * Updates the mark values of match entries.
+ */
+void sfe_ipv6_mark_rule_update(struct sfe_connection_mark *mark)
+{
+ struct sfe_ipv6_connection *c;
+ struct sfe_ipv6 *si = &__si6;
+
+ spin_lock_bh(&si->lock);
+ c = sfe_ipv6_find_connection(si, mark->protocol,
+ (struct sfe_ipv6_addr *)mark->src_ip,
+ mark->src_port,
+ (struct sfe_ipv6_addr *)mark->dest_ip,
+ mark->dest_port);
+ if (!c) {
+ spin_unlock_bh(&si->lock);
+ DEBUG_WARN("%px: connection not found for mark update\n", mark);
+ return;
+ }
+ c->original_match ->mark = mark->mark;
+ c->reply_match->mark = mark->mark;
+ spin_unlock_bh(&si->lock);
+ DEBUG_TRACE("%px: connection mark updated with %d\n", mark, mark->mark);
+}
+EXPORT_SYMBOL(sfe_ipv6_mark_rule_update);
+
+/*
* sfe_ipv6_xmit_eth_type_check
* Checking if MAC header has to be written.
*/
@@ -1162,7 +1189,7 @@
/*
* Allocate the various connection tracking objects.
*/
- c = (struct sfe_ipv6_connection *)kmalloc(sizeof(struct sfe_ipv6_connection), GFP_ATOMIC);
+ c = (struct sfe_ipv6_connection *)kzalloc(sizeof(struct sfe_ipv6_connection), GFP_ATOMIC);
if (unlikely(!c)) {
DEBUG_WARN("%px: memory allocation of connection entry failed\n", msg);
this_cpu_inc(si->stats_pcpu->connection_create_failures64);
@@ -1171,7 +1198,7 @@
return -ENOMEM;
}
- original_cm = (struct sfe_ipv6_connection_match *)kmalloc(sizeof(struct sfe_ipv6_connection_match), GFP_ATOMIC);
+ original_cm = (struct sfe_ipv6_connection_match *)kzalloc(sizeof(struct sfe_ipv6_connection_match), GFP_ATOMIC);
if (unlikely(!original_cm)) {
this_cpu_inc(si->stats_pcpu->connection_create_failures64);
DEBUG_WARN("%px: memory allocation of connection match entry failed\n", msg);
@@ -1181,7 +1208,7 @@
return -ENOMEM;
}
- reply_cm = (struct sfe_ipv6_connection_match *)kmalloc(sizeof(struct sfe_ipv6_connection_match), GFP_ATOMIC);
+ reply_cm = (struct sfe_ipv6_connection_match *)kzalloc(sizeof(struct sfe_ipv6_connection_match), GFP_ATOMIC);
if (unlikely(!reply_cm)) {
this_cpu_inc(si->stats_pcpu->connection_create_failures64);
DEBUG_WARN("%px: memory allocation of connection match entry failed\n", msg);
@@ -1250,18 +1277,12 @@
original_cm->xlate_dest_ip[0] = *(struct sfe_ipv6_addr *)tuple->return_ip;
original_cm->xlate_dest_port = tuple->return_ident;
- atomic_set(&original_cm->rx_packet_count, 0);
- original_cm->rx_packet_count64 = 0;
- atomic_set(&original_cm->rx_byte_count, 0);
- original_cm->rx_byte_count64 = 0;
original_cm->xmit_dev = dest_dev;
original_cm->xmit_dev_mtu = msg->conn_rule.return_mtu;
original_cm->connection = c;
original_cm->counter_match = reply_cm;
- original_cm->l2_hdr_size = 0;
- original_cm->flags = 0;
/*
* Valid in decap direction only
@@ -1350,9 +1371,6 @@
}
}
- reply_cm->l2_hdr_size = 0;
- reply_cm->flags = 0;
-
/*
* Adding PPPoE parameters to original and reply entries based on the direction where
* PPPoE header is valid in ECM rule.
@@ -1449,10 +1467,6 @@
reply_cm->match_src_port = tuple->return_ident;
}
- atomic_set(&original_cm->rx_byte_count, 0);
- reply_cm->rx_packet_count64 = 0;
- atomic_set(&reply_cm->rx_byte_count, 0);
- reply_cm->rx_byte_count64 = 0;
reply_cm->xmit_dev = src_dev;
reply_cm->xmit_dev_mtu = msg->conn_rule.flow_mtu;
diff --git a/sfe_ipv6_tcp.c b/sfe_ipv6_tcp.c
index 6f740d8..5ae829c 100644
--- a/sfe_ipv6_tcp.c
+++ b/sfe_ipv6_tcp.c
@@ -736,9 +736,16 @@
* We do per packet condition check before we could fast xmit the
* packet.
*/
- if (likely(fast_xmit && dev_fast_xmit(skb, xmit_dev, features))) {
- this_cpu_inc(si->stats_pcpu->packets_fast_xmited64);
- return 1;
+ if (likely(fast_xmit)) {
+ if (likely(!skb_is_gso(skb))) {
+ if (likely(dev_fast_xmit(skb, xmit_dev, features))) {
+ this_cpu_inc(si->stats_pcpu->packets_fast_xmited64);
+ return 1;
+ }
+ } else {
+ cm->flags &= ~SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT;
+ DEBUG_TRACE("%px: fast xmit disabled for xmit dev %s", skb, xmit_dev->name);
+ }
}
/*