[shortcut-fe] add pppoe support

Patch from Dave via email

Change-Id: Ibf4b6e18db2709cb575f448444eea848e73e0cfb
Signed-off-by: Matthew McClintock <mmcclint@codeaurora.org>
diff --git a/shortcut-fe/sfe_cm.c b/shortcut-fe/sfe_cm.c
index 5f6c33b..054efc2 100644
--- a/shortcut-fe/sfe_cm.c
+++ b/shortcut-fe/sfe_cm.c
@@ -108,21 +108,21 @@
 }
 
 /*
- * sfe_cm_find_mac_addr()
- *	Find the MAC address for a given IPv4 address.
+ * sfe_cm_find_dev_and_mac_addr()
+ *	Find the device and MAC address for a given IPv4 address.
  *
- * Returns true if we find the MAC address, otherwise false.
+ * Returns true if we find the device and MAC address, otherwise false.
  *
  * We look up the rtable entry for the address and, from its neighbour
  * structure, obtain the hardware address.  This means this function also
  * works if the neighbours are routers too.
  */
-static bool sfe_cm_find_mac_addr(uint32_t addr, uint8_t *mac_addr)
+static bool sfe_cm_find_dev_and_mac_addr(uint32_t addr, struct net_device **dev, uint8_t *mac_addr)
 {
 	struct neighbour *neigh;
 	struct rtable *rt;
 	struct dst_entry *dst;
-	struct net_device *dev;
+	struct net_device *mac_dev;
 
 	/*
 	 * Look up the rtable entry for the IP address then get the hardware
@@ -150,14 +150,17 @@
 		return false;
 	}
 
-	dev = neigh->dev;
-	if (!dev) {
+	mac_dev = neigh->dev;
+	if (!mac_dev) {
 		rcu_read_unlock();
 		dst_release(dst);
 		return false;
 	}
 
-	memcpy(mac_addr, neigh->ha, (size_t)dev->addr_len);
+	memcpy(mac_addr, neigh->ha, (size_t)mac_dev->addr_len);
+
+	dev_hold(mac_dev);
+	*dev = mac_dev;
 	rcu_read_unlock();
 
 	dst_release(dst);
@@ -168,6 +171,7 @@
 	 */
 	if (is_multicast_ether_addr(mac_addr)) {
 		DEBUG_TRACE("MAC is non-unicast - ignoring\n");
+		dev_put(mac_dev);
 		return false;
 	}
 
@@ -188,6 +192,7 @@
 	struct net_device *in;
 	struct nf_conn *ct;
 	enum ip_conntrack_info ctinfo;
+	struct net_device *dev;
 	struct net_device *src_dev;
 	struct net_device *dest_dev;
 	struct net_device *src_br_dev = NULL;
@@ -211,24 +216,12 @@
 	 * Don't process packets that are not being forwarded.
 	 */
 	in = dev_get_by_index(&init_net, skb->skb_iif);
-	if  (!in) {
+	if (!in) {
 		DEBUG_TRACE("packet not forwarding\n");
 		return NF_ACCEPT;
 	}
 
-	/*
-	 * Don't process packets with non-standard 802.3 MAC address sizes.
-	 */
-	if (unlikely(in->addr_len != ETH_ALEN)) {
-		DEBUG_TRACE("in device: %s not 802.3 hw addr len: %u, ignoring\n",
-				in->name, (unsigned)in->addr_len);
-		goto done1;
-	}
-	if (unlikely(out->addr_len != ETH_ALEN)) {
-		DEBUG_TRACE("out device: %s not 802.3 hw addr len: %u, ignoring\n",
-				out->name, (unsigned)out->addr_len);
-		goto done1;
-	}
+	dev_put(in);
 
 	/*
 	 * Don't process packets that aren't being tracked by conntrack.
@@ -236,7 +229,7 @@
 	ct = nf_ct_get(skb, &ctinfo);
 	if (unlikely(!ct)) {
 		DEBUG_TRACE("no conntrack connection, ignoring\n");
-		goto done1;
+		return NF_ACCEPT;
 	}
 
 	/*
@@ -244,7 +237,7 @@
 	 */
 	if (unlikely(ct == &nf_conntrack_untracked)) {
 		DEBUG_TRACE("untracked connection\n");
-		goto done1;
+		return NF_ACCEPT;
 	}
 
 	/*
@@ -252,7 +245,7 @@
 	 */
 	if (unlikely(nfct_help(ct))) {
 		DEBUG_TRACE("connection has helper\n");
-		goto done1;
+		return NF_ACCEPT;
 	}
 
 	/*
@@ -305,7 +298,7 @@
 		 */
 		if (!test_bit(IPS_ASSURED_BIT, &ct->status)) {
 			DEBUG_TRACE("non-established connection\n");
-			goto done1;
+			return NF_ACCEPT;
 		}
 
 		/*
@@ -319,7 +312,7 @@
 			DEBUG_TRACE("connection in termination state: %#x, s: %pI4:%u, d: %pI4:%u\n",
 				    ct->proto.tcp.state, &sic.src_ip, ntohs(sic.src_port),
 				    &sic.dest_ip, ntohs(sic.dest_port));
-			goto done1;
+			return NF_ACCEPT;
 		}
 		spin_unlock_bh(&ct->lock);
 		break;
@@ -333,47 +326,37 @@
 
 	default:
 		DEBUG_TRACE("unhandled protocol %d\n", sic.protocol);
-		goto done1;
+		return NF_ACCEPT;
 	}
 
 	/*
-	 * Get the MAC addresses that correspond to source and destination host addresses.
+	 * Get the net device and MAC addresses that correspond to the various source and
+	 * destination host addresses.
 	 */
-	if (!sfe_cm_find_mac_addr(sic.src_ip, sic.src_mac)) {
+	if (!sfe_cm_find_dev_and_mac_addr(sic.src_ip, &src_dev, sic.src_mac)) {
 		DEBUG_TRACE("failed to find MAC address for src IP: %pI4\n", &sic.src_ip);
-		goto done1;
+		return NF_ACCEPT;
 	}
 
-	if (!sfe_cm_find_mac_addr(sic.src_ip_xlate, sic.src_mac_xlate)) {
+	if (!sfe_cm_find_dev_and_mac_addr(sic.src_ip_xlate, &dev, sic.src_mac_xlate)) {
 		DEBUG_TRACE("failed to find MAC address for xlate src IP: %pI4\n", &sic.src_ip_xlate);
 		goto done1;
 	}
 
-	/*
-	 * Do dest now
-	 */
-	if (!sfe_cm_find_mac_addr(sic.dest_ip, sic.dest_mac)) {
+	dev_put(dev);
+
+	if (!sfe_cm_find_dev_and_mac_addr(sic.dest_ip, &dev, sic.dest_mac)) {
 		DEBUG_TRACE("failed to find MAC address for dest IP: %pI4\n", &sic.dest_ip);
 		goto done1;
 	}
 
-	if (!sfe_cm_find_mac_addr(sic.dest_ip_xlate, sic.dest_mac_xlate)) {
+	dev_put(dev);
+
+	if (!sfe_cm_find_dev_and_mac_addr(sic.dest_ip_xlate, &dest_dev, sic.dest_mac_xlate)) {
 		DEBUG_TRACE("failed to find MAC address for xlate dest IP: %pI4\n", &sic.dest_ip_xlate);
 		goto done1;
 	}
 
-	/*
-	 * Get our device info.  If we're dealing with the "reply" direction here then
-	 * we'll need things swapped around.
-	 */
-	if (ctinfo < IP_CT_IS_REPLY) {
-		src_dev = in;
-		dest_dev = (struct net_device *)out;
-	} else {
-		src_dev = (struct net_device *)out;
-		dest_dev = in;
-	}
-
 #if (!SFE_HOOK_ABOVE_BRIDGE)
 	/*
 	 * Now our devices may actually be a bridge interface.  If that's
@@ -383,7 +366,7 @@
 		src_br_dev = br_port_dev_get(src_dev, sic.src_mac);
 		if (!src_br_dev) {
 			DEBUG_TRACE("no port found on bridge\n");
-			goto done1;
+			goto done2;
 		}
 
 		src_dev = src_br_dev;
@@ -393,7 +376,7 @@
 		dest_br_dev = br_port_dev_get(dest_dev, sic.dest_mac_xlate);
 		if (!dest_br_dev) {
 			DEBUG_TRACE("no port found on bridge\n");
-			goto done2;
+			goto done3;
 		}
 
 		dest_dev = dest_br_dev;
@@ -407,7 +390,7 @@
 		src_br_dev = src_dev->master;
 		if (!src_br_dev) {
 			DEBUG_TRACE("no bridge found for: %s\n", src_dev->name);
-			goto done1;
+			goto done2;
 		}
 
 		dev_hold(src_br_dev);
@@ -418,7 +401,7 @@
 		dest_br_dev = dest_dev->master;
 		if (!dest_br_dev) {
 			DEBUG_TRACE("no bridge found for: %s\n", dest_dev->name);
-			goto done2;
+			goto done3;
 		}
 
 		dev_hold(dest_br_dev);
@@ -429,9 +412,8 @@
 	sic.src_dev = src_dev;
 	sic.dest_dev = dest_dev;
 
-// XXX - these MTUs need handling correctly!
-	sic.src_mtu = 1500;
-	sic.dest_mtu = 1500;
+	sic.src_mtu = src_dev->mtu;
+	sic.dest_mtu = dest_dev->mtu;
 
 	sfe_ipv4_create_rule(&sic);
 
@@ -442,16 +424,16 @@
 		dev_put(dest_br_dev);
 	}
 
-done2:
+done3:
 	if (src_br_dev) {
 		dev_put(src_br_dev);
 	}
 
+done2:
+	dev_put(dest_dev);
+
 done1:
-	/*
-	 * Release the interface on which this skb arrived
-	 */
-	dev_put(in);
+	dev_put(src_dev);
 
 	return NF_ACCEPT;
 }
diff --git a/shortcut-fe/sfe_ipv4.c b/shortcut-fe/sfe_ipv4.c
index 056a2b2..4b72b4f 100644
--- a/shortcut-fe/sfe_ipv4.c
+++ b/shortcut-fe/sfe_ipv4.c
@@ -36,22 +36,22 @@
 #endif
 
 /*
- * The default Linux ethhdr structure is "packed".  It also has byte aligned
- * MAC addresses and this leads to poor performance.  This version is not
- * packed and has better alignment for the MAC addresses.
- */
-struct sfe_ipv4_ethhdr {
-	__be16 h_dest[ETH_ALEN / 2];
-	__be16 h_source[ETH_ALEN / 2];
-	__be16 h_proto;
-};
-
-/*
- * Based on the Linux IPv4 header, but with an optional "packed" attribute to
+ * An Ethernet header, but with an optional "packed" attribute to
  * help with performance on some platforms (see the definition of
  * SFE_IPV4_UNALIGNED_STRUCT)
  */
-struct sfe_ipv4_iphdr {
+struct sfe_ipv4_eth_hdr {
+	__be16 h_dest[ETH_ALEN / 2];
+	__be16 h_source[ETH_ALEN / 2];
+	__be16 h_proto;
+} SFE_IPV4_UNALIGNED_STRUCT;
+
+/*
+ * An IPv4 header, but with an optional "packed" attribute to
+ * help with performance on some platforms (see the definition of
+ * SFE_IPV4_UNALIGNED_STRUCT)
+ */
+struct sfe_ipv4_ip_hdr {
 #if defined(__LITTLE_ENDIAN_BITFIELD)
 	__u8 ihl:4,
 	     version:4;
@@ -77,11 +77,11 @@
 } SFE_IPV4_UNALIGNED_STRUCT;
 
 /*
- * Based on the Linux UDP header, but with an optional "packed" attribute to
+ * A UDP header, but with an optional "packed" attribute to
  * help with performance on some platforms (see the definition of
  * SFE_IPV4_UNALIGNED_STRUCT)
  */
-struct sfe_ipv4_udphdr {
+struct sfe_ipv4_udp_hdr {
 	__be16 source;
 	__be16 dest;
 	__be16 len;
@@ -89,11 +89,11 @@
 } SFE_IPV4_UNALIGNED_STRUCT;
 
 /*
- * Based on the Linux TCP header, but with an optional "packed" attribute to
+ * A TCP header, but with an optional "packed" attribute to
  * help with performance on some platforms (see the definition of
  * SFE_IPV4_UNALIGNED_STRUCT)
  */
-struct sfe_ipv4_tcphdr {
+struct sfe_ipv4_tcp_hdr {
 	__be16 source;
 	__be16 dest;
 	__be32 seq;
@@ -152,7 +152,9 @@
 					/* Perform destination translation */
 #define SFE_IPV4_CONNECTION_MATCH_FLAG_NO_SEQ_CHECK 0x4
 					/* Ignore TCP sequence numbers */
-#define SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_ETH_HDR 0x8
+#define SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_FAST_ETH_HDR 0x8
+					/* Fast Ethernet header write */
+#define SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_L2_HDR 0x10
 					/* Fast Ethernet header write */
 
 /*
@@ -458,7 +460,7 @@
  *
  * Note that this function assumes that we have only 20 bytes of IP header.
  */
-static inline uint16_t sfe_ipv4_gen_ip_csum(struct sfe_ipv4_iphdr *iph)
+static inline uint16_t sfe_ipv4_gen_ip_csum(struct sfe_ipv4_ip_hdr *iph)
 {
 	uint32_t sum;
 	uint16_t *i = (uint16_t *)iph;
@@ -824,10 +826,11 @@
 {
 	struct sfe_ipv4 *si = &__si;
 	struct sfe_ipv4_connection *c;
+
 	spin_lock(&si->lock);
 	c = sfe_ipv4_find_sfe_ipv4_connection(si, mark->protocol,
-			mark->src_ip, mark->src_port,
-			mark->dest_ip, mark->dest_port);
+					      mark->src_ip, mark->src_port,
+					      mark->dest_ip, mark->dest_port);
 	if (c) {
 		DEBUG_TRACE("Matching connection found for mark, "
 			    "setting from %08x to %08x\n",
@@ -1090,9 +1093,9 @@
  *	Handle UDP packet receives and forwarding.
  */
 static int sfe_ipv4_recv_udp(struct sfe_ipv4 *si, struct sk_buff *skb, struct net_device *dev,
-			     unsigned int len, struct sfe_ipv4_iphdr *iph, unsigned int ihl, bool flush_on_find)
+			     unsigned int len, struct sfe_ipv4_ip_hdr *iph, unsigned int ihl, bool flush_on_find)
 {
-	struct sfe_ipv4_udphdr *udph;
+	struct sfe_ipv4_udp_hdr *udph;
 	__be32 src_ip;
 	__be32 dest_ip;
 	__be16 src_port;
@@ -1104,7 +1107,7 @@
 	/*
 	 * Is our packet too short to contain a valid UDP header?
 	 */
-	if (unlikely(len < (sizeof(struct sfe_ipv4_udphdr) + ihl))) {
+	if (unlikely(len < (sizeof(struct sfe_ipv4_udp_hdr) + ihl))) {
 		spin_lock(&si->lock);
 		si->exception_events[SFE_IPV4_EXCEPTION_EVENT_UDP_HEADER_INCOMPLETE]++;
 		si->packets_not_forwarded++;
@@ -1122,7 +1125,7 @@
 	src_ip = iph->saddr;
 	dest_ip = iph->daddr;
 
-	udph = (struct sfe_ipv4_udphdr *)(skb->data + ihl);
+	udph = (struct sfe_ipv4_udp_hdr *)(skb->data + ihl);
 	src_port = udph->source;
 	dest_port = udph->dest;
 
@@ -1246,13 +1249,6 @@
 	 */
 	iph->check = sfe_ipv4_gen_ip_csum(iph);
 
-//	if ((nat_entry_data->tos & FASTNAT_DSCP_MASK) != (iph->tos & FASTNAT_DSCP_MASK)) {
-//		ipv4_change_dsfield(iph, (u_int8_t)(~FASTNAT_DSCP_MASK), nat_entry_data->tos);
-//	}
-
-//	skb->priority = nat_entry_data->priority;
-//	skb->mark = nat_entry_data->mark;
-
 	/*
 	 * Update traffic stats.
 	 */
@@ -1278,29 +1274,30 @@
 	skb->dev = xmit_dev;
 
 	/*
-	 * Do we have a simple Ethernet header to write?
+	 * Check to see if we need to write a header.
 	 */
-	if (unlikely(!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_ETH_HDR))) {
-		/*
-		 * If this is anything other than a point-to-point interface then we need to
-		 * create a header based on MAC addresses.
-		 */
-		if (likely(!(xmit_dev->flags & IFF_POINTOPOINT))) {
+	if (likely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_L2_HDR)) {
+		if (unlikely(!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_FAST_ETH_HDR))) {
 			xmit_dev->header_ops->create(skb, xmit_dev, ETH_P_IP,
 						     cm->xmit_dest_mac, cm->xmit_src_mac, len);
+		} else {
+			/*
+			 * For the simple case we write this really fast.
+			 */
+			struct sfe_ipv4_eth_hdr *eth = (struct sfe_ipv4_eth_hdr *)__skb_push(skb, ETH_HLEN);
+			eth->h_proto = htons(ETH_P_IP);
+			eth->h_dest[0] = htons(cm->xmit_dest_mac[0]);
+			eth->h_dest[1] = htons(cm->xmit_dest_mac[1]);
+			eth->h_dest[2] = htons(cm->xmit_dest_mac[2]);
+			eth->h_source[0] = htons(cm->xmit_src_mac[0]);
+			eth->h_source[1] = htons(cm->xmit_src_mac[1]);
+			eth->h_source[2] = htons(cm->xmit_src_mac[2]);
 		}
-	} else {
-		struct sfe_ipv4_ethhdr *eth = (struct sfe_ipv4_ethhdr *)__skb_push(skb, ETH_HLEN);
-		eth->h_proto = htons(ETH_P_IP);
-		eth->h_dest[0] = htons(cm->xmit_dest_mac[0]);
-		eth->h_dest[1] = htons(cm->xmit_dest_mac[1]);
-		eth->h_dest[2] = htons(cm->xmit_dest_mac[2]);
-		eth->h_source[0] = htons(cm->xmit_src_mac[0]);
-		eth->h_source[1] = htons(cm->xmit_src_mac[1]);
-		eth->h_source[2] = htons(cm->xmit_src_mac[2]);
 	}
 
-	/* Mark outgoing packet */
+	/*
+	 * Mark outgoing packet.
+	 */
 	skb->mark = cm->connection->mark;
 	if (skb->mark) {
 		DEBUG_TRACE("SKB MARK is NON ZERO %x\n", skb->mark);
@@ -1327,12 +1324,12 @@
  * sfe_ipv4_process_tcp_option_sack()
  *	Parse TCP SACK option and update ack according
  */
-static bool sfe_ipv4_process_tcp_option_sack(const struct sfe_ipv4_tcphdr *th, const uint32_t data_offs,
+static bool sfe_ipv4_process_tcp_option_sack(const struct sfe_ipv4_tcp_hdr *th, const uint32_t data_offs,
 					     uint32_t *ack) __attribute__((always_inline));
-static bool sfe_ipv4_process_tcp_option_sack(const struct sfe_ipv4_tcphdr *th, const uint32_t data_offs,
+static bool sfe_ipv4_process_tcp_option_sack(const struct sfe_ipv4_tcp_hdr *th, const uint32_t data_offs,
 					     uint32_t *ack)
 {
-	uint32_t length = sizeof(struct sfe_ipv4_tcphdr);
+	uint32_t length = sizeof(struct sfe_ipv4_tcp_hdr);
 	uint8_t *ptr = (uint8_t *)th + length;
 
 	/*
@@ -1410,9 +1407,9 @@
  *	Handle TCP packet receives and forwarding.
  */
 static int sfe_ipv4_recv_tcp(struct sfe_ipv4 *si, struct sk_buff *skb, struct net_device *dev,
-			     unsigned int len, struct sfe_ipv4_iphdr *iph, unsigned int ihl, bool flush_on_find)
+			     unsigned int len, struct sfe_ipv4_ip_hdr *iph, unsigned int ihl, bool flush_on_find)
 {
-	struct sfe_ipv4_tcphdr *tcph;
+	struct sfe_ipv4_tcp_hdr *tcph;
 	__be32 src_ip;
 	__be32 dest_ip;
 	__be16 src_port;
@@ -1426,7 +1423,7 @@
 	/*
 	 * Is our packet too short to contain a valid UDP header?
 	 */
-	if (unlikely(len < (sizeof(struct sfe_ipv4_tcphdr) + ihl))) {
+	if (unlikely(len < (sizeof(struct sfe_ipv4_tcp_hdr) + ihl))) {
 		spin_lock(&si->lock);
 		si->exception_events[SFE_IPV4_EXCEPTION_EVENT_TCP_HEADER_INCOMPLETE]++;
 		si->packets_not_forwarded++;
@@ -1444,7 +1441,7 @@
 	src_ip = iph->saddr;
 	dest_ip = iph->daddr;
 
-	tcph = (struct sfe_ipv4_tcphdr *)(skb->data + ihl);
+	tcph = (struct sfe_ipv4_tcp_hdr *)(skb->data + ihl);
 	src_port = tcph->source;
 	dest_port = tcph->dest;
 	flags = tcp_flag_word(tcph);
@@ -1580,7 +1577,7 @@
 		 * Check that our TCP data offset isn't too short.
 		 */
 		data_offs = tcph->doff << 2;
-		if (unlikely(data_offs < sizeof(struct sfe_ipv4_tcphdr))) {
+		if (unlikely(data_offs < sizeof(struct sfe_ipv4_tcp_hdr))) {
 			struct sfe_ipv4_connection *c = cm->connection;
 			sfe_ipv4_remove_sfe_ipv4_connection(si, c);
 			si->exception_events[SFE_IPV4_EXCEPTION_EVENT_TCP_SMALL_DATA_OFFS]++;
@@ -1612,7 +1609,7 @@
 		/*
 		 * Check that our TCP data offset isn't past the end of the packet.
 		 */
-		data_offs += sizeof(struct sfe_ipv4_iphdr);
+		data_offs += sizeof(struct sfe_ipv4_ip_hdr);
 		if (unlikely(len < data_offs)) {
 			struct sfe_ipv4_connection *c = cm->connection;
 			sfe_ipv4_remove_sfe_ipv4_connection(si, c);
@@ -1757,13 +1754,6 @@
 	 */
 	iph->check = sfe_ipv4_gen_ip_csum(iph);
 
-//	if ((nat_entry_data->tos & FASTNAT_DSCP_MASK) != (iph->tos & FASTNAT_DSCP_MASK)) {
-//		ipv4_change_dsfield(iph, (u_int8_t)(~FASTNAT_DSCP_MASK), nat_entry_data->tos);
-//	}
-
-//	skb->priority = nat_entry_data->priority;
-//	skb->mark = nat_entry_data->mark;
-
 	/*
 	 * Update traffic stats.
 	 */
@@ -1789,26 +1779,25 @@
 	skb->dev = xmit_dev;
 
 	/*
-	 * Do we have a simple Ethernet header to write?
+	 * Check to see if we need to write a header.
 	 */
-	if (unlikely(!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_ETH_HDR))) {
-		/*
-		 * If this is anything other than a point-to-point interface then we need to
-		 * create a header based on MAC addresses.
-		 */
-		if (likely(!(xmit_dev->flags & IFF_POINTOPOINT))) {
+	if (likely(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_L2_HDR)) {
+		if (unlikely(!(cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_FAST_ETH_HDR))) {
 			xmit_dev->header_ops->create(skb, xmit_dev, ETH_P_IP,
 						     cm->xmit_dest_mac, cm->xmit_src_mac, len);
+		} else {
+			/*
+			 * For the simple case we write this really fast.
+			 */
+			struct sfe_ipv4_eth_hdr *eth = (struct sfe_ipv4_eth_hdr *)__skb_push(skb, ETH_HLEN);
+			eth->h_proto = htons(ETH_P_IP);
+			eth->h_dest[0] = htons(cm->xmit_dest_mac[0]);
+			eth->h_dest[1] = htons(cm->xmit_dest_mac[1]);
+			eth->h_dest[2] = htons(cm->xmit_dest_mac[2]);
+			eth->h_source[0] = htons(cm->xmit_src_mac[0]);
+			eth->h_source[1] = htons(cm->xmit_src_mac[1]);
+			eth->h_source[2] = htons(cm->xmit_src_mac[2]);
 		}
-	} else {
-		struct sfe_ipv4_ethhdr *eth = (struct sfe_ipv4_ethhdr *)__skb_push(skb, ETH_HLEN);
-		eth->h_proto = htons(ETH_P_IP);
-		eth->h_dest[0] = htons(cm->xmit_dest_mac[0]);
-		eth->h_dest[1] = htons(cm->xmit_dest_mac[1]);
-		eth->h_dest[2] = htons(cm->xmit_dest_mac[2]);
-		eth->h_source[0] = htons(cm->xmit_src_mac[0]);
-		eth->h_source[1] = htons(cm->xmit_src_mac[1]);
-		eth->h_source[2] = htons(cm->xmit_src_mac[2]);
 	}
 
 	/*
@@ -1847,15 +1836,15 @@
  * within Linux has all of the correct state should it need it.
  */
 static int sfe_ipv4_recv_icmp(struct sfe_ipv4 *si, struct sk_buff *skb, struct net_device *dev,
-			      unsigned int len, struct sfe_ipv4_iphdr *iph, unsigned int ihl)
+			      unsigned int len, struct sfe_ipv4_ip_hdr *iph, unsigned int ihl)
 {
 	struct icmphdr *icmph;
-	struct sfe_ipv4_iphdr *icmp_iph;
+	struct sfe_ipv4_ip_hdr *icmp_iph;
 	unsigned int icmp_ihl_words;
 	unsigned int icmp_ihl;
 	uint32_t *icmp_trans_h;
-	struct sfe_ipv4_udphdr *icmp_udph;
-	struct sfe_ipv4_tcphdr *icmp_tcph;
+	struct sfe_ipv4_udp_hdr *icmp_udph;
+	struct sfe_ipv4_tcp_hdr *icmp_tcph;
 	__be32 src_ip;
 	__be32 dest_ip;
 	__be16 src_port;
@@ -1896,7 +1885,7 @@
 	 * Do we have the full embedded IP header?
 	 */
 	len -= sizeof(struct icmphdr);
-	if (unlikely(len < sizeof(struct sfe_ipv4_iphdr))) {
+	if (unlikely(len < sizeof(struct sfe_ipv4_ip_hdr))) {
 		spin_lock(&si->lock);
 		si->exception_events[SFE_IPV4_EXCEPTION_EVENT_ICMP_IPV4_HEADER_INCOMPLETE]++;
 		si->packets_not_forwarded++;
@@ -1909,7 +1898,7 @@
 	/*
 	 * Is our embedded IP version wrong?
 	 */
-	icmp_iph = (struct sfe_ipv4_iphdr *)(icmph + 1);
+	icmp_iph = (struct sfe_ipv4_ip_hdr *)(icmph + 1);
 	if (unlikely(icmp_iph->version != 4)) {
 		spin_lock(&si->lock);
 		si->exception_events[SFE_IPV4_EXCEPTION_EVENT_ICMP_IPV4_NON_V4]++;
@@ -1957,7 +1946,7 @@
 			return 0;
 		}
 
-		icmp_udph = (struct sfe_ipv4_udphdr *)icmp_trans_h;
+		icmp_udph = (struct sfe_ipv4_udp_hdr *)icmp_trans_h;
 		src_port = icmp_udph->source;
 		dest_port = icmp_udph->dest;
 		break;
@@ -1977,7 +1966,7 @@
 			return 0;
 		}
 
-		icmp_tcph = (struct sfe_ipv4_tcphdr *)icmp_trans_h;
+		icmp_tcph = (struct sfe_ipv4_tcp_hdr *)icmp_trans_h;
 		src_port = icmp_tcph->source;
 		dest_port = icmp_tcph->dest;
 		break;
@@ -2043,14 +2032,14 @@
 	unsigned int ihl;
 	bool flush_on_find;
 	bool ip_options;
-	struct sfe_ipv4_iphdr *iph;
+	struct sfe_ipv4_ip_hdr *iph;
 	uint32_t protocol;
 
 	/*
 	 * Check that we have space for an IP header here.
 	 */
 	len = skb->len;
-	if (unlikely(len < sizeof(struct sfe_ipv4_iphdr))) {
+	if (unlikely(len < sizeof(struct sfe_ipv4_ip_hdr))) {
 		spin_lock(&si->lock);
 		si->exception_events[SFE_IPV4_EXCEPTION_EVENT_HEADER_INCOMPLETE]++;
 		si->packets_not_forwarded++;
@@ -2063,9 +2052,9 @@
 	/*
 	 * Check that our "total length" is large enough for an IP header.
 	 */
-	iph = (struct sfe_ipv4_iphdr *)skb->data;
+	iph = (struct sfe_ipv4_ip_hdr *)skb->data;
 	tot_len = ntohs(iph->tot_len);
-	if (unlikely(tot_len < sizeof(struct sfe_ipv4_iphdr))) {
+	if (unlikely(tot_len < sizeof(struct sfe_ipv4_ip_hdr))) {
 		spin_lock(&si->lock);
 		si->exception_events[SFE_IPV4_EXCEPTION_EVENT_BAD_TOTAL_LENGTH]++;
 		si->packets_not_forwarded++;
@@ -2125,7 +2114,7 @@
 	 * options we need to recheck our header size.
 	 */
 	ihl = iph->ihl << 2;
-	ip_options = unlikely(ihl != sizeof(struct sfe_ipv4_iphdr)) ? true : false;
+	ip_options = unlikely(ihl != sizeof(struct sfe_ipv4_ip_hdr)) ? true : false;
 	if (unlikely(ip_options)) {
 		if (unlikely(len < ihl)) {
 			spin_lock(&si->lock);
@@ -2248,6 +2237,11 @@
 	struct sfe_ipv4_connection *c;
 	struct sfe_ipv4_connection_match *original_cm;
 	struct sfe_ipv4_connection_match *reply_cm;
+	struct net_device *dest_dev;
+	struct net_device *src_dev;
+
+	dest_dev = sic->dest_dev;
+	src_dev = sic->src_dev;
 
 	spin_lock_bh(&si->lock);
 	si->connection_create_requests++;
@@ -2311,7 +2305,7 @@
 	 * we always know both ends of a connection by their translated
 	 * addresses and not their public addresses.
 	 */
-	original_cm->match_dev = sic->src_dev;
+	original_cm->match_dev = src_dev;
 	original_cm->match_protocol = sic->protocol;
 	original_cm->match_src_ip = sic->src_ip;
 	original_cm->match_src_port = sic->src_port;
@@ -2325,9 +2319,9 @@
 	original_cm->rx_packet_count64 = 0;
 	original_cm->rx_byte_count = 0;
 	original_cm->rx_byte_count64 = 0;
-	original_cm->xmit_dev = sic->dest_dev;
+	original_cm->xmit_dev = dest_dev;
 	original_cm->xmit_dev_mtu = sic->dest_mtu;
-	memcpy(original_cm->xmit_src_mac, sic->dest_dev->dev_addr, ETH_ALEN);
+	memcpy(original_cm->xmit_src_mac, dest_dev->dev_addr, ETH_ALEN);
 	memcpy(original_cm->xmit_dest_mac, sic->dest_mac_xlate, ETH_ALEN);
 	original_cm->connection = c;
 	original_cm->counter_match = reply_cm;
@@ -2335,14 +2329,28 @@
 	original_cm->active_next = NULL;
 	original_cm->active_prev = NULL;
 	original_cm->active = false;
-	if (sic->dest_dev->header_ops->create == eth_header) {
-		original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_ETH_HDR;
+
+	/*
+	 * For PPP links we don't write an L2 header.  For everything else we do.
+	 */
+	if (!(dest_dev->flags & IFF_POINTOPOINT)) {
+		original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_L2_HDR;
+
+		/*
+		 * If our dev writes Ethernet headers then we can write a really fast
+		 * version.
+		 */
+		if (dest_dev->header_ops) {
+			if (dest_dev->header_ops->create == eth_header) {
+				original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_FAST_ETH_HDR;
+			}
+		}
 	}
 
 	/*
 	 * Fill in the "reply" direction connection matching object.
 	 */
-	reply_cm->match_dev = sic->dest_dev;
+	reply_cm->match_dev = dest_dev;
 	reply_cm->match_protocol = sic->protocol;
 	reply_cm->match_src_ip = sic->dest_ip_xlate;
 	reply_cm->match_src_port = sic->dest_port_xlate;
@@ -2356,9 +2364,9 @@
 	reply_cm->rx_packet_count64 = 0;
 	reply_cm->rx_byte_count = 0;
 	reply_cm->rx_byte_count64 = 0;
-	reply_cm->xmit_dev = sic->src_dev;
+	reply_cm->xmit_dev = src_dev;
 	reply_cm->xmit_dev_mtu = sic->src_mtu;
-	memcpy(reply_cm->xmit_src_mac, sic->src_dev->dev_addr, ETH_ALEN);
+	memcpy(reply_cm->xmit_src_mac, src_dev->dev_addr, ETH_ALEN);
 	memcpy(reply_cm->xmit_dest_mac, sic->src_mac, ETH_ALEN);
 	reply_cm->connection = c;
 	reply_cm->counter_match = original_cm;
@@ -2366,10 +2374,25 @@
 	reply_cm->active_next = NULL;
 	reply_cm->active_prev = NULL;
 	reply_cm->active = false;
-	if (sic->src_dev->header_ops->create == eth_header) {
-		reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_ETH_HDR;
+
+	/*
+	 * For PPP links we don't write an L2 header.  For everything else we do.
+	 */
+	if (!(src_dev->flags & IFF_POINTOPOINT)) {
+		reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_L2_HDR;
+
+		/*
+		 * If our dev writes Ethernet headers then we can write a really fast
+		 * version.
+		 */
+		if (src_dev->header_ops) {
+			if (src_dev->header_ops->create == eth_header) {
+				reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_FAST_ETH_HDR;
+			}
+		}
 	}
 
+
 	if (sic->dest_ip != sic->dest_ip_xlate || sic->dest_port != sic->dest_port_xlate) {
 		original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_XLATE_DEST;
 		reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_XLATE_SRC;
@@ -2385,13 +2408,13 @@
 	c->src_ip_xlate = sic->src_ip_xlate;
 	c->src_port = sic->src_port;
 	c->src_port_xlate = sic->src_port_xlate;
-	c->original_dev = sic->src_dev;
+	c->original_dev = src_dev;
 	c->original_match = original_cm;
 	c->dest_ip = sic->dest_ip;
 	c->dest_ip_xlate = sic->dest_ip_xlate;
 	c->dest_port = sic->dest_port;
 	c->dest_port_xlate = sic->dest_port_xlate;
-	c->reply_dev = sic->dest_dev;
+	c->reply_dev = dest_dev;
 	c->reply_match = reply_cm;
 	c->mark = sic->mark;
 
@@ -2440,7 +2463,7 @@
 		   sic->mark, sic->protocol,
 		   sic->src_dev->name, sic->src_mac, sic->src_mac_xlate,
 		   &sic->src_ip, &sic->src_ip_xlate, ntohs(sic->src_port), ntohs(sic->src_port_xlate),
-		   sic->dest_dev->name, sic->dest_mac, sic->dest_mac_xlate,
+		   dest_dev->name, sic->dest_mac, sic->dest_mac_xlate,
 		   &sic->dest_ip, &sic->dest_ip_xlate, ntohs(sic->dest_port), ntohs(sic->dest_port_xlate));
 
 	return 0;
@@ -2463,7 +2486,7 @@
 	 * to destroy.  If there isn't then we can't destroy it.
 	 */
 	c = sfe_ipv4_find_sfe_ipv4_connection(si, sid->protocol, sid->src_ip, sid->src_port,
-						sid->dest_ip, sid->dest_port);
+					      sid->dest_ip, sid->dest_port);
 	if (!c) {
 		si->connection_destroy_misses++;
 		spin_unlock_bh(&si->lock);