Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 1 | /* |
| 2 | * sfe_ipv6_tcp.c |
| 3 | * Shortcut forwarding engine file for IPv6 TCP |
| 4 | * |
| 5 | * Copyright (c) 2015-2016, 2019-2020, The Linux Foundation. All rights reserved. |
Guduri Prathyusha | 5f27e23 | 2022-01-06 14:39:04 +0530 | [diff] [blame] | 6 | * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 7 | * |
| 8 | * Permission to use, copy, modify, and/or distribute this software for any |
| 9 | * purpose with or without fee is hereby granted, provided that the above |
| 10 | * copyright notice and this permission notice appear in all copies. |
| 11 | * |
| 12 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES |
| 13 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF |
| 14 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR |
| 15 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES |
| 16 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN |
| 17 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF |
| 18 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. |
| 19 | */ |
| 20 | |
| 21 | #include <linux/skbuff.h> |
| 22 | #include <net/tcp.h> |
| 23 | #include <linux/etherdevice.h> |
| 24 | #include <linux/version.h> |
| 25 | |
| 26 | #include "sfe_debug.h" |
| 27 | #include "sfe_api.h" |
| 28 | #include "sfe.h" |
| 29 | #include "sfe_flow_cookie.h" |
| 30 | #include "sfe_ipv6.h" |
Guduri Prathyusha | 79a5fee | 2021-11-11 17:59:10 +0530 | [diff] [blame] | 31 | #include "sfe_pppoe.h" |
Wayne Tan | bb7f178 | 2021-12-13 11:16:04 -0800 | [diff] [blame] | 32 | #include "sfe_vlan.h" |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 33 | |
| 34 | /* |
| 35 | * sfe_ipv6_process_tcp_option_sack() |
| 36 | * Parse TCP SACK option and update ack according |
| 37 | */ |
| 38 | static bool sfe_ipv6_process_tcp_option_sack(const struct tcphdr *th, const u32 data_offs, |
| 39 | u32 *ack) |
| 40 | { |
| 41 | u32 length = sizeof(struct tcphdr); |
| 42 | u8 *ptr = (u8 *)th + length; |
| 43 | |
| 44 | /* |
| 45 | * Ignore processing if TCP packet has only TIMESTAMP option. |
| 46 | */ |
| 47 | if (likely(data_offs == length + TCPOLEN_TIMESTAMP + 1 + 1) |
| 48 | && likely(ptr[0] == TCPOPT_NOP) |
| 49 | && likely(ptr[1] == TCPOPT_NOP) |
| 50 | && likely(ptr[2] == TCPOPT_TIMESTAMP) |
| 51 | && likely(ptr[3] == TCPOLEN_TIMESTAMP)) { |
| 52 | return true; |
| 53 | } |
| 54 | |
| 55 | /* |
| 56 | * TCP options. Parse SACK option. |
| 57 | */ |
| 58 | while (length < data_offs) { |
| 59 | u8 size; |
| 60 | u8 kind; |
| 61 | |
| 62 | ptr = (u8 *)th + length; |
| 63 | kind = *ptr; |
| 64 | |
| 65 | /* |
| 66 | * NOP, for padding |
| 67 | * Not in the switch because to fast escape and to not calculate size |
| 68 | */ |
| 69 | if (kind == TCPOPT_NOP) { |
| 70 | length++; |
| 71 | continue; |
| 72 | } |
| 73 | |
| 74 | if (kind == TCPOPT_SACK) { |
| 75 | u32 sack = 0; |
| 76 | u8 re = 1 + 1; |
| 77 | |
| 78 | size = *(ptr + 1); |
| 79 | if ((size < (1 + 1 + TCPOLEN_SACK_PERBLOCK)) |
| 80 | || ((size - (1 + 1)) % (TCPOLEN_SACK_PERBLOCK)) |
| 81 | || (size > (data_offs - length))) { |
| 82 | return false; |
| 83 | } |
| 84 | |
| 85 | re += 4; |
| 86 | while (re < size) { |
| 87 | u32 sack_re; |
| 88 | u8 *sptr = ptr + re; |
| 89 | sack_re = (sptr[0] << 24) | (sptr[1] << 16) | (sptr[2] << 8) | sptr[3]; |
| 90 | if (sack_re > sack) { |
| 91 | sack = sack_re; |
| 92 | } |
| 93 | re += TCPOLEN_SACK_PERBLOCK; |
| 94 | } |
| 95 | if (sack > *ack) { |
| 96 | *ack = sack; |
| 97 | } |
| 98 | length += size; |
| 99 | continue; |
| 100 | } |
| 101 | if (kind == TCPOPT_EOL) { |
| 102 | return true; |
| 103 | } |
| 104 | size = *(ptr + 1); |
| 105 | if (size < 2) { |
| 106 | return false; |
| 107 | } |
| 108 | length += size; |
| 109 | } |
| 110 | |
| 111 | return true; |
| 112 | } |
| 113 | |
| 114 | /* |
| 115 | * sfe_ipv6_recv_tcp() |
| 116 | * Handle TCP packet receives and forwarding. |
| 117 | */ |
| 118 | int sfe_ipv6_recv_tcp(struct sfe_ipv6 *si, struct sk_buff *skb, struct net_device *dev, |
Ken Zhu | 88c5815 | 2021-12-09 15:12:06 -0800 | [diff] [blame] | 119 | unsigned int len, struct ipv6hdr *iph, unsigned int ihl, bool sync_on_find, struct sfe_l2_info *l2_info) |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 120 | { |
| 121 | struct tcphdr *tcph; |
| 122 | struct sfe_ipv6_addr *src_ip; |
| 123 | struct sfe_ipv6_addr *dest_ip; |
| 124 | __be16 src_port; |
| 125 | __be16 dest_port; |
| 126 | struct sfe_ipv6_connection_match *cm; |
| 127 | struct sfe_ipv6_connection_match *counter_cm; |
| 128 | u32 flags; |
| 129 | struct net_device *xmit_dev; |
| 130 | bool ret; |
Ratheesh Kannoth | a3cf0e0 | 2021-12-09 09:44:10 +0530 | [diff] [blame] | 131 | bool hw_csum; |
Ratheesh Kannoth | 71fc51e | 2022-01-05 10:02:47 +0530 | [diff] [blame] | 132 | bool bridge_flow; |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 133 | |
| 134 | /* |
Wayne Tan | bb7f178 | 2021-12-13 11:16:04 -0800 | [diff] [blame] | 135 | * Is our packet too short to contain a valid TCP header? |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 136 | */ |
| 137 | if (!pskb_may_pull(skb, (sizeof(struct tcphdr) + ihl))) { |
| 138 | |
| 139 | sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TCP_HEADER_INCOMPLETE); |
| 140 | DEBUG_TRACE("packet too short for TCP header\n"); |
| 141 | return 0; |
| 142 | } |
| 143 | |
| 144 | /* |
| 145 | * Read the IP address and port information. Read the IP header data first |
| 146 | * because we've almost certainly got that in the cache. We may not yet have |
| 147 | * the TCP header cached though so allow more time for any prefetching. |
| 148 | */ |
| 149 | src_ip = (struct sfe_ipv6_addr *)iph->saddr.s6_addr32; |
| 150 | dest_ip = (struct sfe_ipv6_addr *)iph->daddr.s6_addr32; |
| 151 | |
| 152 | tcph = (struct tcphdr *)(skb->data + ihl); |
| 153 | src_port = tcph->source; |
| 154 | dest_port = tcph->dest; |
| 155 | flags = tcp_flag_word(tcph); |
| 156 | |
| 157 | rcu_read_lock(); |
| 158 | |
| 159 | /* |
| 160 | * Look for a connection match. |
| 161 | */ |
| 162 | #ifdef CONFIG_NF_FLOW_COOKIE |
| 163 | cm = si->sfe_flow_cookie_table[skb->flow_cookie & SFE_FLOW_COOKIE_MASK].match; |
| 164 | if (unlikely(!cm)) { |
| 165 | cm = sfe_ipv6_find_connection_match_rcu(si, dev, IPPROTO_TCP, src_ip, src_port, dest_ip, dest_port); |
| 166 | } |
| 167 | #else |
| 168 | cm = sfe_ipv6_find_connection_match_rcu(si, dev, IPPROTO_TCP, src_ip, src_port, dest_ip, dest_port); |
| 169 | #endif |
| 170 | if (unlikely(!cm)) { |
| 171 | /* |
| 172 | * We didn't get a connection but as TCP is connection-oriented that |
| 173 | * may be because this is a non-fast connection (not running established). |
| 174 | * For diagnostic purposes we differentiate this here. |
| 175 | */ |
| 176 | if (likely((flags & (TCP_FLAG_SYN | TCP_FLAG_RST | TCP_FLAG_FIN | TCP_FLAG_ACK)) == TCP_FLAG_ACK)) { |
| 177 | rcu_read_unlock(); |
| 178 | |
| 179 | sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TCP_NO_CONNECTION_FAST_FLAGS); |
| 180 | |
| 181 | DEBUG_TRACE("no connection found - fast flags\n"); |
| 182 | return 0; |
| 183 | } |
| 184 | |
| 185 | rcu_read_unlock(); |
| 186 | |
| 187 | sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TCP_NO_CONNECTION_SLOW_FLAGS); |
| 188 | DEBUG_TRACE("no connection found - slow flags: 0x%x\n", |
| 189 | flags & (TCP_FLAG_SYN | TCP_FLAG_RST | TCP_FLAG_FIN | TCP_FLAG_ACK)); |
| 190 | return 0; |
| 191 | } |
| 192 | |
| 193 | /* |
Ratheesh Kannoth | 5dee377 | 2022-01-18 11:27:14 +0530 | [diff] [blame] | 194 | * Source interface validate. |
| 195 | */ |
| 196 | if (unlikely((cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_SRC_INTERFACE_CHECK) && (cm->match_dev != dev))) { |
| 197 | struct sfe_ipv6_connection *c = cm->connection; |
| 198 | spin_lock_bh(&si->lock); |
| 199 | ret = sfe_ipv6_remove_connection(si, c); |
| 200 | spin_unlock_bh(&si->lock); |
| 201 | |
| 202 | if (ret) { |
| 203 | sfe_ipv6_flush_connection(si, c, SFE_SYNC_REASON_FLUSH); |
| 204 | } |
| 205 | rcu_read_unlock(); |
| 206 | sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_INVALID_SRC_IFACE); |
| 207 | DEBUG_TRACE("flush on wrong source interface check failure\n"); |
| 208 | return 0; |
| 209 | } |
| 210 | |
| 211 | /* |
| 212 | * If our packet has beern marked as "flush on find" we can't actually |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 213 | * forward it in the fast path, but now that we've found an associated |
Ken Zhu | 88c5815 | 2021-12-09 15:12:06 -0800 | [diff] [blame] | 214 | * connection we need sync its status before throw it slow path. |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 215 | */ |
Ken Zhu | 88c5815 | 2021-12-09 15:12:06 -0800 | [diff] [blame] | 216 | if (unlikely(sync_on_find)) { |
| 217 | sfe_ipv6_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS); |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 218 | rcu_read_unlock(); |
| 219 | |
| 220 | sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TCP_IP_OPTIONS_OR_INITIAL_FRAGMENT); |
Ken Zhu | 88c5815 | 2021-12-09 15:12:06 -0800 | [diff] [blame] | 221 | DEBUG_TRACE("Sync on find\n"); |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 222 | return 0; |
| 223 | } |
| 224 | |
| 225 | #ifdef CONFIG_XFRM |
| 226 | /* |
| 227 | * We can't accelerate the flow on this direction, just let it go |
| 228 | * through the slow path. |
| 229 | */ |
| 230 | if (unlikely(!cm->flow_accel)) { |
| 231 | rcu_read_unlock(); |
| 232 | this_cpu_inc(si->stats_pcpu->packets_not_forwarded64); |
| 233 | return 0; |
| 234 | } |
| 235 | #endif |
| 236 | |
Wayne Tan | bb7f178 | 2021-12-13 11:16:04 -0800 | [diff] [blame] | 237 | /* |
| 238 | * Do we expect an ingress VLAN tag for this flow? |
| 239 | */ |
| 240 | if (unlikely(!sfe_vlan_validate_ingress_tag(skb, cm->ingress_vlan_hdr_cnt, cm->ingress_vlan_hdr, l2_info))) { |
| 241 | rcu_read_unlock(); |
| 242 | sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_INGRESS_VLAN_TAG_MISMATCH); |
| 243 | DEBUG_TRACE("VLAN tag mismatch. skb=%px\n", skb); |
| 244 | return 0; |
| 245 | } |
| 246 | |
Ratheesh Kannoth | 71fc51e | 2022-01-05 10:02:47 +0530 | [diff] [blame] | 247 | bridge_flow = !!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_BRIDGE_FLOW); |
| 248 | |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 249 | /* |
| 250 | * Does our hop_limit allow forwarding? |
| 251 | */ |
Ratheesh Kannoth | 71fc51e | 2022-01-05 10:02:47 +0530 | [diff] [blame] | 252 | if (likely(!bridge_flow)) { |
| 253 | if (unlikely(iph->hop_limit < 2)) { |
Ken Zhu | 88c5815 | 2021-12-09 15:12:06 -0800 | [diff] [blame] | 254 | sfe_ipv6_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS); |
Ratheesh Kannoth | 71fc51e | 2022-01-05 10:02:47 +0530 | [diff] [blame] | 255 | rcu_read_unlock(); |
| 256 | |
| 257 | sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TCP_SMALL_TTL); |
Ken Zhu | 88c5815 | 2021-12-09 15:12:06 -0800 | [diff] [blame] | 258 | DEBUG_TRACE("hop_limit too low\n"); |
Ratheesh Kannoth | 71fc51e | 2022-01-05 10:02:47 +0530 | [diff] [blame] | 259 | return 0; |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 260 | } |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 261 | } |
| 262 | |
| 263 | /* |
| 264 | * If our packet is larger than the MTU of the transmit interface then |
| 265 | * we can't forward it easily. |
| 266 | */ |
| 267 | if (unlikely((len > cm->xmit_dev_mtu) && !skb_is_gso(skb))) { |
Ken Zhu | 88c5815 | 2021-12-09 15:12:06 -0800 | [diff] [blame] | 268 | sfe_ipv6_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS); |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 269 | rcu_read_unlock(); |
| 270 | |
| 271 | sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TCP_NEEDS_FRAGMENTATION); |
Ken Zhu | 88c5815 | 2021-12-09 15:12:06 -0800 | [diff] [blame] | 272 | DEBUG_TRACE("Larger than MTU\n"); |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 273 | return 0; |
| 274 | } |
| 275 | |
| 276 | /* |
| 277 | * Look at our TCP flags. Anything missing an ACK or that has RST, SYN or FIN |
| 278 | * set is not a fast path packet. |
| 279 | */ |
| 280 | if (unlikely((flags & (TCP_FLAG_SYN | TCP_FLAG_RST | TCP_FLAG_FIN | TCP_FLAG_ACK)) != TCP_FLAG_ACK)) { |
| 281 | struct sfe_ipv6_connection *c = cm->connection; |
| 282 | spin_lock_bh(&si->lock); |
| 283 | ret = sfe_ipv6_remove_connection(si, c); |
| 284 | spin_unlock_bh(&si->lock); |
| 285 | |
| 286 | DEBUG_TRACE("TCP flags: 0x%x are not fast\n", |
| 287 | flags & (TCP_FLAG_SYN | TCP_FLAG_RST | TCP_FLAG_FIN | TCP_FLAG_ACK)); |
| 288 | if (ret) { |
| 289 | sfe_ipv6_flush_connection(si, c, SFE_SYNC_REASON_FLUSH); |
| 290 | } |
| 291 | rcu_read_unlock(); |
| 292 | |
| 293 | sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TCP_FLAGS); |
| 294 | return 0; |
| 295 | } |
| 296 | |
| 297 | counter_cm = cm->counter_match; |
| 298 | |
| 299 | /* |
| 300 | * Are we doing sequence number checking? |
| 301 | */ |
| 302 | if (likely(!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_NO_SEQ_CHECK))) { |
| 303 | u32 seq; |
| 304 | u32 ack; |
| 305 | u32 sack; |
| 306 | u32 data_offs; |
| 307 | u32 end; |
| 308 | u32 left_edge; |
| 309 | u32 scaled_win; |
| 310 | u32 max_end; |
| 311 | |
| 312 | /* |
| 313 | * Is our sequence fully past the right hand edge of the window? |
| 314 | */ |
| 315 | seq = ntohl(tcph->seq); |
| 316 | if (unlikely((s32)(seq - (cm->protocol_state.tcp.max_end + 1)) > 0)) { |
| 317 | struct sfe_ipv6_connection *c = cm->connection; |
| 318 | spin_lock_bh(&si->lock); |
| 319 | ret = sfe_ipv6_remove_connection(si, c); |
| 320 | spin_unlock_bh(&si->lock); |
| 321 | |
| 322 | DEBUG_TRACE("seq: %u exceeds right edge: %u\n", |
| 323 | seq, cm->protocol_state.tcp.max_end + 1); |
| 324 | if (ret) { |
| 325 | sfe_ipv6_flush_connection(si, c, SFE_SYNC_REASON_FLUSH); |
| 326 | } |
| 327 | rcu_read_unlock(); |
| 328 | |
| 329 | sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TCP_SEQ_EXCEEDS_RIGHT_EDGE); |
| 330 | return 0; |
| 331 | } |
| 332 | |
| 333 | /* |
| 334 | * Check that our TCP data offset isn't too short. |
| 335 | */ |
| 336 | data_offs = tcph->doff << 2; |
| 337 | if (unlikely(data_offs < sizeof(struct tcphdr))) { |
| 338 | struct sfe_ipv6_connection *c = cm->connection; |
| 339 | spin_lock_bh(&si->lock); |
| 340 | ret = sfe_ipv6_remove_connection(si, c); |
| 341 | spin_unlock_bh(&si->lock); |
| 342 | |
| 343 | DEBUG_TRACE("TCP data offset: %u, too small\n", data_offs); |
| 344 | if (ret) { |
| 345 | sfe_ipv6_flush_connection(si, c, SFE_SYNC_REASON_FLUSH); |
| 346 | } |
| 347 | rcu_read_unlock(); |
| 348 | |
| 349 | sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TCP_SMALL_DATA_OFFS); |
| 350 | return 0; |
| 351 | } |
| 352 | |
| 353 | /* |
| 354 | * Update ACK according to any SACK option. |
| 355 | */ |
| 356 | ack = ntohl(tcph->ack_seq); |
| 357 | sack = ack; |
| 358 | if (unlikely(!sfe_ipv6_process_tcp_option_sack(tcph, data_offs, &sack))) { |
| 359 | struct sfe_ipv6_connection *c = cm->connection; |
| 360 | spin_lock_bh(&si->lock); |
| 361 | ret = sfe_ipv6_remove_connection(si, c); |
| 362 | spin_unlock_bh(&si->lock); |
| 363 | |
| 364 | DEBUG_TRACE("TCP option SACK size is wrong\n"); |
| 365 | if (ret) { |
| 366 | sfe_ipv6_flush_connection(si, c, SFE_SYNC_REASON_FLUSH); |
| 367 | } |
| 368 | rcu_read_unlock(); |
| 369 | |
| 370 | sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TCP_BAD_SACK); |
| 371 | return 0; |
| 372 | } |
| 373 | |
| 374 | /* |
| 375 | * Check that our TCP data offset isn't past the end of the packet. |
| 376 | */ |
| 377 | data_offs += sizeof(struct ipv6hdr); |
| 378 | if (unlikely(len < data_offs)) { |
| 379 | struct sfe_ipv6_connection *c = cm->connection; |
| 380 | spin_lock_bh(&si->lock); |
| 381 | ret = sfe_ipv6_remove_connection(si, c); |
| 382 | spin_unlock_bh(&si->lock); |
| 383 | |
| 384 | DEBUG_TRACE("TCP data offset: %u, past end of packet: %u\n", |
| 385 | data_offs, len); |
| 386 | if (ret) { |
| 387 | sfe_ipv6_flush_connection(si, c, SFE_SYNC_REASON_FLUSH); |
| 388 | } |
| 389 | rcu_read_unlock(); |
| 390 | |
| 391 | sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TCP_BIG_DATA_OFFS); |
| 392 | return 0; |
| 393 | } |
| 394 | |
| 395 | end = seq + len - data_offs; |
| 396 | |
| 397 | /* |
| 398 | * Is our sequence fully before the left hand edge of the window? |
| 399 | */ |
| 400 | if (unlikely((s32)(end - (cm->protocol_state.tcp.end |
| 401 | - counter_cm->protocol_state.tcp.max_win - 1)) < 0)) { |
| 402 | struct sfe_ipv6_connection *c = cm->connection; |
| 403 | spin_lock_bh(&si->lock); |
| 404 | ret = sfe_ipv6_remove_connection(si, c); |
| 405 | spin_unlock_bh(&si->lock); |
| 406 | |
| 407 | DEBUG_TRACE("seq: %u before left edge: %u\n", |
| 408 | end, cm->protocol_state.tcp.end - counter_cm->protocol_state.tcp.max_win - 1); |
| 409 | if (ret) { |
| 410 | sfe_ipv6_flush_connection(si, c, SFE_SYNC_REASON_FLUSH); |
| 411 | } |
| 412 | rcu_read_unlock(); |
| 413 | |
| 414 | sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TCP_SEQ_BEFORE_LEFT_EDGE); |
| 415 | return 0; |
| 416 | } |
| 417 | |
| 418 | /* |
| 419 | * Are we acking data that is to the right of what has been sent? |
| 420 | */ |
| 421 | if (unlikely((s32)(sack - (counter_cm->protocol_state.tcp.end + 1)) > 0)) { |
| 422 | struct sfe_ipv6_connection *c = cm->connection; |
| 423 | spin_lock_bh(&si->lock); |
| 424 | ret = sfe_ipv6_remove_connection(si, c); |
| 425 | spin_unlock_bh(&si->lock); |
| 426 | |
| 427 | DEBUG_TRACE("ack: %u exceeds right edge: %u\n", |
| 428 | sack, counter_cm->protocol_state.tcp.end + 1); |
| 429 | if (ret) { |
| 430 | sfe_ipv6_flush_connection(si, c, SFE_SYNC_REASON_FLUSH); |
| 431 | } |
| 432 | rcu_read_unlock(); |
| 433 | |
| 434 | sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TCP_ACK_EXCEEDS_RIGHT_EDGE); |
| 435 | return 0; |
| 436 | } |
| 437 | |
| 438 | /* |
| 439 | * Is our ack too far before the left hand edge of the window? |
| 440 | */ |
| 441 | left_edge = counter_cm->protocol_state.tcp.end |
| 442 | - cm->protocol_state.tcp.max_win |
| 443 | - SFE_IPV6_TCP_MAX_ACK_WINDOW |
| 444 | - 1; |
| 445 | if (unlikely((s32)(sack - left_edge) < 0)) { |
| 446 | struct sfe_ipv6_connection *c = cm->connection; |
| 447 | spin_lock_bh(&si->lock); |
| 448 | ret = sfe_ipv6_remove_connection(si, c); |
| 449 | spin_unlock_bh(&si->lock); |
| 450 | |
| 451 | DEBUG_TRACE("ack: %u before left edge: %u\n", sack, left_edge); |
| 452 | if (ret) { |
| 453 | sfe_ipv6_flush_connection(si, c, SFE_SYNC_REASON_FLUSH); |
| 454 | } |
| 455 | rcu_read_unlock(); |
| 456 | |
| 457 | sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_TCP_ACK_BEFORE_LEFT_EDGE); |
| 458 | return 0; |
| 459 | } |
| 460 | |
| 461 | /* |
| 462 | * Have we just seen the largest window size yet for this connection? If yes |
| 463 | * then we need to record the new value. |
| 464 | */ |
| 465 | scaled_win = ntohs(tcph->window) << cm->protocol_state.tcp.win_scale; |
| 466 | scaled_win += (sack - ack); |
| 467 | if (unlikely(cm->protocol_state.tcp.max_win < scaled_win)) { |
| 468 | cm->protocol_state.tcp.max_win = scaled_win; |
| 469 | } |
| 470 | |
| 471 | /* |
| 472 | * If our sequence and/or ack numbers have advanced then record the new state. |
| 473 | */ |
| 474 | if (likely((s32)(end - cm->protocol_state.tcp.end) >= 0)) { |
| 475 | cm->protocol_state.tcp.end = end; |
| 476 | } |
| 477 | |
| 478 | max_end = sack + scaled_win; |
| 479 | if (likely((s32)(max_end - counter_cm->protocol_state.tcp.max_end) >= 0)) { |
| 480 | counter_cm->protocol_state.tcp.max_end = max_end; |
| 481 | } |
| 482 | } |
| 483 | |
| 484 | /* |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 485 | * Check if skb was cloned. If it was, unshare it. Because |
| 486 | * the data area is going to be written in this path and we don't want to |
| 487 | * change the cloned skb's data section. |
| 488 | */ |
| 489 | if (unlikely(skb_cloned(skb))) { |
| 490 | DEBUG_TRACE("%px: skb is a cloned skb\n", skb); |
| 491 | skb = skb_unshare(skb, GFP_ATOMIC); |
| 492 | if (!skb) { |
| 493 | DEBUG_WARN("Failed to unshare the cloned skb\n"); |
| 494 | rcu_read_unlock(); |
| 495 | return 0; |
| 496 | } |
| 497 | |
| 498 | /* |
| 499 | * Update the iph and tcph pointers with the unshared skb's data area. |
| 500 | */ |
| 501 | iph = (struct ipv6hdr *)skb->data; |
| 502 | tcph = (struct tcphdr *)(skb->data + ihl); |
| 503 | } |
| 504 | |
| 505 | /* |
Guduri Prathyusha | 5f27e23 | 2022-01-06 14:39:04 +0530 | [diff] [blame] | 506 | * For PPPoE packets, match server MAC and session id |
| 507 | */ |
| 508 | if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_PPPOE_DECAP)) { |
| 509 | struct pppoe_hdr *ph; |
| 510 | struct ethhdr *eth; |
| 511 | |
| 512 | if (unlikely(!sfe_l2_parse_flag_check(l2_info, SFE_L2_PARSE_FLAGS_PPPOE_INGRESS))) { |
| 513 | rcu_read_unlock(); |
| 514 | DEBUG_TRACE("%px: PPPoE header not present in packet for PPPoE rule\n", skb); |
| 515 | sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_INVALID_PPPOE_SESSION); |
| 516 | return 0; |
| 517 | } |
| 518 | |
| 519 | ph = (struct pppoe_hdr *)(skb->head + sfe_l2_pppoe_hdr_offset_get(l2_info)); |
| 520 | eth = (struct ethhdr *)(skb->head + sfe_l2_hdr_offset_get(l2_info)); |
| 521 | |
| 522 | if (unlikely(cm->pppoe_session_id != ntohs(ph->sid)) || unlikely(!(ether_addr_equal((u8*)cm->pppoe_remote_mac, (u8 *)eth->h_source)))) { |
| 523 | DEBUG_TRACE("%px: PPPoE sessions with session IDs %d and %d or server MACs %pM and %pM did not match \n", |
| 524 | skb, cm->pppoe_session_id, htons(ph->sid), cm->pppoe_remote_mac, eth->h_source); |
| 525 | rcu_read_unlock(); |
| 526 | sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_INVALID_PPPOE_SESSION); |
| 527 | return 0; |
| 528 | } |
| 529 | skb->protocol = htons(l2_info->protocol); |
| 530 | this_cpu_inc(si->stats_pcpu->pppoe_decap_packets_forwarded64); |
| 531 | |
| 532 | } else if (unlikely(sfe_l2_parse_flag_check(l2_info, SFE_L2_PARSE_FLAGS_PPPOE_INGRESS))) { |
| 533 | |
| 534 | /* |
Guduri Prathyusha | 034d635 | 2022-01-12 16:49:04 +0530 | [diff] [blame] | 535 | * If packet contains PPPoE header but CME doesn't contain PPPoE flag yet we are exceptioning the packet to linux |
Guduri Prathyusha | 5f27e23 | 2022-01-06 14:39:04 +0530 | [diff] [blame] | 536 | */ |
Guduri Prathyusha | 034d635 | 2022-01-12 16:49:04 +0530 | [diff] [blame] | 537 | if (unlikely(!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_BRIDGE_FLOW))) { |
| 538 | rcu_read_unlock(); |
| 539 | DEBUG_TRACE("%px: CME doesn't contain PPPoE flag but packet has PPPoE header\n", skb); |
| 540 | sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_PPPOE_NOT_SET_IN_CME); |
| 541 | return 0; |
| 542 | |
| 543 | } |
| 544 | |
| 545 | /* |
| 546 | * For bridged flows when packet contains PPPoE header, restore the header back and forward to xmit interface |
| 547 | */ |
| 548 | __skb_push(skb, (sizeof(struct pppoe_hdr) + sizeof(struct sfe_ppp_hdr))); |
| 549 | l2_info->l2_hdr_size -= (sizeof(struct pppoe_hdr) + sizeof(struct sfe_ppp_hdr)); |
| 550 | this_cpu_inc(si->stats_pcpu->pppoe_bridge_packets_forwarded64); |
Guduri Prathyusha | 5f27e23 | 2022-01-06 14:39:04 +0530 | [diff] [blame] | 551 | } |
| 552 | |
| 553 | /* |
Wayne Tan | bb7f178 | 2021-12-13 11:16:04 -0800 | [diff] [blame] | 554 | * Check if skb has enough headroom to write L2 headers |
| 555 | */ |
| 556 | if (unlikely(skb_headroom(skb) < cm->l2_hdr_size)) { |
| 557 | rcu_read_unlock(); |
| 558 | DEBUG_WARN("%px: Not enough headroom: %u\n", skb, skb_headroom(skb)); |
| 559 | sfe_ipv6_exception_stats_inc(si, SFE_IPV6_EXCEPTION_EVENT_NO_HEADROOM); |
| 560 | return 0; |
| 561 | } |
| 562 | |
| 563 | /* |
Guduri Prathyusha | 5f27e23 | 2022-01-06 14:39:04 +0530 | [diff] [blame] | 564 | * From this point on we're good to modify the packet. |
| 565 | */ |
| 566 | |
| 567 | /* |
Guduri Prathyusha | 79a5fee | 2021-11-11 17:59:10 +0530 | [diff] [blame] | 568 | * For PPPoE flows, add PPPoE header before L2 header is added. |
| 569 | */ |
Guduri Prathyusha | 034d635 | 2022-01-12 16:49:04 +0530 | [diff] [blame] | 570 | if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_PPPOE_ENCAP)) { |
Wayne Tan | bb7f178 | 2021-12-13 11:16:04 -0800 | [diff] [blame] | 571 | sfe_pppoe_add_header(skb, cm->pppoe_session_id, PPP_IPV6); |
Guduri Prathyusha | 79a5fee | 2021-11-11 17:59:10 +0530 | [diff] [blame] | 572 | this_cpu_inc(si->stats_pcpu->pppoe_encap_packets_forwarded64); |
| 573 | } |
| 574 | |
| 575 | /* |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 576 | * Update DSCP |
| 577 | */ |
| 578 | if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_DSCP_REMARK)) { |
| 579 | sfe_ipv6_change_dsfield(iph, cm->dscp); |
| 580 | } |
| 581 | |
| 582 | /* |
| 583 | * Decrement our hop_limit. |
| 584 | */ |
Ratheesh Kannoth | 71fc51e | 2022-01-05 10:02:47 +0530 | [diff] [blame] | 585 | if (likely(!bridge_flow)) { |
| 586 | iph->hop_limit -= 1; |
| 587 | } |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 588 | |
| 589 | /* |
Ratheesh Kannoth | a3cf0e0 | 2021-12-09 09:44:10 +0530 | [diff] [blame] | 590 | * Enable HW csum if rx checksum is verified and xmit interface is CSUM offload capable. |
| 591 | * Note: If L4 csum at Rx was found to be incorrect, we (router) should use incremental L4 checksum here |
| 592 | * so that HW does not re-calculate/replace the L4 csum |
| 593 | */ |
| 594 | hw_csum = !!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_CSUM_OFFLOAD) && (skb->ip_summed == CHECKSUM_UNNECESSARY); |
| 595 | |
| 596 | /* |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 597 | * Do we have to perform translations of the source address/port? |
| 598 | */ |
| 599 | if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_XLATE_SRC)) { |
| 600 | u16 tcp_csum; |
| 601 | u32 sum; |
| 602 | |
| 603 | iph->saddr.s6_addr32[0] = cm->xlate_src_ip[0].addr[0]; |
| 604 | iph->saddr.s6_addr32[1] = cm->xlate_src_ip[0].addr[1]; |
| 605 | iph->saddr.s6_addr32[2] = cm->xlate_src_ip[0].addr[2]; |
| 606 | iph->saddr.s6_addr32[3] = cm->xlate_src_ip[0].addr[3]; |
| 607 | tcph->source = cm->xlate_src_port; |
| 608 | |
Ratheesh Kannoth | a3cf0e0 | 2021-12-09 09:44:10 +0530 | [diff] [blame] | 609 | if (unlikely(!hw_csum)) { |
| 610 | tcp_csum = tcph->check; |
| 611 | sum = tcp_csum + cm->xlate_src_csum_adjustment; |
| 612 | sum = (sum & 0xffff) + (sum >> 16); |
| 613 | tcph->check = (u16)sum; |
| 614 | } |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 615 | } |
| 616 | |
| 617 | /* |
| 618 | * Do we have to perform translations of the destination address/port? |
| 619 | */ |
| 620 | if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_XLATE_DEST)) { |
| 621 | u16 tcp_csum; |
| 622 | u32 sum; |
| 623 | |
| 624 | iph->daddr.s6_addr32[0] = cm->xlate_dest_ip[0].addr[0]; |
| 625 | iph->daddr.s6_addr32[1] = cm->xlate_dest_ip[0].addr[1]; |
| 626 | iph->daddr.s6_addr32[2] = cm->xlate_dest_ip[0].addr[2]; |
| 627 | iph->daddr.s6_addr32[3] = cm->xlate_dest_ip[0].addr[3]; |
| 628 | tcph->dest = cm->xlate_dest_port; |
| 629 | |
Ratheesh Kannoth | a3cf0e0 | 2021-12-09 09:44:10 +0530 | [diff] [blame] | 630 | if (unlikely(!hw_csum)) { |
| 631 | tcp_csum = tcph->check; |
| 632 | sum = tcp_csum + cm->xlate_dest_csum_adjustment; |
| 633 | sum = (sum & 0xffff) + (sum >> 16); |
| 634 | tcph->check = (u16)sum; |
| 635 | } |
| 636 | } |
| 637 | |
| 638 | /* |
| 639 | * If HW checksum offload is not possible, incremental L4 checksum is used to update the packet. |
| 640 | * Setting ip_summed to CHECKSUM_UNNECESSARY ensures checksum is not recalculated further in packet |
| 641 | * path. |
| 642 | */ |
| 643 | if (likely(hw_csum)) { |
| 644 | skb->ip_summed = CHECKSUM_PARTIAL; |
| 645 | } else { |
| 646 | skb->ip_summed = CHECKSUM_UNNECESSARY; |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 647 | } |
| 648 | |
| 649 | /* |
| 650 | * Update traffic stats. |
| 651 | */ |
| 652 | atomic_inc(&cm->rx_packet_count); |
| 653 | atomic_add(len, &cm->rx_byte_count); |
| 654 | |
| 655 | xmit_dev = cm->xmit_dev; |
| 656 | skb->dev = xmit_dev; |
| 657 | |
| 658 | /* |
Wayne Tan | bb7f178 | 2021-12-13 11:16:04 -0800 | [diff] [blame] | 659 | * Check to see if we need to add VLAN tags |
| 660 | */ |
| 661 | if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_INSERT_EGRESS_VLAN_TAG)) { |
| 662 | sfe_vlan_add_tag(skb, cm->egress_vlan_hdr_cnt, cm->egress_vlan_hdr); |
| 663 | } |
| 664 | |
| 665 | /* |
| 666 | * Check to see if we need to write an Ethernet header. |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 667 | */ |
| 668 | if (likely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_WRITE_L2_HDR)) { |
| 669 | if (unlikely(!(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_WRITE_FAST_ETH_HDR))) { |
Guduri Prathyusha | 5f27e23 | 2022-01-06 14:39:04 +0530 | [diff] [blame] | 670 | dev_hard_header(skb, xmit_dev, ntohs(skb->protocol), |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 671 | cm->xmit_dest_mac, cm->xmit_src_mac, len); |
| 672 | } else { |
| 673 | /* |
| 674 | * For the simple case we write this really fast. |
| 675 | */ |
| 676 | struct ethhdr *eth = (struct ethhdr *)__skb_push(skb, ETH_HLEN); |
Guduri Prathyusha | 647fe3e | 2021-11-22 19:17:51 +0530 | [diff] [blame] | 677 | |
Guduri Prathyusha | 5f27e23 | 2022-01-06 14:39:04 +0530 | [diff] [blame] | 678 | eth->h_proto = skb->protocol; |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 679 | ether_addr_copy((u8 *)eth->h_dest, (u8 *)cm->xmit_dest_mac); |
| 680 | ether_addr_copy((u8 *)eth->h_source, (u8 *)cm->xmit_src_mac); |
| 681 | } |
| 682 | } |
| 683 | |
| 684 | /* |
| 685 | * Update priority of skb. |
| 686 | */ |
| 687 | if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_PRIORITY_REMARK)) { |
| 688 | skb->priority = cm->priority; |
| 689 | } |
| 690 | |
| 691 | /* |
| 692 | * Mark outgoing packet |
| 693 | */ |
Ken Zhu | 37040ea | 2021-09-09 21:11:15 -0700 | [diff] [blame] | 694 | if (unlikely(cm->flags & SFE_IPV6_CONNECTION_MATCH_FLAG_MARK)) { |
| 695 | skb->mark = cm->mark; |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 696 | } |
| 697 | |
| 698 | rcu_read_unlock(); |
| 699 | |
| 700 | this_cpu_inc(si->stats_pcpu->packets_forwarded64); |
| 701 | |
| 702 | /* |
| 703 | * We're going to check for GSO flags when we transmit the packet so |
| 704 | * start fetching the necessary cache line now. |
| 705 | */ |
| 706 | prefetch(skb_shinfo(skb)); |
| 707 | |
| 708 | /* |
| 709 | * Mark that this packet has been fast forwarded. |
| 710 | */ |
| 711 | skb->fast_forwarded = 1; |
| 712 | |
| 713 | /* |
| 714 | * Send the packet on its way. |
| 715 | */ |
| 716 | dev_queue_xmit(skb); |
| 717 | |
| 718 | return 1; |
| 719 | } |