Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1 | /* |
| 2 | * sfe_ipv4.c |
| 3 | * Shortcut forwarding engine - IPv4 edition. |
| 4 | * |
Ratheesh Kannoth | 24fb1db | 2021-10-20 07:28:06 +0530 | [diff] [blame] | 5 | * Copyright (c) 2013-2016, 2019-2020, The Linux Foundation. All rights reserved. |
| 6 | * Copyright (c) 2021 Qualcomm Innovation Center, Inc. All rights reserved. |
| 7 | * |
| 8 | * Permission to use, copy, modify, and/or distribute this software for any |
| 9 | * purpose with or without fee is hereby granted, provided that the above |
| 10 | * copyright notice and this permission notice appear in all copies. |
| 11 | * |
Xiaoping Fan | a42c68b | 2015-08-07 18:00:39 -0700 | [diff] [blame] | 12 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES |
| 13 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF |
| 14 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR |
| 15 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES |
| 16 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN |
Ratheesh Kannoth | 24fb1db | 2021-10-20 07:28:06 +0530 | [diff] [blame] | 17 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF |
| 18 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 19 | */ |
Matthew McClintock | a322194 | 2014-01-16 11:44:26 -0600 | [diff] [blame] | 20 | |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 21 | #include <linux/module.h> |
Dave Hudson | dcd08fb | 2013-11-22 09:25:16 -0600 | [diff] [blame] | 22 | #include <linux/sysfs.h> |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 23 | #include <linux/skbuff.h> |
| 24 | #include <linux/icmp.h> |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 25 | #include <net/tcp.h> |
Dave Hudson | dcd08fb | 2013-11-22 09:25:16 -0600 | [diff] [blame] | 26 | #include <linux/etherdevice.h> |
Tian Yang | 45f39c8 | 2020-10-06 14:07:47 -0700 | [diff] [blame] | 27 | #include <linux/version.h> |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 28 | #include <linux/lockdep.h> |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 29 | |
Ratheesh Kannoth | 24fb1db | 2021-10-20 07:28:06 +0530 | [diff] [blame] | 30 | #include "sfe_debug.h" |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 31 | #include "sfe_api.h" |
Dave Hudson | dcd08fb | 2013-11-22 09:25:16 -0600 | [diff] [blame] | 32 | #include "sfe.h" |
Ratheesh Kannoth | 24fb1db | 2021-10-20 07:28:06 +0530 | [diff] [blame] | 33 | #include "sfe_flow_cookie.h" |
| 34 | #include "sfe_ipv4.h" |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 35 | #include "sfe_ipv4_udp.h" |
| 36 | #include "sfe_ipv4_tcp.h" |
| 37 | #include "sfe_ipv4_icmp.h" |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 38 | |
| 39 | static char *sfe_ipv4_exception_events_string[SFE_IPV4_EXCEPTION_EVENT_LAST] = { |
| 40 | "UDP_HEADER_INCOMPLETE", |
| 41 | "UDP_NO_CONNECTION", |
| 42 | "UDP_IP_OPTIONS_OR_INITIAL_FRAGMENT", |
| 43 | "UDP_SMALL_TTL", |
| 44 | "UDP_NEEDS_FRAGMENTATION", |
| 45 | "TCP_HEADER_INCOMPLETE", |
| 46 | "TCP_NO_CONNECTION_SLOW_FLAGS", |
| 47 | "TCP_NO_CONNECTION_FAST_FLAGS", |
| 48 | "TCP_IP_OPTIONS_OR_INITIAL_FRAGMENT", |
| 49 | "TCP_SMALL_TTL", |
| 50 | "TCP_NEEDS_FRAGMENTATION", |
| 51 | "TCP_FLAGS", |
| 52 | "TCP_SEQ_EXCEEDS_RIGHT_EDGE", |
| 53 | "TCP_SMALL_DATA_OFFS", |
| 54 | "TCP_BAD_SACK", |
| 55 | "TCP_BIG_DATA_OFFS", |
| 56 | "TCP_SEQ_BEFORE_LEFT_EDGE", |
| 57 | "TCP_ACK_EXCEEDS_RIGHT_EDGE", |
| 58 | "TCP_ACK_BEFORE_LEFT_EDGE", |
| 59 | "ICMP_HEADER_INCOMPLETE", |
| 60 | "ICMP_UNHANDLED_TYPE", |
| 61 | "ICMP_IPV4_HEADER_INCOMPLETE", |
| 62 | "ICMP_IPV4_NON_V4", |
| 63 | "ICMP_IPV4_IP_OPTIONS_INCOMPLETE", |
| 64 | "ICMP_IPV4_UDP_HEADER_INCOMPLETE", |
| 65 | "ICMP_IPV4_TCP_HEADER_INCOMPLETE", |
| 66 | "ICMP_IPV4_UNHANDLED_PROTOCOL", |
| 67 | "ICMP_NO_CONNECTION", |
| 68 | "ICMP_FLUSHED_CONNECTION", |
| 69 | "HEADER_INCOMPLETE", |
Ratheesh Kannoth | 43d64f8 | 2021-10-20 08:23:29 +0530 | [diff] [blame] | 70 | "HEADER_CSUM_BAD", |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 71 | "BAD_TOTAL_LENGTH", |
| 72 | "NON_V4", |
| 73 | "NON_INITIAL_FRAGMENT", |
| 74 | "DATAGRAM_INCOMPLETE", |
| 75 | "IP_OPTIONS_INCOMPLETE", |
| 76 | "UNHANDLED_PROTOCOL" |
| 77 | }; |
| 78 | |
Xiaoping Fan | 6a1672f | 2016-08-17 19:58:12 -0700 | [diff] [blame] | 79 | static struct sfe_ipv4 __si; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 80 | |
| 81 | /* |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 82 | * sfe_ipv4_gen_ip_csum() |
| 83 | * Generate the IP checksum for an IPv4 header. |
| 84 | * |
| 85 | * Note that this function assumes that we have only 20 bytes of IP header. |
| 86 | */ |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 87 | u16 sfe_ipv4_gen_ip_csum(struct iphdr *iph) |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 88 | { |
Xiaoping Fan | 6a1672f | 2016-08-17 19:58:12 -0700 | [diff] [blame] | 89 | u32 sum; |
| 90 | u16 *i = (u16 *)iph; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 91 | |
| 92 | iph->check = 0; |
| 93 | |
| 94 | /* |
| 95 | * Generate the sum. |
| 96 | */ |
| 97 | sum = i[0] + i[1] + i[2] + i[3] + i[4] + i[5] + i[6] + i[7] + i[8] + i[9]; |
| 98 | |
| 99 | /* |
| 100 | * Fold it to ones-complement form. |
| 101 | */ |
| 102 | sum = (sum & 0xffff) + (sum >> 16); |
| 103 | sum = (sum & 0xffff) + (sum >> 16); |
| 104 | |
Xiaoping Fan | 6a1672f | 2016-08-17 19:58:12 -0700 | [diff] [blame] | 105 | return (u16)sum ^ 0xffff; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 106 | } |
| 107 | |
| 108 | /* |
| 109 | * sfe_ipv4_get_connection_match_hash() |
| 110 | * Generate the hash used in connection match lookups. |
| 111 | */ |
Xiaoping Fan | 6a1672f | 2016-08-17 19:58:12 -0700 | [diff] [blame] | 112 | static inline unsigned int sfe_ipv4_get_connection_match_hash(struct net_device *dev, u8 protocol, |
Dave Hudson | 87973cd | 2013-10-22 16:00:04 +0100 | [diff] [blame] | 113 | __be32 src_ip, __be16 src_port, |
| 114 | __be32 dest_ip, __be16 dest_port) |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 115 | { |
| 116 | size_t dev_addr = (size_t)dev; |
Xiaoping Fan | 6a1672f | 2016-08-17 19:58:12 -0700 | [diff] [blame] | 117 | u32 hash = ((u32)dev_addr) ^ ntohl(src_ip ^ dest_ip) ^ protocol ^ ntohs(src_port ^ dest_port); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 118 | return ((hash >> SFE_IPV4_CONNECTION_HASH_SHIFT) ^ hash) & SFE_IPV4_CONNECTION_HASH_MASK; |
| 119 | } |
| 120 | |
| 121 | /* |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 122 | * sfe_ipv4_find_connection_match_rcu() |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 123 | * Get the IPv4 flow match info that corresponds to a particular 5-tuple. |
| 124 | * |
| 125 | * On entry we must be holding the lock that protects the hash table. |
| 126 | */ |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 127 | struct sfe_ipv4_connection_match * |
| 128 | sfe_ipv4_find_connection_match_rcu(struct sfe_ipv4 *si, struct net_device *dev, u8 protocol, |
Dave Hudson | 87973cd | 2013-10-22 16:00:04 +0100 | [diff] [blame] | 129 | __be32 src_ip, __be16 src_port, |
| 130 | __be32 dest_ip, __be16 dest_port) |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 131 | { |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 132 | struct sfe_ipv4_connection_match *cm = NULL; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 133 | unsigned int conn_match_idx; |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 134 | struct hlist_head *lhead; |
| 135 | |
| 136 | WARN_ON_ONCE(!rcu_read_lock_held()); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 137 | |
| 138 | conn_match_idx = sfe_ipv4_get_connection_match_hash(dev, protocol, src_ip, src_port, dest_ip, dest_port); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 139 | |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 140 | lhead = &si->hlist_conn_match_hash_head[conn_match_idx]; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 141 | |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 142 | hlist_for_each_entry_rcu(cm, lhead, hnode) { |
| 143 | if (cm->match_src_port != src_port |
| 144 | || cm->match_dest_port != dest_port |
| 145 | || cm->match_src_ip != src_ip |
| 146 | || cm->match_dest_ip != dest_ip |
| 147 | || cm->match_protocol != protocol |
| 148 | || cm->match_dev != dev) { |
| 149 | continue; |
| 150 | } |
| 151 | |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 152 | this_cpu_inc(si->stats_pcpu->connection_match_hash_hits64); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 153 | |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 154 | break; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 155 | } |
| 156 | |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 157 | return cm; |
| 158 | } |
| 159 | |
| 160 | /* |
| 161 | * sfe_ipv4_connection_match_update_summary_stats() |
| 162 | * Update the summary stats for a connection match entry. |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 163 | * |
| 164 | * Stats are incremented atomically. So use atomic substraction to update summary |
| 165 | * stats. |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 166 | */ |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 167 | static inline void sfe_ipv4_connection_match_update_summary_stats(struct sfe_ipv4_connection_match *cm, |
| 168 | u32 *packets, u32 *bytes) |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 169 | { |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 170 | u32 packet_count, byte_count; |
| 171 | |
| 172 | packet_count = atomic_read(&cm->rx_packet_count); |
| 173 | cm->rx_packet_count64 += packet_count; |
| 174 | atomic_sub(packet_count, &cm->rx_packet_count); |
| 175 | |
| 176 | byte_count = atomic_read(&cm->rx_byte_count); |
| 177 | cm->rx_byte_count64 += byte_count; |
| 178 | atomic_sub(byte_count, &cm->rx_byte_count); |
| 179 | |
| 180 | *packets = packet_count; |
| 181 | *bytes = byte_count; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 182 | } |
| 183 | |
| 184 | /* |
| 185 | * sfe_ipv4_connection_match_compute_translations() |
| 186 | * Compute port and address translations for a connection match entry. |
| 187 | */ |
| 188 | static void sfe_ipv4_connection_match_compute_translations(struct sfe_ipv4_connection_match *cm) |
| 189 | { |
| 190 | /* |
| 191 | * Before we insert the entry look to see if this is tagged as doing address |
| 192 | * translations. If it is then work out the adjustment that we need to apply |
| 193 | * to the transport checksum. |
| 194 | */ |
| 195 | if (cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_XLATE_SRC) { |
| 196 | /* |
| 197 | * Precompute an incremental checksum adjustment so we can |
| 198 | * edit packets in this stream very quickly. The algorithm is from RFC1624. |
| 199 | */ |
Xiaoping Fan | 6a1672f | 2016-08-17 19:58:12 -0700 | [diff] [blame] | 200 | u16 src_ip_hi = cm->match_src_ip >> 16; |
| 201 | u16 src_ip_lo = cm->match_src_ip & 0xffff; |
| 202 | u32 xlate_src_ip = ~cm->xlate_src_ip; |
| 203 | u16 xlate_src_ip_hi = xlate_src_ip >> 16; |
| 204 | u16 xlate_src_ip_lo = xlate_src_ip & 0xffff; |
| 205 | u16 xlate_src_port = ~cm->xlate_src_port; |
| 206 | u32 adj; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 207 | |
| 208 | /* |
| 209 | * When we compute this fold it down to a 16-bit offset |
| 210 | * as that way we can avoid having to do a double |
| 211 | * folding of the twos-complement result because the |
| 212 | * addition of 2 16-bit values cannot cause a double |
| 213 | * wrap-around! |
| 214 | */ |
| 215 | adj = src_ip_hi + src_ip_lo + cm->match_src_port |
| 216 | + xlate_src_ip_hi + xlate_src_ip_lo + xlate_src_port; |
| 217 | adj = (adj & 0xffff) + (adj >> 16); |
| 218 | adj = (adj & 0xffff) + (adj >> 16); |
Xiaoping Fan | 6a1672f | 2016-08-17 19:58:12 -0700 | [diff] [blame] | 219 | cm->xlate_src_csum_adjustment = (u16)adj; |
Nicolas Costa | ac2979c | 2014-01-14 10:35:24 -0600 | [diff] [blame] | 220 | |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 221 | } |
| 222 | |
| 223 | if (cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_XLATE_DEST) { |
| 224 | /* |
| 225 | * Precompute an incremental checksum adjustment so we can |
| 226 | * edit packets in this stream very quickly. The algorithm is from RFC1624. |
| 227 | */ |
Xiaoping Fan | 6a1672f | 2016-08-17 19:58:12 -0700 | [diff] [blame] | 228 | u16 dest_ip_hi = cm->match_dest_ip >> 16; |
| 229 | u16 dest_ip_lo = cm->match_dest_ip & 0xffff; |
| 230 | u32 xlate_dest_ip = ~cm->xlate_dest_ip; |
| 231 | u16 xlate_dest_ip_hi = xlate_dest_ip >> 16; |
| 232 | u16 xlate_dest_ip_lo = xlate_dest_ip & 0xffff; |
| 233 | u16 xlate_dest_port = ~cm->xlate_dest_port; |
| 234 | u32 adj; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 235 | |
| 236 | /* |
| 237 | * When we compute this fold it down to a 16-bit offset |
| 238 | * as that way we can avoid having to do a double |
| 239 | * folding of the twos-complement result because the |
| 240 | * addition of 2 16-bit values cannot cause a double |
| 241 | * wrap-around! |
| 242 | */ |
| 243 | adj = dest_ip_hi + dest_ip_lo + cm->match_dest_port |
| 244 | + xlate_dest_ip_hi + xlate_dest_ip_lo + xlate_dest_port; |
| 245 | adj = (adj & 0xffff) + (adj >> 16); |
| 246 | adj = (adj & 0xffff) + (adj >> 16); |
Xiaoping Fan | 6a1672f | 2016-08-17 19:58:12 -0700 | [diff] [blame] | 247 | cm->xlate_dest_csum_adjustment = (u16)adj; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 248 | } |
Xiaoping Fan | ad755af | 2015-04-01 16:58:46 -0700 | [diff] [blame] | 249 | |
| 250 | if (cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_XLATE_SRC) { |
Xiaoping Fan | 6a1672f | 2016-08-17 19:58:12 -0700 | [diff] [blame] | 251 | u32 adj = ~cm->match_src_ip + cm->xlate_src_ip; |
Xiaoping Fan | ad755af | 2015-04-01 16:58:46 -0700 | [diff] [blame] | 252 | if (adj < cm->xlate_src_ip) { |
| 253 | adj++; |
| 254 | } |
| 255 | |
| 256 | adj = (adj & 0xffff) + (adj >> 16); |
| 257 | adj = (adj & 0xffff) + (adj >> 16); |
Xiaoping Fan | 6a1672f | 2016-08-17 19:58:12 -0700 | [diff] [blame] | 258 | cm->xlate_src_partial_csum_adjustment = (u16)adj; |
Xiaoping Fan | ad755af | 2015-04-01 16:58:46 -0700 | [diff] [blame] | 259 | } |
| 260 | |
| 261 | if (cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_XLATE_DEST) { |
Xiaoping Fan | 6a1672f | 2016-08-17 19:58:12 -0700 | [diff] [blame] | 262 | u32 adj = ~cm->match_dest_ip + cm->xlate_dest_ip; |
Xiaoping Fan | ad755af | 2015-04-01 16:58:46 -0700 | [diff] [blame] | 263 | if (adj < cm->xlate_dest_ip) { |
| 264 | adj++; |
| 265 | } |
| 266 | |
| 267 | adj = (adj & 0xffff) + (adj >> 16); |
| 268 | adj = (adj & 0xffff) + (adj >> 16); |
Xiaoping Fan | 6a1672f | 2016-08-17 19:58:12 -0700 | [diff] [blame] | 269 | cm->xlate_dest_partial_csum_adjustment = (u16)adj; |
Xiaoping Fan | ad755af | 2015-04-01 16:58:46 -0700 | [diff] [blame] | 270 | } |
| 271 | |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 272 | } |
| 273 | |
| 274 | /* |
| 275 | * sfe_ipv4_update_summary_stats() |
| 276 | * Update the summary stats. |
| 277 | */ |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 278 | static void sfe_ipv4_update_summary_stats(struct sfe_ipv4 *si, struct sfe_ipv4_stats *stats) |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 279 | { |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 280 | int i = 0; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 281 | |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 282 | memset(stats, 0, sizeof(*stats)); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 283 | |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 284 | for_each_possible_cpu(i) { |
| 285 | const struct sfe_ipv4_stats *s = per_cpu_ptr(si->stats_pcpu, i); |
| 286 | |
| 287 | stats->connection_create_requests64 += s->connection_create_requests64; |
| 288 | stats->connection_create_collisions64 += s->connection_create_collisions64; |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 289 | stats->connection_create_failures64 += s->connection_create_failures64; |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 290 | stats->connection_destroy_requests64 += s->connection_destroy_requests64; |
| 291 | stats->connection_destroy_misses64 += s->connection_destroy_misses64; |
| 292 | stats->connection_match_hash_hits64 += s->connection_match_hash_hits64; |
| 293 | stats->connection_match_hash_reorders64 += s->connection_match_hash_reorders64; |
| 294 | stats->connection_flushes64 += s->connection_flushes64; |
| 295 | stats->packets_forwarded64 += s->packets_forwarded64; |
| 296 | stats->packets_not_forwarded64 += s->packets_not_forwarded64; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 297 | } |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 298 | |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 299 | } |
| 300 | |
| 301 | /* |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 302 | * sfe_ipv4_insert_connection_match() |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 303 | * Insert a connection match into the hash. |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 304 | */ |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 305 | static inline void sfe_ipv4_insert_connection_match(struct sfe_ipv4 *si, |
Xiaoping Fan | 6a1672f | 2016-08-17 19:58:12 -0700 | [diff] [blame] | 306 | struct sfe_ipv4_connection_match *cm) |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 307 | { |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 308 | unsigned int conn_match_idx |
| 309 | = sfe_ipv4_get_connection_match_hash(cm->match_dev, cm->match_protocol, |
| 310 | cm->match_src_ip, cm->match_src_port, |
| 311 | cm->match_dest_ip, cm->match_dest_port); |
Xiaoping Fan | 6a1672f | 2016-08-17 19:58:12 -0700 | [diff] [blame] | 312 | |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 313 | lockdep_assert_held(&si->lock); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 314 | |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 315 | hlist_add_head_rcu(&cm->hnode, &si->hlist_conn_match_hash_head[conn_match_idx]); |
Xiaoping Fan | d1dc7b2 | 2015-01-23 00:43:56 -0800 | [diff] [blame] | 316 | #ifdef CONFIG_NF_FLOW_COOKIE |
Xiaoping Fan | 640faf4 | 2015-08-28 15:50:55 -0700 | [diff] [blame] | 317 | if (!si->flow_cookie_enable) |
| 318 | return; |
| 319 | |
Xiaoping Fan | d1dc7b2 | 2015-01-23 00:43:56 -0800 | [diff] [blame] | 320 | /* |
| 321 | * Configure hardware to put a flow cookie in packet of this flow, |
| 322 | * then we can accelerate the lookup process when we received this packet. |
| 323 | */ |
| 324 | for (conn_match_idx = 1; conn_match_idx < SFE_FLOW_COOKIE_SIZE; conn_match_idx++) { |
| 325 | struct sfe_flow_cookie_entry *entry = &si->sfe_flow_cookie_table[conn_match_idx]; |
| 326 | |
| 327 | if ((NULL == entry->match) && time_is_before_jiffies(entry->last_clean_time + HZ)) { |
| 328 | flow_cookie_set_func_t func; |
| 329 | |
| 330 | rcu_read_lock(); |
| 331 | func = rcu_dereference(si->flow_cookie_set_func); |
| 332 | if (func) { |
Xiaoping Fan | 5917642 | 2015-05-22 15:58:10 -0700 | [diff] [blame] | 333 | if (!func(cm->match_protocol, cm->match_src_ip, cm->match_src_port, |
Xiaoping Fan | d1dc7b2 | 2015-01-23 00:43:56 -0800 | [diff] [blame] | 334 | cm->match_dest_ip, cm->match_dest_port, conn_match_idx)) { |
| 335 | entry->match = cm; |
| 336 | cm->flow_cookie = conn_match_idx; |
| 337 | } |
| 338 | } |
| 339 | rcu_read_unlock(); |
| 340 | |
| 341 | break; |
| 342 | } |
| 343 | } |
| 344 | #endif |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 345 | } |
| 346 | |
| 347 | /* |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 348 | * sfe_ipv4_remove_connection_match() |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 349 | * Remove a connection match object from the hash. |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 350 | */ |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 351 | static inline void sfe_ipv4_remove_connection_match(struct sfe_ipv4 *si, struct sfe_ipv4_connection_match *cm) |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 352 | { |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 353 | |
| 354 | lockdep_assert_held(&si->lock); |
| 355 | |
Xiaoping Fan | d1dc7b2 | 2015-01-23 00:43:56 -0800 | [diff] [blame] | 356 | #ifdef CONFIG_NF_FLOW_COOKIE |
Xiaoping Fan | 640faf4 | 2015-08-28 15:50:55 -0700 | [diff] [blame] | 357 | if (si->flow_cookie_enable) { |
| 358 | /* |
| 359 | * Tell hardware that we no longer need a flow cookie in packet of this flow |
| 360 | */ |
| 361 | unsigned int conn_match_idx; |
Xiaoping Fan | d1dc7b2 | 2015-01-23 00:43:56 -0800 | [diff] [blame] | 362 | |
Xiaoping Fan | 640faf4 | 2015-08-28 15:50:55 -0700 | [diff] [blame] | 363 | for (conn_match_idx = 1; conn_match_idx < SFE_FLOW_COOKIE_SIZE; conn_match_idx++) { |
| 364 | struct sfe_flow_cookie_entry *entry = &si->sfe_flow_cookie_table[conn_match_idx]; |
Xiaoping Fan | d1dc7b2 | 2015-01-23 00:43:56 -0800 | [diff] [blame] | 365 | |
Xiaoping Fan | 640faf4 | 2015-08-28 15:50:55 -0700 | [diff] [blame] | 366 | if (cm == entry->match) { |
| 367 | flow_cookie_set_func_t func; |
Xiaoping Fan | d1dc7b2 | 2015-01-23 00:43:56 -0800 | [diff] [blame] | 368 | |
Xiaoping Fan | 640faf4 | 2015-08-28 15:50:55 -0700 | [diff] [blame] | 369 | rcu_read_lock(); |
| 370 | func = rcu_dereference(si->flow_cookie_set_func); |
| 371 | if (func) { |
| 372 | func(cm->match_protocol, cm->match_src_ip, cm->match_src_port, |
| 373 | cm->match_dest_ip, cm->match_dest_port, 0); |
| 374 | } |
| 375 | rcu_read_unlock(); |
| 376 | |
| 377 | cm->flow_cookie = 0; |
| 378 | entry->match = NULL; |
| 379 | entry->last_clean_time = jiffies; |
| 380 | break; |
Xiaoping Fan | d1dc7b2 | 2015-01-23 00:43:56 -0800 | [diff] [blame] | 381 | } |
Xiaoping Fan | d1dc7b2 | 2015-01-23 00:43:56 -0800 | [diff] [blame] | 382 | } |
| 383 | } |
| 384 | #endif |
| 385 | |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 386 | hlist_del_init_rcu(&cm->hnode); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 387 | |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 388 | } |
| 389 | |
| 390 | /* |
| 391 | * sfe_ipv4_get_connection_hash() |
| 392 | * Generate the hash used in connection lookups. |
| 393 | */ |
Xiaoping Fan | 6a1672f | 2016-08-17 19:58:12 -0700 | [diff] [blame] | 394 | static inline unsigned int sfe_ipv4_get_connection_hash(u8 protocol, __be32 src_ip, __be16 src_port, |
Dave Hudson | 87973cd | 2013-10-22 16:00:04 +0100 | [diff] [blame] | 395 | __be32 dest_ip, __be16 dest_port) |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 396 | { |
Xiaoping Fan | 6a1672f | 2016-08-17 19:58:12 -0700 | [diff] [blame] | 397 | u32 hash = ntohl(src_ip ^ dest_ip) ^ protocol ^ ntohs(src_port ^ dest_port); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 398 | return ((hash >> SFE_IPV4_CONNECTION_HASH_SHIFT) ^ hash) & SFE_IPV4_CONNECTION_HASH_MASK; |
| 399 | } |
| 400 | |
| 401 | /* |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 402 | * sfe_ipv4_find_connection() |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 403 | * Get the IPv4 connection info that corresponds to a particular 5-tuple. |
| 404 | * |
| 405 | * On entry we must be holding the lock that protects the hash table. |
| 406 | */ |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 407 | static inline struct sfe_ipv4_connection *sfe_ipv4_find_connection(struct sfe_ipv4 *si, u32 protocol, |
Dave Hudson | 87973cd | 2013-10-22 16:00:04 +0100 | [diff] [blame] | 408 | __be32 src_ip, __be16 src_port, |
| 409 | __be32 dest_ip, __be16 dest_port) |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 410 | { |
| 411 | struct sfe_ipv4_connection *c; |
| 412 | unsigned int conn_idx = sfe_ipv4_get_connection_hash(protocol, src_ip, src_port, dest_ip, dest_port); |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 413 | |
| 414 | lockdep_assert_held(&si->lock); |
| 415 | |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 416 | c = si->conn_hash[conn_idx]; |
| 417 | |
| 418 | /* |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 419 | * Will need connection entry for next create/destroy metadata, |
| 420 | * So no need to re-order entry for these requests |
| 421 | */ |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 422 | while (c) { |
| 423 | if ((c->src_port == src_port) |
| 424 | && (c->dest_port == dest_port) |
| 425 | && (c->src_ip == src_ip) |
| 426 | && (c->dest_ip == dest_ip) |
| 427 | && (c->protocol == protocol)) { |
| 428 | return c; |
| 429 | } |
| 430 | |
| 431 | c = c->next; |
| 432 | } |
| 433 | |
| 434 | return NULL; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 435 | } |
| 436 | |
| 437 | /* |
Matthew McClintock | be7b47d | 2013-11-27 13:26:23 -0600 | [diff] [blame] | 438 | * sfe_ipv4_mark_rule() |
| 439 | * Updates the mark for a current offloaded connection |
| 440 | * |
| 441 | * Will take hash lock upon entry |
| 442 | */ |
Xiaoping Fan | d44a5b4 | 2015-05-26 17:37:37 -0700 | [diff] [blame] | 443 | void sfe_ipv4_mark_rule(struct sfe_connection_mark *mark) |
Matthew McClintock | be7b47d | 2013-11-27 13:26:23 -0600 | [diff] [blame] | 444 | { |
| 445 | struct sfe_ipv4 *si = &__si; |
| 446 | struct sfe_ipv4_connection *c; |
Matthew McClintock | db5ac51 | 2014-01-16 17:01:40 -0600 | [diff] [blame] | 447 | |
Xiaoping Fan | 3c423e3 | 2015-07-03 03:09:29 -0700 | [diff] [blame] | 448 | spin_lock_bh(&si->lock); |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 449 | c = sfe_ipv4_find_connection(si, mark->protocol, |
Xiaoping Fan | d44a5b4 | 2015-05-26 17:37:37 -0700 | [diff] [blame] | 450 | mark->src_ip.ip, mark->src_port, |
| 451 | mark->dest_ip.ip, mark->dest_port); |
Matthew McClintock | be7b47d | 2013-11-27 13:26:23 -0600 | [diff] [blame] | 452 | if (c) { |
Nicolas Costa | f53d6fe | 2014-01-13 16:03:46 -0600 | [diff] [blame] | 453 | WARN_ON((0 != c->mark) && (0 == mark->mark)); |
Matthew McClintock | be7b47d | 2013-11-27 13:26:23 -0600 | [diff] [blame] | 454 | c->mark = mark->mark; |
| 455 | } |
Xiaoping Fan | 3c423e3 | 2015-07-03 03:09:29 -0700 | [diff] [blame] | 456 | spin_unlock_bh(&si->lock); |
Xiaoping Fan | 6a1672f | 2016-08-17 19:58:12 -0700 | [diff] [blame] | 457 | |
| 458 | if (c) { |
| 459 | DEBUG_TRACE("Matching connection found for mark, " |
| 460 | "setting from %08x to %08x\n", |
| 461 | c->mark, mark->mark); |
| 462 | } |
Matthew McClintock | be7b47d | 2013-11-27 13:26:23 -0600 | [diff] [blame] | 463 | } |
| 464 | |
| 465 | /* |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 466 | * sfe_ipv4_insert_connection() |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 467 | * Insert a connection into the hash. |
| 468 | * |
| 469 | * On entry we must be holding the lock that protects the hash table. |
| 470 | */ |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 471 | static void sfe_ipv4_insert_connection(struct sfe_ipv4 *si, struct sfe_ipv4_connection *c) |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 472 | { |
| 473 | struct sfe_ipv4_connection **hash_head; |
| 474 | struct sfe_ipv4_connection *prev_head; |
| 475 | unsigned int conn_idx; |
| 476 | |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 477 | lockdep_assert_held(&si->lock); |
| 478 | |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 479 | /* |
| 480 | * Insert entry into the connection hash. |
| 481 | */ |
| 482 | conn_idx = sfe_ipv4_get_connection_hash(c->protocol, c->src_ip, c->src_port, |
| 483 | c->dest_ip, c->dest_port); |
| 484 | hash_head = &si->conn_hash[conn_idx]; |
| 485 | prev_head = *hash_head; |
| 486 | c->prev = NULL; |
| 487 | if (prev_head) { |
| 488 | prev_head->prev = c; |
| 489 | } |
| 490 | |
| 491 | c->next = prev_head; |
| 492 | *hash_head = c; |
| 493 | |
| 494 | /* |
| 495 | * Insert entry into the "all connections" list. |
| 496 | */ |
| 497 | if (si->all_connections_tail) { |
| 498 | c->all_connections_prev = si->all_connections_tail; |
| 499 | si->all_connections_tail->all_connections_next = c; |
| 500 | } else { |
| 501 | c->all_connections_prev = NULL; |
| 502 | si->all_connections_head = c; |
| 503 | } |
| 504 | |
| 505 | si->all_connections_tail = c; |
| 506 | c->all_connections_next = NULL; |
| 507 | si->num_connections++; |
| 508 | |
| 509 | /* |
| 510 | * Insert the connection match objects too. |
| 511 | */ |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 512 | sfe_ipv4_insert_connection_match(si, c->original_match); |
| 513 | sfe_ipv4_insert_connection_match(si, c->reply_match); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 514 | } |
| 515 | |
| 516 | /* |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 517 | * sfe_ipv4_remove_connection() |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 518 | * Remove a sfe_ipv4_connection object from the hash. |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 519 | */ |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 520 | bool sfe_ipv4_remove_connection(struct sfe_ipv4 *si, struct sfe_ipv4_connection *c) |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 521 | { |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 522 | lockdep_assert_held(&si->lock); |
| 523 | |
| 524 | if (c->removed) { |
| 525 | DEBUG_ERROR("%px: Connection has been removed already\n", c); |
| 526 | return false; |
| 527 | } |
| 528 | |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 529 | /* |
| 530 | * Remove the connection match objects. |
| 531 | */ |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 532 | sfe_ipv4_remove_connection_match(si, c->reply_match); |
| 533 | sfe_ipv4_remove_connection_match(si, c->original_match); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 534 | |
| 535 | /* |
| 536 | * Unlink the connection. |
| 537 | */ |
| 538 | if (c->prev) { |
| 539 | c->prev->next = c->next; |
| 540 | } else { |
| 541 | unsigned int conn_idx = sfe_ipv4_get_connection_hash(c->protocol, c->src_ip, c->src_port, |
| 542 | c->dest_ip, c->dest_port); |
| 543 | si->conn_hash[conn_idx] = c->next; |
| 544 | } |
| 545 | |
| 546 | if (c->next) { |
| 547 | c->next->prev = c->prev; |
| 548 | } |
Xiaoping Fan | 3458647 | 2015-07-03 02:20:35 -0700 | [diff] [blame] | 549 | |
| 550 | /* |
| 551 | * Unlink connection from all_connections list |
| 552 | */ |
| 553 | if (c->all_connections_prev) { |
| 554 | c->all_connections_prev->all_connections_next = c->all_connections_next; |
| 555 | } else { |
| 556 | si->all_connections_head = c->all_connections_next; |
| 557 | } |
| 558 | |
| 559 | if (c->all_connections_next) { |
| 560 | c->all_connections_next->all_connections_prev = c->all_connections_prev; |
| 561 | } else { |
| 562 | si->all_connections_tail = c->all_connections_prev; |
| 563 | } |
| 564 | |
Ken Zhu | dc42367 | 2021-09-02 18:27:01 -0700 | [diff] [blame] | 565 | /* |
| 566 | * If I am the next sync connection, move the sync to my next or head. |
| 567 | */ |
| 568 | if (unlikely(si->wc_next == c)) { |
| 569 | si->wc_next = c->all_connections_next; |
| 570 | } |
| 571 | |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 572 | c->removed = true; |
Xiaoping Fan | 3458647 | 2015-07-03 02:20:35 -0700 | [diff] [blame] | 573 | si->num_connections--; |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 574 | return true; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 575 | } |
| 576 | |
| 577 | /* |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 578 | * sfe_ipv4_gen_sync_connection() |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 579 | * Sync a connection. |
| 580 | * |
| 581 | * On entry to this function we expect that the lock for the connection is either |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 582 | * already held (while called from sfe_ipv4_periodic_sync() or isn't required |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 583 | * (while called from sfe_ipv4_flush_connection()) |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 584 | */ |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 585 | static void sfe_ipv4_gen_sync_connection(struct sfe_ipv4 *si, struct sfe_ipv4_connection *c, |
Xiaoping Fan | 99cb4c1 | 2015-08-21 19:07:32 -0700 | [diff] [blame] | 586 | struct sfe_connection_sync *sis, sfe_sync_reason_t reason, |
Xiaoping Fan | 6a1672f | 2016-08-17 19:58:12 -0700 | [diff] [blame] | 587 | u64 now_jiffies) |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 588 | { |
| 589 | struct sfe_ipv4_connection_match *original_cm; |
| 590 | struct sfe_ipv4_connection_match *reply_cm; |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 591 | u32 packet_count, byte_count; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 592 | |
| 593 | /* |
| 594 | * Fill in the update message. |
| 595 | */ |
Xiaoping Fan | d44a5b4 | 2015-05-26 17:37:37 -0700 | [diff] [blame] | 596 | sis->is_v6 = 0; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 597 | sis->protocol = c->protocol; |
Xiaoping Fan | d44a5b4 | 2015-05-26 17:37:37 -0700 | [diff] [blame] | 598 | sis->src_ip.ip = c->src_ip; |
Xiaoping Fan | 99cb4c1 | 2015-08-21 19:07:32 -0700 | [diff] [blame] | 599 | sis->src_ip_xlate.ip = c->src_ip_xlate; |
Xiaoping Fan | d44a5b4 | 2015-05-26 17:37:37 -0700 | [diff] [blame] | 600 | sis->dest_ip.ip = c->dest_ip; |
Xiaoping Fan | 99cb4c1 | 2015-08-21 19:07:32 -0700 | [diff] [blame] | 601 | sis->dest_ip_xlate.ip = c->dest_ip_xlate; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 602 | sis->src_port = c->src_port; |
Xiaoping Fan | 99cb4c1 | 2015-08-21 19:07:32 -0700 | [diff] [blame] | 603 | sis->src_port_xlate = c->src_port_xlate; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 604 | sis->dest_port = c->dest_port; |
Xiaoping Fan | 99cb4c1 | 2015-08-21 19:07:32 -0700 | [diff] [blame] | 605 | sis->dest_port_xlate = c->dest_port_xlate; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 606 | |
| 607 | original_cm = c->original_match; |
| 608 | reply_cm = c->reply_match; |
| 609 | sis->src_td_max_window = original_cm->protocol_state.tcp.max_win; |
| 610 | sis->src_td_end = original_cm->protocol_state.tcp.end; |
| 611 | sis->src_td_max_end = original_cm->protocol_state.tcp.max_end; |
| 612 | sis->dest_td_max_window = reply_cm->protocol_state.tcp.max_win; |
| 613 | sis->dest_td_end = reply_cm->protocol_state.tcp.end; |
| 614 | sis->dest_td_max_end = reply_cm->protocol_state.tcp.max_end; |
| 615 | |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 616 | sfe_ipv4_connection_match_update_summary_stats(original_cm, &packet_count, &byte_count); |
| 617 | sis->src_new_packet_count = packet_count; |
| 618 | sis->src_new_byte_count = byte_count; |
Matthew McClintock | d0cdb80 | 2014-02-24 16:30:35 -0600 | [diff] [blame] | 619 | |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 620 | sfe_ipv4_connection_match_update_summary_stats(reply_cm, &packet_count, &byte_count); |
| 621 | sis->dest_new_packet_count = packet_count; |
| 622 | sis->dest_new_byte_count = byte_count; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 623 | |
Matthew McClintock | d0cdb80 | 2014-02-24 16:30:35 -0600 | [diff] [blame] | 624 | sis->src_dev = original_cm->match_dev; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 625 | sis->src_packet_count = original_cm->rx_packet_count64; |
| 626 | sis->src_byte_count = original_cm->rx_byte_count64; |
Matthew McClintock | d0cdb80 | 2014-02-24 16:30:35 -0600 | [diff] [blame] | 627 | |
| 628 | sis->dest_dev = reply_cm->match_dev; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 629 | sis->dest_packet_count = reply_cm->rx_packet_count64; |
| 630 | sis->dest_byte_count = reply_cm->rx_byte_count64; |
| 631 | |
Xiaoping Fan | 99cb4c1 | 2015-08-21 19:07:32 -0700 | [diff] [blame] | 632 | sis->reason = reason; |
| 633 | |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 634 | /* |
| 635 | * Get the time increment since our last sync. |
| 636 | */ |
| 637 | sis->delta_jiffies = now_jiffies - c->last_sync_jiffies; |
| 638 | c->last_sync_jiffies = now_jiffies; |
| 639 | } |
| 640 | |
| 641 | /* |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 642 | * sfe_ipv4_free_connection_rcu() |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 643 | * Called at RCU qs state to free the connection object. |
| 644 | */ |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 645 | static void sfe_ipv4_free_connection_rcu(struct rcu_head *head) |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 646 | { |
| 647 | struct sfe_ipv4_connection *c; |
| 648 | |
| 649 | /* |
| 650 | * We dont need spin lock as the connection is already removed from link list |
| 651 | */ |
| 652 | c = container_of(head, struct sfe_ipv4_connection, rcu); |
| 653 | |
| 654 | BUG_ON(!c->removed); |
| 655 | |
| 656 | DEBUG_TRACE("%px: connecton has been deleted\n", c); |
| 657 | |
| 658 | /* |
| 659 | * Release our hold of the source and dest devices and free the memory |
| 660 | * for our connection objects. |
| 661 | */ |
| 662 | dev_put(c->original_dev); |
| 663 | dev_put(c->reply_dev); |
| 664 | kfree(c->original_match); |
| 665 | kfree(c->reply_match); |
| 666 | kfree(c); |
| 667 | } |
| 668 | |
| 669 | /* |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 670 | * sfe_ipv4_flush_connection() |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 671 | * Flush a connection and free all associated resources. |
| 672 | * |
| 673 | * We need to be called with bottom halves disabled locally as we need to acquire |
| 674 | * the connection hash lock and release it again. In general we're actually called |
| 675 | * from within a BH and so we're fine, but we're also called when connections are |
| 676 | * torn down. |
| 677 | */ |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 678 | void sfe_ipv4_flush_connection(struct sfe_ipv4 *si, |
Xiaoping Fan | 6a1672f | 2016-08-17 19:58:12 -0700 | [diff] [blame] | 679 | struct sfe_ipv4_connection *c, |
| 680 | sfe_sync_reason_t reason) |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 681 | { |
Xiaoping Fan | 6a1672f | 2016-08-17 19:58:12 -0700 | [diff] [blame] | 682 | u64 now_jiffies; |
Xiaoping Fan | d44a5b4 | 2015-05-26 17:37:37 -0700 | [diff] [blame] | 683 | sfe_sync_rule_callback_t sync_rule_callback; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 684 | |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 685 | BUG_ON(!c->removed); |
| 686 | |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 687 | this_cpu_inc(si->stats_pcpu->connection_flushes64); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 688 | |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 689 | rcu_read_lock(); |
| 690 | sync_rule_callback = rcu_dereference(si->sync_rule_callback); |
| 691 | |
| 692 | /* |
| 693 | * Generate a sync message and then sync. |
| 694 | */ |
Dave Hudson | dcd08fb | 2013-11-22 09:25:16 -0600 | [diff] [blame] | 695 | if (sync_rule_callback) { |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 696 | struct sfe_connection_sync sis; |
Dave Hudson | dcd08fb | 2013-11-22 09:25:16 -0600 | [diff] [blame] | 697 | now_jiffies = get_jiffies_64(); |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 698 | sfe_ipv4_gen_sync_connection(si, c, &sis, reason, now_jiffies); |
Dave Hudson | dcd08fb | 2013-11-22 09:25:16 -0600 | [diff] [blame] | 699 | sync_rule_callback(&sis); |
| 700 | } |
| 701 | |
| 702 | rcu_read_unlock(); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 703 | |
| 704 | /* |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 705 | * Release our hold of the source and dest devices and free the memory |
| 706 | * for our connection objects. |
| 707 | */ |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 708 | call_rcu(&c->rcu, sfe_ipv4_free_connection_rcu); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 709 | } |
| 710 | |
| 711 | /* |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 712 | * sfe_ipv4_exception_stats_inc() |
| 713 | * Increment exception stats. |
| 714 | */ |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 715 | void sfe_ipv4_exception_stats_inc(struct sfe_ipv4 *si, enum sfe_ipv4_exception_events reason) |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 716 | { |
| 717 | struct sfe_ipv4_stats *stats = this_cpu_ptr(si->stats_pcpu); |
| 718 | stats->exception_events64[reason]++; |
| 719 | stats->packets_not_forwarded64++; |
| 720 | } |
| 721 | |
| 722 | /* |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 723 | * sfe_ipv4_recv() |
Matthew McClintock | a8ad796 | 2014-01-16 16:49:30 -0600 | [diff] [blame] | 724 | * Handle packet receives and forwaring. |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 725 | * |
| 726 | * Returns 1 if the packet is forwarded or 0 if it isn't. |
| 727 | */ |
Dave Hudson | dcd08fb | 2013-11-22 09:25:16 -0600 | [diff] [blame] | 728 | int sfe_ipv4_recv(struct net_device *dev, struct sk_buff *skb) |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 729 | { |
| 730 | struct sfe_ipv4 *si = &__si; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 731 | unsigned int len; |
| 732 | unsigned int tot_len; |
| 733 | unsigned int frag_off; |
| 734 | unsigned int ihl; |
| 735 | bool flush_on_find; |
| 736 | bool ip_options; |
Ratheesh Kannoth | 741f799 | 2021-10-20 07:39:52 +0530 | [diff] [blame] | 737 | struct iphdr *iph; |
Xiaoping Fan | 6a1672f | 2016-08-17 19:58:12 -0700 | [diff] [blame] | 738 | u32 protocol; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 739 | |
| 740 | /* |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 741 | * Check that we have space for an IP header here. |
| 742 | */ |
| 743 | len = skb->len; |
Ratheesh Kannoth | 741f799 | 2021-10-20 07:39:52 +0530 | [diff] [blame] | 744 | if (unlikely(!pskb_may_pull(skb, sizeof(struct iphdr)))) { |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 745 | sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_HEADER_INCOMPLETE); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 746 | DEBUG_TRACE("len: %u is too short\n", len); |
| 747 | return 0; |
| 748 | } |
| 749 | |
| 750 | /* |
Ratheesh Kannoth | a3cf0e0 | 2021-12-09 09:44:10 +0530 | [diff] [blame] | 751 | * Validate ip csum if necessary. If ip_summed is set to CHECKSUM_UNNECESSARY, it is assumed |
| 752 | * that the L3 checksum is validated by the Rx interface or the tunnel interface that has |
| 753 | * generated the packet. |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 754 | */ |
Ratheesh Kannoth | 741f799 | 2021-10-20 07:39:52 +0530 | [diff] [blame] | 755 | iph = (struct iphdr *)skb->data; |
Ratheesh Kannoth | 43d64f8 | 2021-10-20 08:23:29 +0530 | [diff] [blame] | 756 | if (unlikely(skb->ip_summed != CHECKSUM_UNNECESSARY) && (ip_fast_csum((u8 *)iph, iph->ihl))) { |
| 757 | sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_HEADER_CSUM_BAD); |
| 758 | |
| 759 | DEBUG_TRACE("Bad IPv4 header csum: 0x%x\n", iph->check); |
| 760 | return 0; |
| 761 | } |
| 762 | |
| 763 | /* |
| 764 | * Check that our "total length" is large enough for an IP header. |
| 765 | */ |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 766 | tot_len = ntohs(iph->tot_len); |
Ratheesh Kannoth | 741f799 | 2021-10-20 07:39:52 +0530 | [diff] [blame] | 767 | if (unlikely(tot_len < sizeof(struct iphdr))) { |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 768 | |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 769 | sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_BAD_TOTAL_LENGTH); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 770 | DEBUG_TRACE("tot_len: %u is too short\n", tot_len); |
| 771 | return 0; |
| 772 | } |
| 773 | |
| 774 | /* |
| 775 | * Is our IP version wrong? |
| 776 | */ |
| 777 | if (unlikely(iph->version != 4)) { |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 778 | sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_NON_V4); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 779 | DEBUG_TRACE("IP version: %u\n", iph->version); |
| 780 | return 0; |
| 781 | } |
| 782 | |
| 783 | /* |
| 784 | * Does our datagram fit inside the skb? |
| 785 | */ |
| 786 | if (unlikely(tot_len > len)) { |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 787 | |
| 788 | DEBUG_TRACE("tot_len: %u, exceeds len: %u\n", tot_len, len); |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 789 | sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_DATAGRAM_INCOMPLETE); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 790 | return 0; |
| 791 | } |
| 792 | |
| 793 | /* |
| 794 | * Do we have a non-initial fragment? |
Nicolas Costa | ac2979c | 2014-01-14 10:35:24 -0600 | [diff] [blame] | 795 | */ |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 796 | frag_off = ntohs(iph->frag_off); |
| 797 | if (unlikely(frag_off & IP_OFFSET)) { |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 798 | sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_NON_INITIAL_FRAGMENT); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 799 | DEBUG_TRACE("non-initial fragment\n"); |
| 800 | return 0; |
| 801 | } |
| 802 | |
| 803 | /* |
| 804 | * If we have a (first) fragment then mark it to cause any connection to flush. |
| 805 | */ |
| 806 | flush_on_find = unlikely(frag_off & IP_MF) ? true : false; |
| 807 | |
| 808 | /* |
| 809 | * Do we have any IP options? That's definite a slow path! If we do have IP |
| 810 | * options we need to recheck our header size. |
| 811 | */ |
| 812 | ihl = iph->ihl << 2; |
Ratheesh Kannoth | 741f799 | 2021-10-20 07:39:52 +0530 | [diff] [blame] | 813 | ip_options = unlikely(ihl != sizeof(struct iphdr)) ? true : false; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 814 | if (unlikely(ip_options)) { |
| 815 | if (unlikely(len < ihl)) { |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 816 | |
| 817 | DEBUG_TRACE("len: %u is too short for header of size: %u\n", len, ihl); |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 818 | sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_IP_OPTIONS_INCOMPLETE); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 819 | return 0; |
| 820 | } |
| 821 | |
| 822 | flush_on_find = true; |
| 823 | } |
| 824 | |
| 825 | protocol = iph->protocol; |
| 826 | if (IPPROTO_UDP == protocol) { |
| 827 | return sfe_ipv4_recv_udp(si, skb, dev, len, iph, ihl, flush_on_find); |
| 828 | } |
| 829 | |
| 830 | if (IPPROTO_TCP == protocol) { |
| 831 | return sfe_ipv4_recv_tcp(si, skb, dev, len, iph, ihl, flush_on_find); |
| 832 | } |
| 833 | |
| 834 | if (IPPROTO_ICMP == protocol) { |
| 835 | return sfe_ipv4_recv_icmp(si, skb, dev, len, iph, ihl); |
| 836 | } |
| 837 | |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 838 | sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_UNHANDLED_PROTOCOL); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 839 | |
| 840 | DEBUG_TRACE("not UDP, TCP or ICMP: %u\n", protocol); |
| 841 | return 0; |
| 842 | } |
| 843 | |
Nicolas Costa | 436926b | 2014-01-14 10:36:22 -0600 | [diff] [blame] | 844 | static void |
| 845 | sfe_ipv4_update_tcp_state(struct sfe_ipv4_connection *c, |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 846 | struct sfe_ipv4_rule_create_msg *msg) |
Nicolas Costa | 436926b | 2014-01-14 10:36:22 -0600 | [diff] [blame] | 847 | { |
| 848 | struct sfe_ipv4_connection_match *orig_cm; |
| 849 | struct sfe_ipv4_connection_match *repl_cm; |
| 850 | struct sfe_ipv4_tcp_connection_match *orig_tcp; |
| 851 | struct sfe_ipv4_tcp_connection_match *repl_tcp; |
| 852 | |
| 853 | orig_cm = c->original_match; |
| 854 | repl_cm = c->reply_match; |
| 855 | orig_tcp = &orig_cm->protocol_state.tcp; |
| 856 | repl_tcp = &repl_cm->protocol_state.tcp; |
| 857 | |
| 858 | /* update orig */ |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 859 | if (orig_tcp->max_win < msg->tcp_rule.flow_max_window) { |
| 860 | orig_tcp->max_win = msg->tcp_rule.flow_max_window; |
Nicolas Costa | 436926b | 2014-01-14 10:36:22 -0600 | [diff] [blame] | 861 | } |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 862 | if ((s32)(orig_tcp->end - msg->tcp_rule.flow_end) < 0) { |
| 863 | orig_tcp->end = msg->tcp_rule.flow_end; |
Nicolas Costa | 436926b | 2014-01-14 10:36:22 -0600 | [diff] [blame] | 864 | } |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 865 | if ((s32)(orig_tcp->max_end - msg->tcp_rule.flow_max_end) < 0) { |
| 866 | orig_tcp->max_end = msg->tcp_rule.flow_max_end; |
Nicolas Costa | 436926b | 2014-01-14 10:36:22 -0600 | [diff] [blame] | 867 | } |
| 868 | |
| 869 | /* update reply */ |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 870 | if (repl_tcp->max_win < msg->tcp_rule.return_max_window) { |
| 871 | repl_tcp->max_win = msg->tcp_rule.return_max_window; |
Nicolas Costa | 436926b | 2014-01-14 10:36:22 -0600 | [diff] [blame] | 872 | } |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 873 | if ((s32)(repl_tcp->end - msg->tcp_rule.return_end) < 0) { |
| 874 | repl_tcp->end = msg->tcp_rule.return_end; |
Nicolas Costa | 436926b | 2014-01-14 10:36:22 -0600 | [diff] [blame] | 875 | } |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 876 | if ((s32)(repl_tcp->max_end - msg->tcp_rule.return_max_end) < 0) { |
| 877 | repl_tcp->max_end = msg->tcp_rule.return_max_end; |
Nicolas Costa | 436926b | 2014-01-14 10:36:22 -0600 | [diff] [blame] | 878 | } |
| 879 | |
| 880 | /* update match flags */ |
| 881 | orig_cm->flags &= ~SFE_IPV4_CONNECTION_MATCH_FLAG_NO_SEQ_CHECK; |
| 882 | repl_cm->flags &= ~SFE_IPV4_CONNECTION_MATCH_FLAG_NO_SEQ_CHECK; |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 883 | if (msg->rule_flags & SFE_RULE_CREATE_FLAG_NO_SEQ_CHECK) { |
| 884 | |
Nicolas Costa | 436926b | 2014-01-14 10:36:22 -0600 | [diff] [blame] | 885 | orig_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_NO_SEQ_CHECK; |
| 886 | repl_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_NO_SEQ_CHECK; |
| 887 | } |
| 888 | } |
| 889 | |
| 890 | static void |
| 891 | sfe_ipv4_update_protocol_state(struct sfe_ipv4_connection *c, |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 892 | struct sfe_ipv4_rule_create_msg *msg) |
Nicolas Costa | 436926b | 2014-01-14 10:36:22 -0600 | [diff] [blame] | 893 | { |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 894 | switch (msg->tuple.protocol) { |
Nicolas Costa | 436926b | 2014-01-14 10:36:22 -0600 | [diff] [blame] | 895 | case IPPROTO_TCP: |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 896 | sfe_ipv4_update_tcp_state(c, msg); |
Nicolas Costa | 436926b | 2014-01-14 10:36:22 -0600 | [diff] [blame] | 897 | break; |
| 898 | } |
| 899 | } |
| 900 | |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 901 | void sfe_ipv4_update_rule(struct sfe_ipv4_rule_create_msg *msg) |
Nicolas Costa | 436926b | 2014-01-14 10:36:22 -0600 | [diff] [blame] | 902 | { |
| 903 | struct sfe_ipv4_connection *c; |
| 904 | struct sfe_ipv4 *si = &__si; |
| 905 | |
| 906 | spin_lock_bh(&si->lock); |
| 907 | |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 908 | c = sfe_ipv4_find_connection(si, |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 909 | msg->tuple.protocol, |
| 910 | msg->tuple.flow_ip, |
| 911 | msg->tuple.flow_ident, |
| 912 | msg->tuple.return_ip, |
| 913 | msg->tuple.return_ident); |
Nicolas Costa | 436926b | 2014-01-14 10:36:22 -0600 | [diff] [blame] | 914 | if (c != NULL) { |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 915 | sfe_ipv4_update_protocol_state(c, msg); |
Nicolas Costa | 436926b | 2014-01-14 10:36:22 -0600 | [diff] [blame] | 916 | } |
| 917 | |
| 918 | spin_unlock_bh(&si->lock); |
| 919 | } |
| 920 | |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 921 | /* |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 922 | * sfe_ipv4_create_rule() |
| 923 | * Create a forwarding rule. |
| 924 | */ |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 925 | int sfe_ipv4_create_rule(struct sfe_ipv4_rule_create_msg *msg) |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 926 | { |
Dave Hudson | dcd08fb | 2013-11-22 09:25:16 -0600 | [diff] [blame] | 927 | struct sfe_ipv4 *si = &__si; |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 928 | struct sfe_ipv4_connection *c, *c_old; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 929 | struct sfe_ipv4_connection_match *original_cm; |
| 930 | struct sfe_ipv4_connection_match *reply_cm; |
Matthew McClintock | db5ac51 | 2014-01-16 17:01:40 -0600 | [diff] [blame] | 931 | struct net_device *dest_dev; |
| 932 | struct net_device *src_dev; |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 933 | struct sfe_ipv4_5tuple *tuple = &msg->tuple; |
Suruchi Suman | c1a4a61 | 2021-10-21 14:50:23 +0530 | [diff] [blame] | 934 | s32 flow_interface_num = msg->conn_rule.flow_top_interface_num; |
| 935 | s32 return_interface_num = msg->conn_rule.return_top_interface_num; |
Matthew McClintock | db5ac51 | 2014-01-16 17:01:40 -0600 | [diff] [blame] | 936 | |
Suruchi Suman | c1a4a61 | 2021-10-21 14:50:23 +0530 | [diff] [blame] | 937 | if (msg->rule_flags & SFE_RULE_CREATE_FLAG_USE_FLOW_BOTTOM_INTERFACE) { |
| 938 | flow_interface_num = msg->conn_rule.flow_interface_num; |
| 939 | } |
| 940 | |
| 941 | if (msg->rule_flags & SFE_RULE_CREATE_FLAG_USE_RETURN_BOTTOM_INTERFACE) { |
| 942 | return_interface_num = msg->conn_rule.return_interface_num; |
| 943 | } |
| 944 | |
| 945 | src_dev = dev_get_by_index(&init_net, flow_interface_num); |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 946 | if (!src_dev) { |
| 947 | DEBUG_WARN("%px: Unable to find src_dev corresponding to %d\n", msg, |
Suruchi Suman | c1a4a61 | 2021-10-21 14:50:23 +0530 | [diff] [blame] | 948 | flow_interface_num); |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 949 | this_cpu_inc(si->stats_pcpu->connection_create_failures64); |
| 950 | return -EINVAL; |
| 951 | } |
| 952 | |
Suruchi Suman | c1a4a61 | 2021-10-21 14:50:23 +0530 | [diff] [blame] | 953 | dest_dev = dev_get_by_index(&init_net, return_interface_num); |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 954 | if (!dest_dev) { |
| 955 | DEBUG_WARN("%px: Unable to find dest_dev corresponding to %d\n", msg, |
Suruchi Suman | c1a4a61 | 2021-10-21 14:50:23 +0530 | [diff] [blame] | 956 | return_interface_num); |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 957 | this_cpu_inc(si->stats_pcpu->connection_create_failures64); |
| 958 | dev_put(src_dev); |
| 959 | return -EINVAL; |
| 960 | } |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 961 | |
Matthew McClintock | 389b42a | 2014-09-24 14:05:51 -0500 | [diff] [blame] | 962 | if (unlikely((dest_dev->reg_state != NETREG_REGISTERED) || |
| 963 | (src_dev->reg_state != NETREG_REGISTERED))) { |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 964 | dev_put(src_dev); |
| 965 | dev_put(dest_dev); |
| 966 | DEBUG_WARN("%px: src_dev=%s and dest_dev=%s are unregistered\n", msg, |
| 967 | src_dev->name, dest_dev->name); |
| 968 | this_cpu_inc(si->stats_pcpu->connection_create_failures64); |
Matthew McClintock | 389b42a | 2014-09-24 14:05:51 -0500 | [diff] [blame] | 969 | return -EINVAL; |
| 970 | } |
| 971 | |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 972 | /* |
| 973 | * Allocate the various connection tracking objects. |
| 974 | */ |
| 975 | c = (struct sfe_ipv4_connection *)kmalloc(sizeof(struct sfe_ipv4_connection), GFP_ATOMIC); |
| 976 | if (unlikely(!c)) { |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 977 | DEBUG_WARN("%px: memory allocation of connection entry failed\n", msg); |
| 978 | this_cpu_inc(si->stats_pcpu->connection_create_failures64); |
| 979 | dev_put(src_dev); |
| 980 | dev_put(dest_dev); |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 981 | return -ENOMEM; |
| 982 | } |
| 983 | |
| 984 | original_cm = (struct sfe_ipv4_connection_match *)kmalloc(sizeof(struct sfe_ipv4_connection_match), GFP_ATOMIC); |
| 985 | if (unlikely(!original_cm)) { |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 986 | DEBUG_WARN("%px: memory allocation of connection match entry failed\n", msg); |
| 987 | this_cpu_inc(si->stats_pcpu->connection_create_failures64); |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 988 | kfree(c); |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 989 | dev_put(src_dev); |
| 990 | dev_put(dest_dev); |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 991 | return -ENOMEM; |
| 992 | } |
| 993 | |
| 994 | reply_cm = (struct sfe_ipv4_connection_match *)kmalloc(sizeof(struct sfe_ipv4_connection_match), GFP_ATOMIC); |
| 995 | if (unlikely(!reply_cm)) { |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 996 | DEBUG_WARN("%px: memory allocation of connection match entry failed\n", msg); |
| 997 | this_cpu_inc(si->stats_pcpu->connection_create_failures64); |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 998 | kfree(original_cm); |
| 999 | kfree(c); |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1000 | dev_put(src_dev); |
| 1001 | dev_put(dest_dev); |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 1002 | return -ENOMEM; |
| 1003 | } |
| 1004 | |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 1005 | this_cpu_inc(si->stats_pcpu->connection_create_requests64); |
| 1006 | |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1007 | spin_lock_bh(&si->lock); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1008 | |
| 1009 | /* |
Nicolas Costa | 436926b | 2014-01-14 10:36:22 -0600 | [diff] [blame] | 1010 | * Check to see if there is already a flow that matches the rule we're |
| 1011 | * trying to create. If there is then we can't create a new one. |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1012 | */ |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 1013 | c_old = sfe_ipv4_find_connection(si, |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1014 | msg->tuple.protocol, |
| 1015 | msg->tuple.flow_ip, |
| 1016 | msg->tuple.flow_ident, |
| 1017 | msg->tuple.return_ip, |
| 1018 | msg->tuple.return_ident); |
| 1019 | |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 1020 | if (c_old != NULL) { |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 1021 | this_cpu_inc(si->stats_pcpu->connection_create_collisions64); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1022 | |
| 1023 | /* |
Nicolas Costa | 436926b | 2014-01-14 10:36:22 -0600 | [diff] [blame] | 1024 | * If we already have the flow then it's likely that this |
| 1025 | * request to create the connection rule contains more |
| 1026 | * up-to-date information. Check and update accordingly. |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1027 | */ |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1028 | sfe_ipv4_update_protocol_state(c, msg); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1029 | spin_unlock_bh(&si->lock); |
| 1030 | |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 1031 | kfree(reply_cm); |
| 1032 | kfree(original_cm); |
| 1033 | kfree(c); |
| 1034 | |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1035 | dev_put(src_dev); |
| 1036 | dev_put(dest_dev); |
| 1037 | |
| 1038 | DEBUG_TRACE("connection already exists - p:%d\n" |
| 1039 | " s: %s:%pM:%pI4:%u, d: %s:%pM:%pI4:%u\n", |
| 1040 | tuple->protocol, |
| 1041 | src_dev->name, msg->conn_rule.flow_mac, &tuple->flow_ip, ntohs(tuple->flow_ident), |
| 1042 | dest_dev->name, msg->conn_rule.return_mac, &tuple->return_ip, ntohs(tuple->return_ident)); |
| 1043 | |
Nicolas Costa | 514fde0 | 2014-01-13 15:50:29 -0600 | [diff] [blame] | 1044 | return -EADDRINUSE; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1045 | } |
| 1046 | |
| 1047 | /* |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1048 | * Fill in the "original" direction connection matching object. |
| 1049 | * Note that the transmit MAC address is "dest_mac_xlate" because |
| 1050 | * we always know both ends of a connection by their translated |
| 1051 | * addresses and not their public addresses. |
| 1052 | */ |
Matthew McClintock | db5ac51 | 2014-01-16 17:01:40 -0600 | [diff] [blame] | 1053 | original_cm->match_dev = src_dev; |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1054 | original_cm->match_protocol = tuple->protocol; |
| 1055 | original_cm->match_src_ip = tuple->flow_ip; |
| 1056 | original_cm->match_src_port = tuple->flow_ident; |
| 1057 | original_cm->match_dest_ip = tuple->return_ip; |
| 1058 | original_cm->match_dest_port = tuple->return_ident; |
| 1059 | |
| 1060 | original_cm->xlate_src_ip = msg->conn_rule.flow_ip_xlate; |
| 1061 | original_cm->xlate_src_port = msg->conn_rule.flow_ident_xlate; |
| 1062 | original_cm->xlate_dest_ip = msg->conn_rule.return_ip_xlate; |
| 1063 | original_cm->xlate_dest_port =msg->conn_rule.return_ident_xlate; |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 1064 | atomic_set(&original_cm->rx_packet_count, 0); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1065 | original_cm->rx_packet_count64 = 0; |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 1066 | atomic_set(&original_cm->rx_byte_count, 0); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1067 | original_cm->rx_byte_count64 = 0; |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1068 | |
Matthew McClintock | db5ac51 | 2014-01-16 17:01:40 -0600 | [diff] [blame] | 1069 | original_cm->xmit_dev = dest_dev; |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1070 | original_cm->xmit_dev_mtu = msg->conn_rule.return_mtu; |
| 1071 | |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1072 | original_cm->connection = c; |
| 1073 | original_cm->counter_match = reply_cm; |
| 1074 | original_cm->flags = 0; |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1075 | |
| 1076 | if (msg->valid_flags & SFE_RULE_CREATE_QOS_VALID) { |
| 1077 | original_cm->priority = msg->qos_rule.flow_qos_tag; |
Xiaoping Fan | e1963d4 | 2015-08-25 17:06:19 -0700 | [diff] [blame] | 1078 | original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_PRIORITY_REMARK; |
| 1079 | } |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1080 | |
| 1081 | if (msg->valid_flags & SFE_RULE_CREATE_DSCP_MARKING_VALID) { |
| 1082 | original_cm->dscp = msg->dscp_rule.flow_dscp << SFE_IPV4_DSCP_SHIFT; |
Xiaoping Fan | e1963d4 | 2015-08-25 17:06:19 -0700 | [diff] [blame] | 1083 | original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_DSCP_REMARK; |
| 1084 | } |
Ratheesh Kannoth | a3cf0e0 | 2021-12-09 09:44:10 +0530 | [diff] [blame] | 1085 | |
Xiaoping Fan | d1dc7b2 | 2015-01-23 00:43:56 -0800 | [diff] [blame] | 1086 | #ifdef CONFIG_NF_FLOW_COOKIE |
| 1087 | original_cm->flow_cookie = 0; |
| 1088 | #endif |
Zhi Chen | 8748eb3 | 2015-06-18 12:58:48 -0700 | [diff] [blame] | 1089 | #ifdef CONFIG_XFRM |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1090 | if (msg->valid_flags & SFE_RULE_CREATE_DIRECTION_VALID) { |
| 1091 | original_cm->flow_accel = msg->direction_rule.flow_accel; |
| 1092 | } else { |
| 1093 | original_cm->flow_accel = 1; |
| 1094 | } |
Zhi Chen | 8748eb3 | 2015-06-18 12:58:48 -0700 | [diff] [blame] | 1095 | #endif |
Ratheesh Kannoth | a3cf0e0 | 2021-12-09 09:44:10 +0530 | [diff] [blame] | 1096 | /* |
| 1097 | * If l2_features are disabled and flow uses l2 features such as macvlan/bridge/pppoe/vlan, |
| 1098 | * bottom interfaces are expected to be disabled in the flow rule and always top interfaces |
| 1099 | * are used. In such cases, do not use HW csum offload. csum offload is used only when we |
| 1100 | * are sending directly to the destination interface that supports it. |
| 1101 | */ |
| 1102 | if (likely(dest_dev->features & NETIF_F_HW_CSUM)) { |
| 1103 | if ((msg->conn_rule.return_top_interface_num == msg->conn_rule.return_interface_num) || |
| 1104 | (msg->rule_flags & SFE_RULE_CREATE_FLAG_USE_RETURN_BOTTOM_INTERFACE)) { |
| 1105 | original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_CSUM_OFFLOAD; |
| 1106 | } |
| 1107 | } |
Matthew McClintock | db5ac51 | 2014-01-16 17:01:40 -0600 | [diff] [blame] | 1108 | |
| 1109 | /* |
Guduri Prathyusha | eb31c90 | 2021-11-10 20:18:50 +0530 | [diff] [blame^] | 1110 | * Adding PPPoE parameters to original and reply entries based on the direction where |
| 1111 | * PPPoE header is valid in ECM rule. |
| 1112 | * |
| 1113 | * If PPPoE is valid in flow direction (from interface is PPPoE), then |
| 1114 | * original cm will have PPPoE at ingress (strip PPPoE header) |
| 1115 | * reply cm will have PPPoE at egress (add PPPoE header) |
| 1116 | * |
| 1117 | * If PPPoE is valid in return direction (to interface is PPPoE), then |
| 1118 | * original cm will have PPPoE at egress (add PPPoE header) |
| 1119 | * reply cm will have PPPoE at ingress (strip PPPoE header) |
| 1120 | */ |
| 1121 | if (msg->valid_flags & SFE_RULE_CREATE_PPPOE_DECAP_VALID) { |
| 1122 | original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_PPPOE_DECAP; |
| 1123 | original_cm->pppoe_session_id = msg->pppoe_rule.flow_pppoe_session_id; |
| 1124 | ether_addr_copy(original_cm->pppoe_remote_mac, msg->pppoe_rule.flow_pppoe_remote_mac); |
| 1125 | |
| 1126 | reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_PPPOE_ENCAP; |
| 1127 | reply_cm->pppoe_session_id = msg->pppoe_rule.flow_pppoe_session_id; |
| 1128 | ether_addr_copy(reply_cm->pppoe_remote_mac, msg->pppoe_rule.flow_pppoe_remote_mac); |
| 1129 | } |
| 1130 | |
| 1131 | if (msg->valid_flags & SFE_RULE_CREATE_PPPOE_ENCAP_VALID) { |
| 1132 | original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_PPPOE_ENCAP; |
| 1133 | original_cm->pppoe_session_id = msg->pppoe_rule.return_pppoe_session_id; |
| 1134 | ether_addr_copy(original_cm->pppoe_remote_mac, msg->pppoe_rule.return_pppoe_remote_mac); |
| 1135 | |
| 1136 | reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_PPPOE_DECAP; |
| 1137 | reply_cm->pppoe_session_id = msg->pppoe_rule.return_pppoe_session_id; |
| 1138 | ether_addr_copy(reply_cm->pppoe_remote_mac, msg->pppoe_rule.return_pppoe_remote_mac); |
| 1139 | } |
| 1140 | |
| 1141 | /* |
Ken Zhu | bbf4965 | 2021-09-12 15:33:09 -0700 | [diff] [blame] | 1142 | * For the non-arp interface, we don't write L2 HDR. |
Matthew McClintock | db5ac51 | 2014-01-16 17:01:40 -0600 | [diff] [blame] | 1143 | */ |
Ken Zhu | bbf4965 | 2021-09-12 15:33:09 -0700 | [diff] [blame] | 1144 | if (!(dest_dev->flags & IFF_NOARP)) { |
Ratheesh Kannoth | 29140aa | 2021-10-20 08:25:02 +0530 | [diff] [blame] | 1145 | |
| 1146 | /* |
| 1147 | * Check whether the rule has configured a specific source MAC address to use. |
| 1148 | * This is needed when virtual L3 interfaces such as br-lan, macvlan, vlan are used during egress |
| 1149 | */ |
| 1150 | if ((msg->valid_flags & SFE_RULE_CREATE_SRC_MAC_VALID) && |
| 1151 | (msg->src_mac_rule.mac_valid_flags & SFE_SRC_MAC_RETURN_VALID)) { |
| 1152 | ether_addr_copy((u8 *)original_cm->xmit_src_mac, (u8 *)msg->src_mac_rule.return_src_mac); |
| 1153 | } else { |
| 1154 | ether_addr_copy((u8 *)original_cm->xmit_src_mac, (u8 *)dest_dev->dev_addr); |
| 1155 | } |
| 1156 | |
| 1157 | ether_addr_copy((u8 *)original_cm->xmit_dest_mac, (u8 *)msg->conn_rule.return_mac); |
| 1158 | |
Matthew McClintock | db5ac51 | 2014-01-16 17:01:40 -0600 | [diff] [blame] | 1159 | original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_L2_HDR; |
| 1160 | |
| 1161 | /* |
| 1162 | * If our dev writes Ethernet headers then we can write a really fast |
| 1163 | * version. |
| 1164 | */ |
| 1165 | if (dest_dev->header_ops) { |
| 1166 | if (dest_dev->header_ops->create == eth_header) { |
| 1167 | original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_FAST_ETH_HDR; |
| 1168 | } |
| 1169 | } |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1170 | } |
| 1171 | |
| 1172 | /* |
| 1173 | * Fill in the "reply" direction connection matching object. |
| 1174 | */ |
Matthew McClintock | db5ac51 | 2014-01-16 17:01:40 -0600 | [diff] [blame] | 1175 | reply_cm->match_dev = dest_dev; |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1176 | reply_cm->match_protocol = tuple->protocol; |
| 1177 | reply_cm->match_src_ip = msg->conn_rule.return_ip_xlate; |
| 1178 | reply_cm->match_src_port = msg->conn_rule.return_ident_xlate; |
| 1179 | reply_cm->match_dest_ip = msg->conn_rule.flow_ip_xlate; |
| 1180 | reply_cm->match_dest_port = msg->conn_rule.flow_ident_xlate; |
| 1181 | |
| 1182 | reply_cm->xlate_src_ip = tuple->return_ip; |
| 1183 | reply_cm->xlate_src_port = tuple->return_ident; |
| 1184 | reply_cm->xlate_dest_ip = tuple->flow_ip; |
| 1185 | reply_cm->xlate_dest_port = tuple->flow_ident;; |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 1186 | |
| 1187 | atomic_set(&reply_cm->rx_packet_count, 0); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1188 | reply_cm->rx_packet_count64 = 0; |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 1189 | atomic_set(&reply_cm->rx_byte_count, 0); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1190 | reply_cm->rx_byte_count64 = 0; |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 1191 | |
Matthew McClintock | db5ac51 | 2014-01-16 17:01:40 -0600 | [diff] [blame] | 1192 | reply_cm->xmit_dev = src_dev; |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1193 | reply_cm->xmit_dev_mtu = msg->conn_rule.flow_mtu; |
Ratheesh Kannoth | 29140aa | 2021-10-20 08:25:02 +0530 | [diff] [blame] | 1194 | |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1195 | reply_cm->connection = c; |
| 1196 | reply_cm->counter_match = original_cm; |
| 1197 | reply_cm->flags = 0; |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1198 | if (msg->valid_flags & SFE_RULE_CREATE_QOS_VALID) { |
| 1199 | reply_cm->priority = msg->qos_rule.return_qos_tag; |
Xiaoping Fan | e1963d4 | 2015-08-25 17:06:19 -0700 | [diff] [blame] | 1200 | reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_PRIORITY_REMARK; |
| 1201 | } |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1202 | if (msg->valid_flags & SFE_RULE_CREATE_DSCP_MARKING_VALID) { |
| 1203 | reply_cm->dscp = msg->dscp_rule.return_dscp << SFE_IPV4_DSCP_SHIFT; |
Xiaoping Fan | e1963d4 | 2015-08-25 17:06:19 -0700 | [diff] [blame] | 1204 | reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_DSCP_REMARK; |
| 1205 | } |
Xiaoping Fan | d1dc7b2 | 2015-01-23 00:43:56 -0800 | [diff] [blame] | 1206 | #ifdef CONFIG_NF_FLOW_COOKIE |
| 1207 | reply_cm->flow_cookie = 0; |
| 1208 | #endif |
Zhi Chen | 8748eb3 | 2015-06-18 12:58:48 -0700 | [diff] [blame] | 1209 | #ifdef CONFIG_XFRM |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1210 | if (msg->valid_flags & SFE_RULE_CREATE_DIRECTION_VALID) { |
| 1211 | reply_cm->flow_accel = msg->direction_rule.return_accel; |
| 1212 | } else { |
| 1213 | reply_cm->flow_accel = 1; |
| 1214 | } |
| 1215 | |
Zhi Chen | 8748eb3 | 2015-06-18 12:58:48 -0700 | [diff] [blame] | 1216 | #endif |
Ratheesh Kannoth | a3cf0e0 | 2021-12-09 09:44:10 +0530 | [diff] [blame] | 1217 | /* |
| 1218 | * If l2_features are disabled and flow uses l2 features such as macvlan/bridge/pppoe/vlan, |
| 1219 | * bottom interfaces are expected to be disabled in the flow rule and always top interfaces |
| 1220 | * are used. In such cases, do not use HW csum offload. csum offload is used only when we |
| 1221 | * are sending directly to the destination interface that supports it. |
| 1222 | */ |
| 1223 | if (likely(src_dev->features & NETIF_F_HW_CSUM)) { |
| 1224 | if ((msg->conn_rule.flow_top_interface_num == msg->conn_rule.flow_interface_num) || |
| 1225 | (msg->rule_flags & SFE_RULE_CREATE_FLAG_USE_FLOW_BOTTOM_INTERFACE)) { |
| 1226 | reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_CSUM_OFFLOAD; |
| 1227 | } |
| 1228 | } |
Matthew McClintock | db5ac51 | 2014-01-16 17:01:40 -0600 | [diff] [blame] | 1229 | |
| 1230 | /* |
Ken Zhu | bbf4965 | 2021-09-12 15:33:09 -0700 | [diff] [blame] | 1231 | * For the non-arp interface, we don't write L2 HDR. |
Matthew McClintock | db5ac51 | 2014-01-16 17:01:40 -0600 | [diff] [blame] | 1232 | */ |
Ken Zhu | bbf4965 | 2021-09-12 15:33:09 -0700 | [diff] [blame] | 1233 | if (!(src_dev->flags & IFF_NOARP)) { |
Ratheesh Kannoth | 29140aa | 2021-10-20 08:25:02 +0530 | [diff] [blame] | 1234 | |
| 1235 | /* |
| 1236 | * Check whether the rule has configured a specific source MAC address to use. |
| 1237 | * This is needed when virtual L3 interfaces such as br-lan, macvlan, vlan are used during egress |
| 1238 | */ |
| 1239 | if ((msg->valid_flags & SFE_RULE_CREATE_SRC_MAC_VALID) && |
| 1240 | (msg->src_mac_rule.mac_valid_flags & SFE_SRC_MAC_FLOW_VALID)) { |
| 1241 | ether_addr_copy((u8 *)reply_cm->xmit_src_mac, (u8 *)msg->src_mac_rule.flow_src_mac); |
| 1242 | } else { |
| 1243 | ether_addr_copy((u8 *)reply_cm->xmit_src_mac, (u8 *)src_dev->dev_addr); |
| 1244 | } |
| 1245 | ether_addr_copy((u8 *)reply_cm->xmit_dest_mac, (u8 *)msg->conn_rule.flow_mac); |
| 1246 | |
Matthew McClintock | db5ac51 | 2014-01-16 17:01:40 -0600 | [diff] [blame] | 1247 | reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_L2_HDR; |
| 1248 | |
| 1249 | /* |
| 1250 | * If our dev writes Ethernet headers then we can write a really fast |
| 1251 | * version. |
| 1252 | */ |
| 1253 | if (src_dev->header_ops) { |
| 1254 | if (src_dev->header_ops->create == eth_header) { |
| 1255 | reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_FAST_ETH_HDR; |
| 1256 | } |
| 1257 | } |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1258 | } |
| 1259 | |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1260 | if ((tuple->return_ip != msg->conn_rule.return_ip_xlate) || |
| 1261 | (tuple->return_ident != msg->conn_rule.return_ident_xlate)) { |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1262 | original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_XLATE_DEST; |
| 1263 | reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_XLATE_SRC; |
| 1264 | } |
| 1265 | |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1266 | if ((tuple->flow_ip != msg->conn_rule.flow_ip_xlate) || |
| 1267 | (tuple->flow_ident != msg->conn_rule.flow_ident_xlate)) { |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1268 | original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_XLATE_SRC; |
| 1269 | reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_XLATE_DEST; |
| 1270 | } |
| 1271 | |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1272 | c->protocol = tuple->protocol; |
| 1273 | c->src_ip = tuple->flow_ip; |
| 1274 | c->src_ip_xlate = msg->conn_rule.flow_ip_xlate; |
| 1275 | c->src_port = tuple->flow_ident; |
| 1276 | c->src_port_xlate = msg->conn_rule.flow_ident_xlate; |
Matthew McClintock | db5ac51 | 2014-01-16 17:01:40 -0600 | [diff] [blame] | 1277 | c->original_dev = src_dev; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1278 | c->original_match = original_cm; |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1279 | c->dest_ip = tuple->return_ip; |
| 1280 | c->dest_ip_xlate = msg->conn_rule.return_ip_xlate; |
| 1281 | c->dest_port = tuple->return_ident; |
| 1282 | c->dest_port_xlate = msg->conn_rule.return_ident_xlate; |
Matthew McClintock | db5ac51 | 2014-01-16 17:01:40 -0600 | [diff] [blame] | 1283 | c->reply_dev = dest_dev; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1284 | c->reply_match = reply_cm; |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1285 | c->mark = 0; /* TODO : no mark setting for create rule */ |
Xiaoping Fan | 3458647 | 2015-07-03 02:20:35 -0700 | [diff] [blame] | 1286 | c->debug_read_seq = 0; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1287 | c->last_sync_jiffies = get_jiffies_64(); |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 1288 | c->removed = false; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1289 | |
| 1290 | /* |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1291 | * Initialize the protocol-specific information that we track. |
| 1292 | */ |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1293 | switch (tuple->protocol) { |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1294 | case IPPROTO_TCP: |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1295 | original_cm->protocol_state.tcp.win_scale = msg->tcp_rule.flow_window_scale; |
| 1296 | original_cm->protocol_state.tcp.max_win = msg->tcp_rule.flow_max_window ? msg->tcp_rule.flow_max_window : 1; |
| 1297 | original_cm->protocol_state.tcp.end = msg->tcp_rule.flow_end; |
| 1298 | original_cm->protocol_state.tcp.max_end = msg->tcp_rule.flow_max_end; |
| 1299 | |
| 1300 | reply_cm->protocol_state.tcp.win_scale = msg->tcp_rule.return_window_scale; |
| 1301 | reply_cm->protocol_state.tcp.max_win = msg->tcp_rule.return_max_window ? msg->tcp_rule.return_max_window : 1; |
| 1302 | reply_cm->protocol_state.tcp.end = msg->tcp_rule.return_end; |
| 1303 | reply_cm->protocol_state.tcp.max_end = msg->tcp_rule.return_max_end; |
| 1304 | |
| 1305 | if (msg->rule_flags & SFE_RULE_CREATE_FLAG_NO_SEQ_CHECK) { |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1306 | original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_NO_SEQ_CHECK; |
| 1307 | reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_NO_SEQ_CHECK; |
| 1308 | } |
| 1309 | break; |
| 1310 | } |
| 1311 | |
| 1312 | sfe_ipv4_connection_match_compute_translations(original_cm); |
| 1313 | sfe_ipv4_connection_match_compute_translations(reply_cm); |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 1314 | sfe_ipv4_insert_connection(si, c); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1315 | |
| 1316 | spin_unlock_bh(&si->lock); |
| 1317 | |
| 1318 | /* |
| 1319 | * We have everything we need! |
| 1320 | */ |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1321 | DEBUG_INFO("new connection - p: %d\n" |
Tian Yang | 45f39c8 | 2020-10-06 14:07:47 -0700 | [diff] [blame] | 1322 | " s: %s:%pxM(%pxM):%pI4(%pI4):%u(%u)\n" |
| 1323 | " d: %s:%pxM(%pxM):%pI4(%pI4):%u(%u)\n", |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1324 | tuple->protocol, |
| 1325 | src_dev->name, msg->conn_rule.flow_mac, NULL, |
| 1326 | &tuple->flow_ip, &msg->conn_rule.flow_ip_xlate, ntohs(tuple->flow_ident), ntohs(msg->conn_rule.flow_ident_xlate), |
| 1327 | dest_dev->name, NULL, msg->conn_rule.return_mac, |
| 1328 | &tuple->return_ip, &msg->conn_rule.return_ip_xlate, ntohs(tuple->return_ident), ntohs(msg->conn_rule.return_ident_xlate)); |
Nicolas Costa | 514fde0 | 2014-01-13 15:50:29 -0600 | [diff] [blame] | 1329 | |
| 1330 | return 0; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1331 | } |
| 1332 | |
| 1333 | /* |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1334 | * sfe_ipv4_destroy_rule() |
| 1335 | * Destroy a forwarding rule. |
| 1336 | */ |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1337 | void sfe_ipv4_destroy_rule(struct sfe_ipv4_rule_destroy_msg *msg) |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1338 | { |
Dave Hudson | dcd08fb | 2013-11-22 09:25:16 -0600 | [diff] [blame] | 1339 | struct sfe_ipv4 *si = &__si; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1340 | struct sfe_ipv4_connection *c; |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 1341 | bool ret; |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1342 | struct sfe_ipv4_5tuple *tuple = &msg->tuple; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1343 | |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 1344 | this_cpu_inc(si->stats_pcpu->connection_destroy_requests64); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1345 | spin_lock_bh(&si->lock); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1346 | |
| 1347 | /* |
| 1348 | * Check to see if we have a flow that matches the rule we're trying |
| 1349 | * to destroy. If there isn't then we can't destroy it. |
| 1350 | */ |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 1351 | c = sfe_ipv4_find_connection(si, tuple->protocol, tuple->flow_ip, tuple->flow_ident, |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1352 | tuple->return_ip, tuple->return_ident); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1353 | if (!c) { |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1354 | spin_unlock_bh(&si->lock); |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 1355 | this_cpu_inc(si->stats_pcpu->connection_destroy_misses64); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1356 | |
| 1357 | DEBUG_TRACE("connection does not exist - p: %d, s: %pI4:%u, d: %pI4:%u\n", |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1358 | tuple->protocol, &tuple->flow_ip, ntohs(tuple->flow_ident), |
| 1359 | &tuple->return_ip, ntohs(tuple->return_ident)); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1360 | return; |
| 1361 | } |
| 1362 | |
| 1363 | /* |
| 1364 | * Remove our connection details from the hash tables. |
| 1365 | */ |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 1366 | ret = sfe_ipv4_remove_connection(si, c); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1367 | spin_unlock_bh(&si->lock); |
| 1368 | |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 1369 | if (ret) { |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 1370 | sfe_ipv4_flush_connection(si, c, SFE_SYNC_REASON_DESTROY); |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 1371 | } |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1372 | |
| 1373 | DEBUG_INFO("connection destroyed - p: %d, s: %pI4:%u, d: %pI4:%u\n", |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1374 | tuple->protocol, &tuple->flow_ip, ntohs(tuple->flow_ident), |
| 1375 | &tuple->return_ip, ntohs(tuple->return_ident)); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1376 | } |
| 1377 | |
| 1378 | /* |
Dave Hudson | dcd08fb | 2013-11-22 09:25:16 -0600 | [diff] [blame] | 1379 | * sfe_ipv4_register_sync_rule_callback() |
| 1380 | * Register a callback for rule synchronization. |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1381 | */ |
Xiaoping Fan | d44a5b4 | 2015-05-26 17:37:37 -0700 | [diff] [blame] | 1382 | void sfe_ipv4_register_sync_rule_callback(sfe_sync_rule_callback_t sync_rule_callback) |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1383 | { |
| 1384 | struct sfe_ipv4 *si = &__si; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1385 | |
| 1386 | spin_lock_bh(&si->lock); |
Dave Hudson | dcd08fb | 2013-11-22 09:25:16 -0600 | [diff] [blame] | 1387 | rcu_assign_pointer(si->sync_rule_callback, sync_rule_callback); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1388 | spin_unlock_bh(&si->lock); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1389 | } |
| 1390 | |
| 1391 | /* |
| 1392 | * sfe_ipv4_get_debug_dev() |
| 1393 | */ |
| 1394 | static ssize_t sfe_ipv4_get_debug_dev(struct device *dev, |
| 1395 | struct device_attribute *attr, |
| 1396 | char *buf) |
| 1397 | { |
| 1398 | struct sfe_ipv4 *si = &__si; |
| 1399 | ssize_t count; |
| 1400 | int num; |
| 1401 | |
| 1402 | spin_lock_bh(&si->lock); |
| 1403 | num = si->debug_dev; |
| 1404 | spin_unlock_bh(&si->lock); |
| 1405 | |
| 1406 | count = snprintf(buf, (ssize_t)PAGE_SIZE, "%d\n", num); |
| 1407 | return count; |
| 1408 | } |
| 1409 | |
| 1410 | /* |
Dave Hudson | dcd08fb | 2013-11-22 09:25:16 -0600 | [diff] [blame] | 1411 | * sysfs attributes. |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1412 | */ |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1413 | static const struct device_attribute sfe_ipv4_debug_dev_attr = |
Xiaoping Fan | e70da41 | 2016-02-26 16:47:57 -0800 | [diff] [blame] | 1414 | __ATTR(debug_dev, S_IWUSR | S_IRUGO, sfe_ipv4_get_debug_dev, NULL); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1415 | |
| 1416 | /* |
Dave Hudson | dcd08fb | 2013-11-22 09:25:16 -0600 | [diff] [blame] | 1417 | * sfe_ipv4_destroy_all_rules_for_dev() |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1418 | * Destroy all connections that match a particular device. |
| 1419 | * |
| 1420 | * If we pass dev as NULL then this destroys all connections. |
| 1421 | */ |
Dave Hudson | dcd08fb | 2013-11-22 09:25:16 -0600 | [diff] [blame] | 1422 | void sfe_ipv4_destroy_all_rules_for_dev(struct net_device *dev) |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1423 | { |
Dave Hudson | dcd08fb | 2013-11-22 09:25:16 -0600 | [diff] [blame] | 1424 | struct sfe_ipv4 *si = &__si; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1425 | struct sfe_ipv4_connection *c; |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 1426 | bool ret; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1427 | |
Xiaoping Fan | 3458647 | 2015-07-03 02:20:35 -0700 | [diff] [blame] | 1428 | another_round: |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1429 | spin_lock_bh(&si->lock); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1430 | |
Xiaoping Fan | 3458647 | 2015-07-03 02:20:35 -0700 | [diff] [blame] | 1431 | for (c = si->all_connections_head; c; c = c->all_connections_next) { |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1432 | /* |
Xiaoping Fan | 3458647 | 2015-07-03 02:20:35 -0700 | [diff] [blame] | 1433 | * Does this connection relate to the device we are destroying? |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1434 | */ |
| 1435 | if (!dev |
| 1436 | || (dev == c->original_dev) |
| 1437 | || (dev == c->reply_dev)) { |
Xiaoping Fan | 3458647 | 2015-07-03 02:20:35 -0700 | [diff] [blame] | 1438 | break; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1439 | } |
Xiaoping Fan | 3458647 | 2015-07-03 02:20:35 -0700 | [diff] [blame] | 1440 | } |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1441 | |
Xiaoping Fan | 3458647 | 2015-07-03 02:20:35 -0700 | [diff] [blame] | 1442 | if (c) { |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 1443 | ret = sfe_ipv4_remove_connection(si, c); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1444 | } |
| 1445 | |
| 1446 | spin_unlock_bh(&si->lock); |
Xiaoping Fan | 3458647 | 2015-07-03 02:20:35 -0700 | [diff] [blame] | 1447 | |
| 1448 | if (c) { |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 1449 | if (ret) { |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 1450 | sfe_ipv4_flush_connection(si, c, SFE_SYNC_REASON_DESTROY); |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 1451 | } |
Xiaoping Fan | 3458647 | 2015-07-03 02:20:35 -0700 | [diff] [blame] | 1452 | goto another_round; |
| 1453 | } |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1454 | } |
| 1455 | |
| 1456 | /* |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1457 | * sfe_ipv4_periodic_sync() |
| 1458 | */ |
Ken Zhu | 137722d | 2021-09-23 17:57:36 -0700 | [diff] [blame] | 1459 | static void sfe_ipv4_periodic_sync(struct work_struct *work) |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1460 | { |
Ken Zhu | 137722d | 2021-09-23 17:57:36 -0700 | [diff] [blame] | 1461 | struct sfe_ipv4 *si = container_of((struct delayed_work *)work, struct sfe_ipv4, sync_dwork); |
Xiaoping Fan | 6a1672f | 2016-08-17 19:58:12 -0700 | [diff] [blame] | 1462 | u64 now_jiffies; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1463 | int quota; |
Xiaoping Fan | d44a5b4 | 2015-05-26 17:37:37 -0700 | [diff] [blame] | 1464 | sfe_sync_rule_callback_t sync_rule_callback; |
Ken Zhu | dc42367 | 2021-09-02 18:27:01 -0700 | [diff] [blame] | 1465 | struct sfe_ipv4_connection *c; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1466 | |
| 1467 | now_jiffies = get_jiffies_64(); |
| 1468 | |
Dave Hudson | dcd08fb | 2013-11-22 09:25:16 -0600 | [diff] [blame] | 1469 | rcu_read_lock(); |
| 1470 | sync_rule_callback = rcu_dereference(si->sync_rule_callback); |
| 1471 | if (!sync_rule_callback) { |
| 1472 | rcu_read_unlock(); |
| 1473 | goto done; |
| 1474 | } |
| 1475 | |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1476 | spin_lock_bh(&si->lock); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1477 | |
| 1478 | /* |
Ken Zhu | dc42367 | 2021-09-02 18:27:01 -0700 | [diff] [blame] | 1479 | * If we have reached the end of the connection list, walk from |
| 1480 | * the connection head. |
| 1481 | */ |
| 1482 | c = si->wc_next; |
| 1483 | if (unlikely(!c)) { |
| 1484 | c = si->all_connections_head; |
| 1485 | } |
| 1486 | |
| 1487 | /* |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1488 | * Get an estimate of the number of connections to parse in this sync. |
| 1489 | */ |
| 1490 | quota = (si->num_connections + 63) / 64; |
| 1491 | |
| 1492 | /* |
Ken Zhu | dc42367 | 2021-09-02 18:27:01 -0700 | [diff] [blame] | 1493 | * Walk the "all connection" list and sync the connection state. |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1494 | */ |
Ken Zhu | dc42367 | 2021-09-02 18:27:01 -0700 | [diff] [blame] | 1495 | while (likely(c && quota)) { |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1496 | struct sfe_ipv4_connection_match *cm; |
| 1497 | struct sfe_ipv4_connection_match *counter_cm; |
Xiaoping Fan | d44a5b4 | 2015-05-26 17:37:37 -0700 | [diff] [blame] | 1498 | struct sfe_connection_sync sis; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1499 | |
Ken Zhu | dc42367 | 2021-09-02 18:27:01 -0700 | [diff] [blame] | 1500 | cm = c->original_match; |
| 1501 | counter_cm = c->reply_match; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1502 | |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1503 | /* |
Ken Zhu | dc42367 | 2021-09-02 18:27:01 -0700 | [diff] [blame] | 1504 | * Didn't receive packets in the original direction or reply |
| 1505 | * direction, move to the next connection. |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1506 | */ |
Ken Zhu | dc42367 | 2021-09-02 18:27:01 -0700 | [diff] [blame] | 1507 | if ((!atomic_read(&cm->rx_packet_count)) && !(atomic_read(&counter_cm->rx_packet_count))) { |
| 1508 | c = c->all_connections_next; |
| 1509 | continue; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1510 | } |
| 1511 | |
Ken Zhu | dc42367 | 2021-09-02 18:27:01 -0700 | [diff] [blame] | 1512 | quota--; |
Matthew McClintock | af48f1e | 2014-01-23 15:29:19 -0600 | [diff] [blame] | 1513 | |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 1514 | sfe_ipv4_gen_sync_connection(si, c, &sis, SFE_SYNC_REASON_STATS, now_jiffies); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1515 | |
Ken Zhu | dc42367 | 2021-09-02 18:27:01 -0700 | [diff] [blame] | 1516 | si->wc_next = c->all_connections_next; |
| 1517 | |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1518 | /* |
| 1519 | * We don't want to be holding the lock when we sync! |
| 1520 | */ |
| 1521 | spin_unlock_bh(&si->lock); |
Dave Hudson | dcd08fb | 2013-11-22 09:25:16 -0600 | [diff] [blame] | 1522 | sync_rule_callback(&sis); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1523 | spin_lock_bh(&si->lock); |
Ken Zhu | dc42367 | 2021-09-02 18:27:01 -0700 | [diff] [blame] | 1524 | |
| 1525 | /* |
| 1526 | * c must be set and used in the same lock/unlock window; |
| 1527 | * because c could be removed when we don't hold the lock, |
| 1528 | * so delay grabbing until after the callback and relock. |
| 1529 | */ |
| 1530 | c = si->wc_next; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1531 | } |
| 1532 | |
Ken Zhu | dc42367 | 2021-09-02 18:27:01 -0700 | [diff] [blame] | 1533 | /* |
| 1534 | * At the end of the sync, put the wc_next to the connection we left. |
| 1535 | */ |
| 1536 | si->wc_next = c; |
| 1537 | |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1538 | spin_unlock_bh(&si->lock); |
Dave Hudson | dcd08fb | 2013-11-22 09:25:16 -0600 | [diff] [blame] | 1539 | rcu_read_unlock(); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1540 | |
Dave Hudson | dcd08fb | 2013-11-22 09:25:16 -0600 | [diff] [blame] | 1541 | done: |
Ken Zhu | 137722d | 2021-09-23 17:57:36 -0700 | [diff] [blame] | 1542 | schedule_delayed_work_on(si->work_cpu, (struct delayed_work *)work, ((HZ + 99) / 100)); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1543 | } |
| 1544 | |
| 1545 | #define CHAR_DEV_MSG_SIZE 768 |
| 1546 | |
| 1547 | /* |
| 1548 | * sfe_ipv4_debug_dev_read_start() |
| 1549 | * Generate part of the XML output. |
| 1550 | */ |
| 1551 | static bool sfe_ipv4_debug_dev_read_start(struct sfe_ipv4 *si, char *buffer, char *msg, size_t *length, |
| 1552 | int *total_read, struct sfe_ipv4_debug_xml_write_state *ws) |
| 1553 | { |
| 1554 | int bytes_read; |
| 1555 | |
Xiaoping Fan | 3458647 | 2015-07-03 02:20:35 -0700 | [diff] [blame] | 1556 | si->debug_read_seq++; |
| 1557 | |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1558 | bytes_read = snprintf(msg, CHAR_DEV_MSG_SIZE, "<sfe_ipv4>\n"); |
| 1559 | if (copy_to_user(buffer + *total_read, msg, CHAR_DEV_MSG_SIZE)) { |
| 1560 | return false; |
| 1561 | } |
| 1562 | |
| 1563 | *length -= bytes_read; |
| 1564 | *total_read += bytes_read; |
| 1565 | |
| 1566 | ws->state++; |
| 1567 | return true; |
| 1568 | } |
| 1569 | |
| 1570 | /* |
| 1571 | * sfe_ipv4_debug_dev_read_connections_start() |
| 1572 | * Generate part of the XML output. |
| 1573 | */ |
| 1574 | static bool sfe_ipv4_debug_dev_read_connections_start(struct sfe_ipv4 *si, char *buffer, char *msg, size_t *length, |
| 1575 | int *total_read, struct sfe_ipv4_debug_xml_write_state *ws) |
| 1576 | { |
| 1577 | int bytes_read; |
| 1578 | |
| 1579 | bytes_read = snprintf(msg, CHAR_DEV_MSG_SIZE, "\t<connections>\n"); |
| 1580 | if (copy_to_user(buffer + *total_read, msg, CHAR_DEV_MSG_SIZE)) { |
| 1581 | return false; |
| 1582 | } |
| 1583 | |
| 1584 | *length -= bytes_read; |
| 1585 | *total_read += bytes_read; |
| 1586 | |
| 1587 | ws->state++; |
| 1588 | return true; |
| 1589 | } |
| 1590 | |
| 1591 | /* |
| 1592 | * sfe_ipv4_debug_dev_read_connections_connection() |
| 1593 | * Generate part of the XML output. |
| 1594 | */ |
| 1595 | static bool sfe_ipv4_debug_dev_read_connections_connection(struct sfe_ipv4 *si, char *buffer, char *msg, size_t *length, |
| 1596 | int *total_read, struct sfe_ipv4_debug_xml_write_state *ws) |
| 1597 | { |
| 1598 | struct sfe_ipv4_connection *c; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1599 | struct sfe_ipv4_connection_match *original_cm; |
| 1600 | struct sfe_ipv4_connection_match *reply_cm; |
| 1601 | int bytes_read; |
| 1602 | int protocol; |
| 1603 | struct net_device *src_dev; |
Dave Hudson | 87973cd | 2013-10-22 16:00:04 +0100 | [diff] [blame] | 1604 | __be32 src_ip; |
| 1605 | __be32 src_ip_xlate; |
| 1606 | __be16 src_port; |
| 1607 | __be16 src_port_xlate; |
Xiaoping Fan | 6a1672f | 2016-08-17 19:58:12 -0700 | [diff] [blame] | 1608 | u64 src_rx_packets; |
| 1609 | u64 src_rx_bytes; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1610 | struct net_device *dest_dev; |
Dave Hudson | 87973cd | 2013-10-22 16:00:04 +0100 | [diff] [blame] | 1611 | __be32 dest_ip; |
| 1612 | __be32 dest_ip_xlate; |
| 1613 | __be16 dest_port; |
| 1614 | __be16 dest_port_xlate; |
Xiaoping Fan | 6a1672f | 2016-08-17 19:58:12 -0700 | [diff] [blame] | 1615 | u64 dest_rx_packets; |
| 1616 | u64 dest_rx_bytes; |
| 1617 | u64 last_sync_jiffies; |
| 1618 | u32 mark, src_priority, dest_priority, src_dscp, dest_dscp; |
Guduri Prathyusha | eb31c90 | 2021-11-10 20:18:50 +0530 | [diff] [blame^] | 1619 | u32 packet, byte, original_cm_flags; |
| 1620 | u16 pppoe_session_id; |
| 1621 | u8 pppoe_remote_mac[ETH_ALEN]; |
Xiaoping Fan | d1dc7b2 | 2015-01-23 00:43:56 -0800 | [diff] [blame] | 1622 | #ifdef CONFIG_NF_FLOW_COOKIE |
| 1623 | int src_flow_cookie, dst_flow_cookie; |
| 1624 | #endif |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1625 | |
| 1626 | spin_lock_bh(&si->lock); |
Xiaoping Fan | 3458647 | 2015-07-03 02:20:35 -0700 | [diff] [blame] | 1627 | |
| 1628 | for (c = si->all_connections_head; c; c = c->all_connections_next) { |
| 1629 | if (c->debug_read_seq < si->debug_read_seq) { |
| 1630 | c->debug_read_seq = si->debug_read_seq; |
| 1631 | break; |
| 1632 | } |
| 1633 | } |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1634 | |
| 1635 | /* |
Xiaoping Fan | 3458647 | 2015-07-03 02:20:35 -0700 | [diff] [blame] | 1636 | * If there were no connections then move to the next state. |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1637 | */ |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 1638 | if (!c || c->removed) { |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1639 | spin_unlock_bh(&si->lock); |
Xiaoping Fan | 3458647 | 2015-07-03 02:20:35 -0700 | [diff] [blame] | 1640 | ws->state++; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1641 | return true; |
| 1642 | } |
| 1643 | |
| 1644 | original_cm = c->original_match; |
| 1645 | reply_cm = c->reply_match; |
| 1646 | |
| 1647 | protocol = c->protocol; |
| 1648 | src_dev = c->original_dev; |
| 1649 | src_ip = c->src_ip; |
| 1650 | src_ip_xlate = c->src_ip_xlate; |
| 1651 | src_port = c->src_port; |
| 1652 | src_port_xlate = c->src_port_xlate; |
Xiaoping Fan | e1963d4 | 2015-08-25 17:06:19 -0700 | [diff] [blame] | 1653 | src_priority = original_cm->priority; |
| 1654 | src_dscp = original_cm->dscp >> SFE_IPV4_DSCP_SHIFT; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1655 | |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 1656 | sfe_ipv4_connection_match_update_summary_stats(original_cm, &packet, &byte); |
| 1657 | sfe_ipv4_connection_match_update_summary_stats(reply_cm, &packet, &byte); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1658 | |
| 1659 | src_rx_packets = original_cm->rx_packet_count64; |
| 1660 | src_rx_bytes = original_cm->rx_byte_count64; |
| 1661 | dest_dev = c->reply_dev; |
| 1662 | dest_ip = c->dest_ip; |
| 1663 | dest_ip_xlate = c->dest_ip_xlate; |
| 1664 | dest_port = c->dest_port; |
| 1665 | dest_port_xlate = c->dest_port_xlate; |
Xiaoping Fan | e1963d4 | 2015-08-25 17:06:19 -0700 | [diff] [blame] | 1666 | dest_priority = reply_cm->priority; |
| 1667 | dest_dscp = reply_cm->dscp >> SFE_IPV4_DSCP_SHIFT; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1668 | dest_rx_packets = reply_cm->rx_packet_count64; |
| 1669 | dest_rx_bytes = reply_cm->rx_byte_count64; |
| 1670 | last_sync_jiffies = get_jiffies_64() - c->last_sync_jiffies; |
Cristian Prundeanu | 592265e | 2013-12-26 11:01:22 -0600 | [diff] [blame] | 1671 | mark = c->mark; |
Guduri Prathyusha | eb31c90 | 2021-11-10 20:18:50 +0530 | [diff] [blame^] | 1672 | original_cm_flags = original_cm->flags; |
| 1673 | pppoe_session_id = original_cm->pppoe_session_id; |
| 1674 | ether_addr_copy(pppoe_remote_mac, original_cm->pppoe_remote_mac); |
| 1675 | |
Xiaoping Fan | d1dc7b2 | 2015-01-23 00:43:56 -0800 | [diff] [blame] | 1676 | #ifdef CONFIG_NF_FLOW_COOKIE |
| 1677 | src_flow_cookie = original_cm->flow_cookie; |
| 1678 | dst_flow_cookie = reply_cm->flow_cookie; |
| 1679 | #endif |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1680 | spin_unlock_bh(&si->lock); |
| 1681 | |
| 1682 | bytes_read = snprintf(msg, CHAR_DEV_MSG_SIZE, "\t\t<connection " |
| 1683 | "protocol=\"%u\" " |
| 1684 | "src_dev=\"%s\" " |
| 1685 | "src_ip=\"%pI4\" src_ip_xlate=\"%pI4\" " |
| 1686 | "src_port=\"%u\" src_port_xlate=\"%u\" " |
Xiaoping Fan | e1963d4 | 2015-08-25 17:06:19 -0700 | [diff] [blame] | 1687 | "src_priority=\"%u\" src_dscp=\"%u\" " |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1688 | "src_rx_pkts=\"%llu\" src_rx_bytes=\"%llu\" " |
| 1689 | "dest_dev=\"%s\" " |
| 1690 | "dest_ip=\"%pI4\" dest_ip_xlate=\"%pI4\" " |
| 1691 | "dest_port=\"%u\" dest_port_xlate=\"%u\" " |
Xiaoping Fan | e1963d4 | 2015-08-25 17:06:19 -0700 | [diff] [blame] | 1692 | "dest_priority=\"%u\" dest_dscp=\"%u\" " |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1693 | "dest_rx_pkts=\"%llu\" dest_rx_bytes=\"%llu\" " |
Xiaoping Fan | d1dc7b2 | 2015-01-23 00:43:56 -0800 | [diff] [blame] | 1694 | #ifdef CONFIG_NF_FLOW_COOKIE |
| 1695 | "src_flow_cookie=\"%d\" dst_flow_cookie=\"%d\" " |
| 1696 | #endif |
Cristian Prundeanu | 592265e | 2013-12-26 11:01:22 -0600 | [diff] [blame] | 1697 | "last_sync=\"%llu\" " |
Guduri Prathyusha | eb31c90 | 2021-11-10 20:18:50 +0530 | [diff] [blame^] | 1698 | "mark=\"%08x\" ", |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1699 | protocol, |
| 1700 | src_dev->name, |
| 1701 | &src_ip, &src_ip_xlate, |
Dave Hudson | 87973cd | 2013-10-22 16:00:04 +0100 | [diff] [blame] | 1702 | ntohs(src_port), ntohs(src_port_xlate), |
Xiaoping Fan | e1963d4 | 2015-08-25 17:06:19 -0700 | [diff] [blame] | 1703 | src_priority, src_dscp, |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1704 | src_rx_packets, src_rx_bytes, |
| 1705 | dest_dev->name, |
| 1706 | &dest_ip, &dest_ip_xlate, |
Dave Hudson | 87973cd | 2013-10-22 16:00:04 +0100 | [diff] [blame] | 1707 | ntohs(dest_port), ntohs(dest_port_xlate), |
Xiaoping Fan | e1963d4 | 2015-08-25 17:06:19 -0700 | [diff] [blame] | 1708 | dest_priority, dest_dscp, |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1709 | dest_rx_packets, dest_rx_bytes, |
Xiaoping Fan | d1dc7b2 | 2015-01-23 00:43:56 -0800 | [diff] [blame] | 1710 | #ifdef CONFIG_NF_FLOW_COOKIE |
| 1711 | src_flow_cookie, dst_flow_cookie, |
| 1712 | #endif |
Cristian Prundeanu | 592265e | 2013-12-26 11:01:22 -0600 | [diff] [blame] | 1713 | last_sync_jiffies, mark); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1714 | |
Guduri Prathyusha | eb31c90 | 2021-11-10 20:18:50 +0530 | [diff] [blame^] | 1715 | if (original_cm_flags &= (SFE_IPV4_CONNECTION_MATCH_FLAG_PPPOE_DECAP | SFE_IPV4_CONNECTION_MATCH_FLAG_PPPOE_ENCAP)) { |
| 1716 | bytes_read += snprintf(msg + bytes_read, CHAR_DEV_MSG_SIZE, "pppoe session_id=\"%u\" pppoe server MAC=\"%pM\" ", |
| 1717 | pppoe_session_id, pppoe_remote_mac); |
| 1718 | } |
| 1719 | |
| 1720 | bytes_read += snprintf(msg + bytes_read, CHAR_DEV_MSG_SIZE, "/>\n"); |
| 1721 | |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1722 | if (copy_to_user(buffer + *total_read, msg, CHAR_DEV_MSG_SIZE)) { |
| 1723 | return false; |
| 1724 | } |
| 1725 | |
| 1726 | *length -= bytes_read; |
| 1727 | *total_read += bytes_read; |
| 1728 | |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1729 | return true; |
| 1730 | } |
| 1731 | |
| 1732 | /* |
| 1733 | * sfe_ipv4_debug_dev_read_connections_end() |
| 1734 | * Generate part of the XML output. |
| 1735 | */ |
| 1736 | static bool sfe_ipv4_debug_dev_read_connections_end(struct sfe_ipv4 *si, char *buffer, char *msg, size_t *length, |
| 1737 | int *total_read, struct sfe_ipv4_debug_xml_write_state *ws) |
| 1738 | { |
| 1739 | int bytes_read; |
| 1740 | |
| 1741 | bytes_read = snprintf(msg, CHAR_DEV_MSG_SIZE, "\t</connections>\n"); |
| 1742 | if (copy_to_user(buffer + *total_read, msg, CHAR_DEV_MSG_SIZE)) { |
| 1743 | return false; |
| 1744 | } |
| 1745 | |
| 1746 | *length -= bytes_read; |
| 1747 | *total_read += bytes_read; |
| 1748 | |
| 1749 | ws->state++; |
| 1750 | return true; |
| 1751 | } |
| 1752 | |
| 1753 | /* |
| 1754 | * sfe_ipv4_debug_dev_read_exceptions_start() |
| 1755 | * Generate part of the XML output. |
| 1756 | */ |
| 1757 | static bool sfe_ipv4_debug_dev_read_exceptions_start(struct sfe_ipv4 *si, char *buffer, char *msg, size_t *length, |
| 1758 | int *total_read, struct sfe_ipv4_debug_xml_write_state *ws) |
| 1759 | { |
| 1760 | int bytes_read; |
| 1761 | |
| 1762 | bytes_read = snprintf(msg, CHAR_DEV_MSG_SIZE, "\t<exceptions>\n"); |
| 1763 | if (copy_to_user(buffer + *total_read, msg, CHAR_DEV_MSG_SIZE)) { |
| 1764 | return false; |
| 1765 | } |
| 1766 | |
| 1767 | *length -= bytes_read; |
| 1768 | *total_read += bytes_read; |
| 1769 | |
| 1770 | ws->state++; |
| 1771 | return true; |
| 1772 | } |
| 1773 | |
| 1774 | /* |
| 1775 | * sfe_ipv4_debug_dev_read_exceptions_exception() |
| 1776 | * Generate part of the XML output. |
| 1777 | */ |
| 1778 | static bool sfe_ipv4_debug_dev_read_exceptions_exception(struct sfe_ipv4 *si, char *buffer, char *msg, size_t *length, |
| 1779 | int *total_read, struct sfe_ipv4_debug_xml_write_state *ws) |
| 1780 | { |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 1781 | int i; |
| 1782 | u64 val = 0; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1783 | |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 1784 | for_each_possible_cpu(i) { |
| 1785 | const struct sfe_ipv4_stats *s = per_cpu_ptr(si->stats_pcpu, i); |
| 1786 | val += s->exception_events64[ws->iter_exception]; |
| 1787 | } |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1788 | |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 1789 | if (val) { |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1790 | int bytes_read; |
| 1791 | |
| 1792 | bytes_read = snprintf(msg, CHAR_DEV_MSG_SIZE, |
| 1793 | "\t\t<exception name=\"%s\" count=\"%llu\" />\n", |
| 1794 | sfe_ipv4_exception_events_string[ws->iter_exception], |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 1795 | val); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1796 | if (copy_to_user(buffer + *total_read, msg, CHAR_DEV_MSG_SIZE)) { |
| 1797 | return false; |
| 1798 | } |
| 1799 | |
| 1800 | *length -= bytes_read; |
| 1801 | *total_read += bytes_read; |
| 1802 | } |
| 1803 | |
| 1804 | ws->iter_exception++; |
| 1805 | if (ws->iter_exception >= SFE_IPV4_EXCEPTION_EVENT_LAST) { |
| 1806 | ws->iter_exception = 0; |
| 1807 | ws->state++; |
| 1808 | } |
| 1809 | |
| 1810 | return true; |
| 1811 | } |
| 1812 | |
| 1813 | /* |
| 1814 | * sfe_ipv4_debug_dev_read_exceptions_end() |
| 1815 | * Generate part of the XML output. |
| 1816 | */ |
| 1817 | static bool sfe_ipv4_debug_dev_read_exceptions_end(struct sfe_ipv4 *si, char *buffer, char *msg, size_t *length, |
| 1818 | int *total_read, struct sfe_ipv4_debug_xml_write_state *ws) |
| 1819 | { |
| 1820 | int bytes_read; |
| 1821 | |
| 1822 | bytes_read = snprintf(msg, CHAR_DEV_MSG_SIZE, "\t</exceptions>\n"); |
| 1823 | if (copy_to_user(buffer + *total_read, msg, CHAR_DEV_MSG_SIZE)) { |
| 1824 | return false; |
| 1825 | } |
| 1826 | |
| 1827 | *length -= bytes_read; |
| 1828 | *total_read += bytes_read; |
| 1829 | |
| 1830 | ws->state++; |
| 1831 | return true; |
| 1832 | } |
| 1833 | |
| 1834 | /* |
| 1835 | * sfe_ipv4_debug_dev_read_stats() |
| 1836 | * Generate part of the XML output. |
| 1837 | */ |
| 1838 | static bool sfe_ipv4_debug_dev_read_stats(struct sfe_ipv4 *si, char *buffer, char *msg, size_t *length, |
| 1839 | int *total_read, struct sfe_ipv4_debug_xml_write_state *ws) |
| 1840 | { |
| 1841 | int bytes_read; |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 1842 | struct sfe_ipv4_stats stats; |
| 1843 | unsigned int num_conn; |
| 1844 | |
| 1845 | sfe_ipv4_update_summary_stats(si, &stats); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1846 | |
| 1847 | spin_lock_bh(&si->lock); |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 1848 | num_conn = si->num_connections; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1849 | spin_unlock_bh(&si->lock); |
| 1850 | |
| 1851 | bytes_read = snprintf(msg, CHAR_DEV_MSG_SIZE, "\t<stats " |
| 1852 | "num_connections=\"%u\" " |
Xiaoping Fan | 5917642 | 2015-05-22 15:58:10 -0700 | [diff] [blame] | 1853 | "pkts_forwarded=\"%llu\" pkts_not_forwarded=\"%llu\" " |
| 1854 | "create_requests=\"%llu\" create_collisions=\"%llu\" " |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1855 | "create_failures=\"%llu\" " |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1856 | "destroy_requests=\"%llu\" destroy_misses=\"%llu\" " |
| 1857 | "flushes=\"%llu\" " |
| 1858 | "hash_hits=\"%llu\" hash_reorders=\"%llu\" />\n", |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 1859 | num_conn, |
| 1860 | stats.packets_forwarded64, |
| 1861 | stats.packets_not_forwarded64, |
| 1862 | stats.connection_create_requests64, |
| 1863 | stats.connection_create_collisions64, |
Ratheesh Kannoth | 89302a7 | 2021-10-20 08:10:37 +0530 | [diff] [blame] | 1864 | stats.connection_create_failures64, |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 1865 | stats.connection_destroy_requests64, |
| 1866 | stats.connection_destroy_misses64, |
| 1867 | stats.connection_flushes64, |
| 1868 | stats.connection_match_hash_hits64, |
| 1869 | stats.connection_match_hash_reorders64); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1870 | if (copy_to_user(buffer + *total_read, msg, CHAR_DEV_MSG_SIZE)) { |
| 1871 | return false; |
| 1872 | } |
| 1873 | |
| 1874 | *length -= bytes_read; |
| 1875 | *total_read += bytes_read; |
| 1876 | |
| 1877 | ws->state++; |
| 1878 | return true; |
| 1879 | } |
| 1880 | |
| 1881 | /* |
| 1882 | * sfe_ipv4_debug_dev_read_end() |
| 1883 | * Generate part of the XML output. |
| 1884 | */ |
| 1885 | static bool sfe_ipv4_debug_dev_read_end(struct sfe_ipv4 *si, char *buffer, char *msg, size_t *length, |
| 1886 | int *total_read, struct sfe_ipv4_debug_xml_write_state *ws) |
| 1887 | { |
| 1888 | int bytes_read; |
| 1889 | |
| 1890 | bytes_read = snprintf(msg, CHAR_DEV_MSG_SIZE, "</sfe_ipv4>\n"); |
| 1891 | if (copy_to_user(buffer + *total_read, msg, CHAR_DEV_MSG_SIZE)) { |
| 1892 | return false; |
| 1893 | } |
| 1894 | |
| 1895 | *length -= bytes_read; |
| 1896 | *total_read += bytes_read; |
| 1897 | |
| 1898 | ws->state++; |
| 1899 | return true; |
| 1900 | } |
| 1901 | |
| 1902 | /* |
| 1903 | * Array of write functions that write various XML elements that correspond to |
| 1904 | * our XML output state machine. |
| 1905 | */ |
Xiaoping Fan | 6a1672f | 2016-08-17 19:58:12 -0700 | [diff] [blame] | 1906 | static sfe_ipv4_debug_xml_write_method_t sfe_ipv4_debug_xml_write_methods[SFE_IPV4_DEBUG_XML_STATE_DONE] = { |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1907 | sfe_ipv4_debug_dev_read_start, |
| 1908 | sfe_ipv4_debug_dev_read_connections_start, |
| 1909 | sfe_ipv4_debug_dev_read_connections_connection, |
| 1910 | sfe_ipv4_debug_dev_read_connections_end, |
| 1911 | sfe_ipv4_debug_dev_read_exceptions_start, |
| 1912 | sfe_ipv4_debug_dev_read_exceptions_exception, |
| 1913 | sfe_ipv4_debug_dev_read_exceptions_end, |
| 1914 | sfe_ipv4_debug_dev_read_stats, |
| 1915 | sfe_ipv4_debug_dev_read_end, |
| 1916 | }; |
| 1917 | |
| 1918 | /* |
| 1919 | * sfe_ipv4_debug_dev_read() |
| 1920 | * Send info to userspace upon read request from user |
| 1921 | */ |
| 1922 | static ssize_t sfe_ipv4_debug_dev_read(struct file *filp, char *buffer, size_t length, loff_t *offset) |
| 1923 | { |
| 1924 | char msg[CHAR_DEV_MSG_SIZE]; |
| 1925 | int total_read = 0; |
| 1926 | struct sfe_ipv4_debug_xml_write_state *ws; |
| 1927 | struct sfe_ipv4 *si = &__si; |
| 1928 | |
| 1929 | ws = (struct sfe_ipv4_debug_xml_write_state *)filp->private_data; |
| 1930 | while ((ws->state != SFE_IPV4_DEBUG_XML_STATE_DONE) && (length > CHAR_DEV_MSG_SIZE)) { |
| 1931 | if ((sfe_ipv4_debug_xml_write_methods[ws->state])(si, buffer, msg, &length, &total_read, ws)) { |
| 1932 | continue; |
| 1933 | } |
| 1934 | } |
| 1935 | |
| 1936 | return total_read; |
| 1937 | } |
| 1938 | |
| 1939 | /* |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1940 | * sfe_ipv4_debug_dev_open() |
| 1941 | */ |
| 1942 | static int sfe_ipv4_debug_dev_open(struct inode *inode, struct file *file) |
| 1943 | { |
| 1944 | struct sfe_ipv4_debug_xml_write_state *ws; |
| 1945 | |
| 1946 | ws = (struct sfe_ipv4_debug_xml_write_state *)file->private_data; |
| 1947 | if (!ws) { |
| 1948 | ws = kzalloc(sizeof(struct sfe_ipv4_debug_xml_write_state), GFP_KERNEL); |
| 1949 | if (!ws) { |
| 1950 | return -ENOMEM; |
| 1951 | } |
| 1952 | |
| 1953 | ws->state = SFE_IPV4_DEBUG_XML_STATE_START; |
| 1954 | file->private_data = ws; |
| 1955 | } |
| 1956 | |
| 1957 | return 0; |
| 1958 | } |
| 1959 | |
| 1960 | /* |
| 1961 | * sfe_ipv4_debug_dev_release() |
| 1962 | */ |
| 1963 | static int sfe_ipv4_debug_dev_release(struct inode *inode, struct file *file) |
| 1964 | { |
| 1965 | struct sfe_ipv4_debug_xml_write_state *ws; |
| 1966 | |
| 1967 | ws = (struct sfe_ipv4_debug_xml_write_state *)file->private_data; |
| 1968 | if (ws) { |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1969 | /* |
| 1970 | * We've finished with our output so free the write state. |
| 1971 | */ |
| 1972 | kfree(ws); |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 1973 | file->private_data = NULL; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1974 | } |
| 1975 | |
| 1976 | return 0; |
| 1977 | } |
| 1978 | |
| 1979 | /* |
| 1980 | * File operations used in the debug char device |
| 1981 | */ |
| 1982 | static struct file_operations sfe_ipv4_debug_dev_fops = { |
| 1983 | .read = sfe_ipv4_debug_dev_read, |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 1984 | .open = sfe_ipv4_debug_dev_open, |
| 1985 | .release = sfe_ipv4_debug_dev_release |
| 1986 | }; |
| 1987 | |
Xiaoping Fan | d1dc7b2 | 2015-01-23 00:43:56 -0800 | [diff] [blame] | 1988 | #ifdef CONFIG_NF_FLOW_COOKIE |
| 1989 | /* |
| 1990 | * sfe_register_flow_cookie_cb |
| 1991 | * register a function in SFE to let SFE use this function to configure flow cookie for a flow |
| 1992 | * |
| 1993 | * Hardware driver which support flow cookie should register a callback function in SFE. Then SFE |
| 1994 | * can use this function to configure flow cookie for a flow. |
| 1995 | * return: 0, success; !=0, fail |
| 1996 | */ |
| 1997 | int sfe_register_flow_cookie_cb(flow_cookie_set_func_t cb) |
| 1998 | { |
| 1999 | struct sfe_ipv4 *si = &__si; |
| 2000 | |
| 2001 | BUG_ON(!cb); |
| 2002 | |
| 2003 | if (si->flow_cookie_set_func) { |
| 2004 | return -1; |
| 2005 | } |
| 2006 | |
| 2007 | rcu_assign_pointer(si->flow_cookie_set_func, cb); |
| 2008 | return 0; |
| 2009 | } |
| 2010 | |
| 2011 | /* |
| 2012 | * sfe_unregister_flow_cookie_cb |
| 2013 | * unregister function which is used to configure flow cookie for a flow |
| 2014 | * |
| 2015 | * return: 0, success; !=0, fail |
| 2016 | */ |
| 2017 | int sfe_unregister_flow_cookie_cb(flow_cookie_set_func_t cb) |
| 2018 | { |
| 2019 | struct sfe_ipv4 *si = &__si; |
| 2020 | |
| 2021 | RCU_INIT_POINTER(si->flow_cookie_set_func, NULL); |
| 2022 | return 0; |
| 2023 | } |
Xiaoping Fan | 640faf4 | 2015-08-28 15:50:55 -0700 | [diff] [blame] | 2024 | |
| 2025 | /* |
| 2026 | * sfe_ipv4_get_flow_cookie() |
| 2027 | */ |
| 2028 | static ssize_t sfe_ipv4_get_flow_cookie(struct device *dev, |
| 2029 | struct device_attribute *attr, |
| 2030 | char *buf) |
| 2031 | { |
| 2032 | struct sfe_ipv4 *si = &__si; |
Xiaoping Fan | 01c67cc | 2015-11-09 11:31:57 -0800 | [diff] [blame] | 2033 | return snprintf(buf, (ssize_t)PAGE_SIZE, "%d\n", si->flow_cookie_enable); |
Xiaoping Fan | 640faf4 | 2015-08-28 15:50:55 -0700 | [diff] [blame] | 2034 | } |
| 2035 | |
| 2036 | /* |
| 2037 | * sfe_ipv4_set_flow_cookie() |
| 2038 | */ |
| 2039 | static ssize_t sfe_ipv4_set_flow_cookie(struct device *dev, |
| 2040 | struct device_attribute *attr, |
| 2041 | const char *buf, size_t size) |
| 2042 | { |
| 2043 | struct sfe_ipv4 *si = &__si; |
Ken Zhu | 137722d | 2021-09-23 17:57:36 -0700 | [diff] [blame] | 2044 | si->flow_cookie_enable = simple_strtol(buf, NULL, 0); |
Xiaoping Fan | 640faf4 | 2015-08-28 15:50:55 -0700 | [diff] [blame] | 2045 | |
| 2046 | return size; |
| 2047 | } |
| 2048 | |
| 2049 | /* |
| 2050 | * sysfs attributes. |
| 2051 | */ |
| 2052 | static const struct device_attribute sfe_ipv4_flow_cookie_attr = |
Xiaoping Fan | e70da41 | 2016-02-26 16:47:57 -0800 | [diff] [blame] | 2053 | __ATTR(flow_cookie_enable, S_IWUSR | S_IRUGO, sfe_ipv4_get_flow_cookie, sfe_ipv4_set_flow_cookie); |
Xiaoping Fan | d1dc7b2 | 2015-01-23 00:43:56 -0800 | [diff] [blame] | 2054 | #endif /*CONFIG_NF_FLOW_COOKIE*/ |
| 2055 | |
Ken Zhu | 137722d | 2021-09-23 17:57:36 -0700 | [diff] [blame] | 2056 | /* |
| 2057 | * sfe_ipv4_get_cpu() |
| 2058 | */ |
| 2059 | static ssize_t sfe_ipv4_get_cpu(struct device *dev, |
| 2060 | struct device_attribute *attr, |
| 2061 | char *buf) |
| 2062 | { |
| 2063 | struct sfe_ipv4 *si = &__si; |
| 2064 | return snprintf(buf, (ssize_t)PAGE_SIZE, "%d\n", si->work_cpu); |
| 2065 | } |
| 2066 | |
| 2067 | /* |
| 2068 | * sfe_ipv4_set_cpu() |
| 2069 | */ |
| 2070 | static ssize_t sfe_ipv4_set_cpu(struct device *dev, |
| 2071 | struct device_attribute *attr, |
| 2072 | const char *buf, size_t size) |
| 2073 | { |
| 2074 | struct sfe_ipv4 *si = &__si; |
| 2075 | int work_cpu; |
| 2076 | work_cpu = simple_strtol(buf, NULL, 0); |
| 2077 | if ((work_cpu >= 0) && (work_cpu <= NR_CPUS)) { |
| 2078 | si->work_cpu = work_cpu; |
| 2079 | } else { |
| 2080 | dev_err(dev, "%s is not in valid range[0,%d]", buf, NR_CPUS); |
| 2081 | } |
| 2082 | return size; |
| 2083 | } |
| 2084 | /* |
| 2085 | * sysfs attributes. |
| 2086 | */ |
| 2087 | static const struct device_attribute sfe_ipv4_cpu_attr = |
| 2088 | __ATTR(stats_work_cpu, S_IWUSR | S_IRUGO, sfe_ipv4_get_cpu, sfe_ipv4_set_cpu); |
| 2089 | |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 2090 | /* |
| 2091 | * sfe_ipv4_conn_match_hash_init() |
| 2092 | * Initialize conn match hash lists |
| 2093 | */ |
| 2094 | static void sfe_ipv4_conn_match_hash_init(struct sfe_ipv4 *si, int len) |
| 2095 | { |
| 2096 | struct hlist_head *hash_list = si->hlist_conn_match_hash_head; |
| 2097 | int i; |
| 2098 | |
| 2099 | for (i = 0; i < len; i++) { |
| 2100 | INIT_HLIST_HEAD(&hash_list[i]); |
| 2101 | } |
| 2102 | } |
| 2103 | |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 2104 | /* |
Dave Hudson | 87973cd | 2013-10-22 16:00:04 +0100 | [diff] [blame] | 2105 | * sfe_ipv4_init() |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 2106 | */ |
Ratheesh Kannoth | 24fb1db | 2021-10-20 07:28:06 +0530 | [diff] [blame] | 2107 | int sfe_ipv4_init(void) |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 2108 | { |
| 2109 | struct sfe_ipv4 *si = &__si; |
| 2110 | int result = -1; |
| 2111 | |
Dave Hudson | dcd08fb | 2013-11-22 09:25:16 -0600 | [diff] [blame] | 2112 | DEBUG_INFO("SFE IPv4 init\n"); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 2113 | |
Ratheesh Kannoth | 94fc5b8 | 2021-10-20 07:45:06 +0530 | [diff] [blame] | 2114 | sfe_ipv4_conn_match_hash_init(si, ARRAY_SIZE(si->hlist_conn_match_hash_head)); |
| 2115 | |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 2116 | si->stats_pcpu = alloc_percpu_gfp(struct sfe_ipv4_stats, GFP_KERNEL | __GFP_ZERO); |
| 2117 | if (!si->stats_pcpu) { |
| 2118 | DEBUG_ERROR("failed to allocate stats memory for sfe_ipv4\n"); |
| 2119 | goto exit0; |
| 2120 | } |
| 2121 | |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 2122 | /* |
| 2123 | * Create sys/sfe_ipv4 |
| 2124 | */ |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 2125 | si->sys_ipv4 = kobject_create_and_add("sfe_ipv4", NULL); |
| 2126 | if (!si->sys_ipv4) { |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 2127 | DEBUG_ERROR("failed to register sfe_ipv4\n"); |
| 2128 | goto exit1; |
| 2129 | } |
| 2130 | |
| 2131 | /* |
| 2132 | * Create files, one for each parameter supported by this module. |
| 2133 | */ |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 2134 | result = sysfs_create_file(si->sys_ipv4, &sfe_ipv4_debug_dev_attr.attr); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 2135 | if (result) { |
| 2136 | DEBUG_ERROR("failed to register debug dev file: %d\n", result); |
Xiaoping Fan | 640faf4 | 2015-08-28 15:50:55 -0700 | [diff] [blame] | 2137 | goto exit2; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 2138 | } |
| 2139 | |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 2140 | result = sysfs_create_file(si->sys_ipv4, &sfe_ipv4_cpu_attr.attr); |
Ken Zhu | 137722d | 2021-09-23 17:57:36 -0700 | [diff] [blame] | 2141 | if (result) { |
| 2142 | DEBUG_ERROR("failed to register debug dev file: %d\n", result); |
| 2143 | goto exit3; |
| 2144 | } |
| 2145 | |
Xiaoping Fan | 640faf4 | 2015-08-28 15:50:55 -0700 | [diff] [blame] | 2146 | #ifdef CONFIG_NF_FLOW_COOKIE |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 2147 | result = sysfs_create_file(si->sys_ipv4, &sfe_ipv4_flow_cookie_attr.attr); |
Xiaoping Fan | 640faf4 | 2015-08-28 15:50:55 -0700 | [diff] [blame] | 2148 | if (result) { |
| 2149 | DEBUG_ERROR("failed to register flow cookie enable file: %d\n", result); |
Ken Zhu | 137722d | 2021-09-23 17:57:36 -0700 | [diff] [blame] | 2150 | goto exit4; |
Xiaoping Fan | 640faf4 | 2015-08-28 15:50:55 -0700 | [diff] [blame] | 2151 | } |
| 2152 | #endif /* CONFIG_NF_FLOW_COOKIE */ |
| 2153 | |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 2154 | /* |
| 2155 | * Register our debug char device. |
| 2156 | */ |
| 2157 | result = register_chrdev(0, "sfe_ipv4", &sfe_ipv4_debug_dev_fops); |
| 2158 | if (result < 0) { |
| 2159 | DEBUG_ERROR("Failed to register chrdev: %d\n", result); |
Ken Zhu | 137722d | 2021-09-23 17:57:36 -0700 | [diff] [blame] | 2160 | goto exit5; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 2161 | } |
| 2162 | |
| 2163 | si->debug_dev = result; |
Ken Zhu | 137722d | 2021-09-23 17:57:36 -0700 | [diff] [blame] | 2164 | si->work_cpu = WORK_CPU_UNBOUND; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 2165 | |
| 2166 | /* |
Ken Zhu | 137722d | 2021-09-23 17:57:36 -0700 | [diff] [blame] | 2167 | * Create a work to handle periodic statistics. |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 2168 | */ |
Ken Zhu | 137722d | 2021-09-23 17:57:36 -0700 | [diff] [blame] | 2169 | INIT_DELAYED_WORK(&(si->sync_dwork), sfe_ipv4_periodic_sync); |
| 2170 | schedule_delayed_work_on(si->work_cpu, &(si->sync_dwork), ((HZ + 99) / 100)); |
| 2171 | |
Dave Hudson | 87973cd | 2013-10-22 16:00:04 +0100 | [diff] [blame] | 2172 | spin_lock_init(&si->lock); |
Dave Hudson | 87973cd | 2013-10-22 16:00:04 +0100 | [diff] [blame] | 2173 | return 0; |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 2174 | |
Ken Zhu | 137722d | 2021-09-23 17:57:36 -0700 | [diff] [blame] | 2175 | exit5: |
Xiaoping Fan | 640faf4 | 2015-08-28 15:50:55 -0700 | [diff] [blame] | 2176 | #ifdef CONFIG_NF_FLOW_COOKIE |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 2177 | sysfs_remove_file(si->sys_ipv4, &sfe_ipv4_flow_cookie_attr.attr); |
Xiaoping Fan | 640faf4 | 2015-08-28 15:50:55 -0700 | [diff] [blame] | 2178 | |
Ken Zhu | 137722d | 2021-09-23 17:57:36 -0700 | [diff] [blame] | 2179 | exit4: |
Xiaoping Fan | 640faf4 | 2015-08-28 15:50:55 -0700 | [diff] [blame] | 2180 | #endif /* CONFIG_NF_FLOW_COOKIE */ |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 2181 | sysfs_remove_file(si->sys_ipv4, &sfe_ipv4_cpu_attr.attr); |
Ken Zhu | 137722d | 2021-09-23 17:57:36 -0700 | [diff] [blame] | 2182 | exit3: |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 2183 | sysfs_remove_file(si->sys_ipv4, &sfe_ipv4_debug_dev_attr.attr); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 2184 | |
Xiaoping Fan | 640faf4 | 2015-08-28 15:50:55 -0700 | [diff] [blame] | 2185 | exit2: |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 2186 | kobject_put(si->sys_ipv4); |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 2187 | |
| 2188 | exit1: |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 2189 | free_percpu(si->stats_pcpu); |
| 2190 | |
| 2191 | exit0: |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 2192 | return result; |
| 2193 | } |
| 2194 | |
| 2195 | /* |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 2196 | * sfe_ipv4_exit() |
| 2197 | */ |
Ratheesh Kannoth | 24fb1db | 2021-10-20 07:28:06 +0530 | [diff] [blame] | 2198 | void sfe_ipv4_exit(void) |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 2199 | { |
Dave Hudson | 87973cd | 2013-10-22 16:00:04 +0100 | [diff] [blame] | 2200 | struct sfe_ipv4 *si = &__si; |
| 2201 | |
Dave Hudson | dcd08fb | 2013-11-22 09:25:16 -0600 | [diff] [blame] | 2202 | DEBUG_INFO("SFE IPv4 exit\n"); |
Dave Hudson | 87973cd | 2013-10-22 16:00:04 +0100 | [diff] [blame] | 2203 | /* |
| 2204 | * Destroy all connections. |
| 2205 | */ |
Dave Hudson | dcd08fb | 2013-11-22 09:25:16 -0600 | [diff] [blame] | 2206 | sfe_ipv4_destroy_all_rules_for_dev(NULL); |
Dave Hudson | 87973cd | 2013-10-22 16:00:04 +0100 | [diff] [blame] | 2207 | |
Ken Zhu | 137722d | 2021-09-23 17:57:36 -0700 | [diff] [blame] | 2208 | cancel_delayed_work_sync(&si->sync_dwork); |
Dave Hudson | 87973cd | 2013-10-22 16:00:04 +0100 | [diff] [blame] | 2209 | |
Dave Hudson | 87973cd | 2013-10-22 16:00:04 +0100 | [diff] [blame] | 2210 | unregister_chrdev(si->debug_dev, "sfe_ipv4"); |
| 2211 | |
Xiaoping Fan | 640faf4 | 2015-08-28 15:50:55 -0700 | [diff] [blame] | 2212 | #ifdef CONFIG_NF_FLOW_COOKIE |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 2213 | sysfs_remove_file(si->sys_ipv4, &sfe_ipv4_flow_cookie_attr.attr); |
Xiaoping Fan | 640faf4 | 2015-08-28 15:50:55 -0700 | [diff] [blame] | 2214 | #endif /* CONFIG_NF_FLOW_COOKIE */ |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 2215 | sysfs_remove_file(si->sys_ipv4, &sfe_ipv4_debug_dev_attr.attr); |
| 2216 | sysfs_remove_file(si->sys_ipv4, &sfe_ipv4_cpu_attr.attr); |
Dave Hudson | 87973cd | 2013-10-22 16:00:04 +0100 | [diff] [blame] | 2217 | |
Ratheesh Kannoth | 6307bec | 2021-11-25 08:26:39 +0530 | [diff] [blame] | 2218 | kobject_put(si->sys_ipv4); |
Dave Hudson | 87973cd | 2013-10-22 16:00:04 +0100 | [diff] [blame] | 2219 | |
Ratheesh Kannoth | 3aeb289 | 2021-10-20 07:57:15 +0530 | [diff] [blame] | 2220 | free_percpu(si->stats_pcpu); |
| 2221 | |
Dave Hudson | aaf97ca | 2013-06-13 17:52:29 +0100 | [diff] [blame] | 2222 | } |
| 2223 | |
Xiaoping Fan | d1dc7b2 | 2015-01-23 00:43:56 -0800 | [diff] [blame] | 2224 | #ifdef CONFIG_NF_FLOW_COOKIE |
| 2225 | EXPORT_SYMBOL(sfe_register_flow_cookie_cb); |
| 2226 | EXPORT_SYMBOL(sfe_unregister_flow_cookie_cb); |
| 2227 | #endif |