blob: 868e9964a36a9a0d3756529d9a5b1a93592ea74e [file] [log] [blame]
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001/*
2 * sfe_ipv4.c
3 * Shortcut forwarding engine - IPv4 edition.
4 *
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +05305 * Copyright (c) 2013-2016, 2019-2020, The Linux Foundation. All rights reserved.
6 * Copyright (c) 2021 Qualcomm Innovation Center, Inc. All rights reserved.
7 *
8 * Permission to use, copy, modify, and/or distribute this software for any
9 * purpose with or without fee is hereby granted, provided that the above
10 * copyright notice and this permission notice appear in all copies.
11 *
Xiaoping Fana42c68b2015-08-07 18:00:39 -070012 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +053017 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
Dave Hudsonaaf97ca2013-06-13 17:52:29 +010019 */
Matthew McClintocka3221942014-01-16 11:44:26 -060020
Dave Hudsonaaf97ca2013-06-13 17:52:29 +010021#include <linux/module.h>
Dave Hudsondcd08fb2013-11-22 09:25:16 -060022#include <linux/sysfs.h>
Dave Hudsonaaf97ca2013-06-13 17:52:29 +010023#include <linux/skbuff.h>
24#include <linux/icmp.h>
Dave Hudsonaaf97ca2013-06-13 17:52:29 +010025#include <net/tcp.h>
Dave Hudsondcd08fb2013-11-22 09:25:16 -060026#include <linux/etherdevice.h>
Tian Yang45f39c82020-10-06 14:07:47 -070027#include <linux/version.h>
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +053028#include <linux/lockdep.h>
Dave Hudsonaaf97ca2013-06-13 17:52:29 +010029
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +053030#include "sfe_debug.h"
Ratheesh Kannoth89302a72021-10-20 08:10:37 +053031#include "sfe_api.h"
Dave Hudsondcd08fb2013-11-22 09:25:16 -060032#include "sfe.h"
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +053033#include "sfe_flow_cookie.h"
34#include "sfe_ipv4.h"
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +053035#include "sfe_ipv4_udp.h"
36#include "sfe_ipv4_tcp.h"
37#include "sfe_ipv4_icmp.h"
Dave Hudsonaaf97ca2013-06-13 17:52:29 +010038
39static char *sfe_ipv4_exception_events_string[SFE_IPV4_EXCEPTION_EVENT_LAST] = {
40 "UDP_HEADER_INCOMPLETE",
41 "UDP_NO_CONNECTION",
42 "UDP_IP_OPTIONS_OR_INITIAL_FRAGMENT",
43 "UDP_SMALL_TTL",
44 "UDP_NEEDS_FRAGMENTATION",
45 "TCP_HEADER_INCOMPLETE",
46 "TCP_NO_CONNECTION_SLOW_FLAGS",
47 "TCP_NO_CONNECTION_FAST_FLAGS",
48 "TCP_IP_OPTIONS_OR_INITIAL_FRAGMENT",
49 "TCP_SMALL_TTL",
50 "TCP_NEEDS_FRAGMENTATION",
51 "TCP_FLAGS",
52 "TCP_SEQ_EXCEEDS_RIGHT_EDGE",
53 "TCP_SMALL_DATA_OFFS",
54 "TCP_BAD_SACK",
55 "TCP_BIG_DATA_OFFS",
56 "TCP_SEQ_BEFORE_LEFT_EDGE",
57 "TCP_ACK_EXCEEDS_RIGHT_EDGE",
58 "TCP_ACK_BEFORE_LEFT_EDGE",
59 "ICMP_HEADER_INCOMPLETE",
60 "ICMP_UNHANDLED_TYPE",
61 "ICMP_IPV4_HEADER_INCOMPLETE",
62 "ICMP_IPV4_NON_V4",
63 "ICMP_IPV4_IP_OPTIONS_INCOMPLETE",
64 "ICMP_IPV4_UDP_HEADER_INCOMPLETE",
65 "ICMP_IPV4_TCP_HEADER_INCOMPLETE",
66 "ICMP_IPV4_UNHANDLED_PROTOCOL",
67 "ICMP_NO_CONNECTION",
68 "ICMP_FLUSHED_CONNECTION",
69 "HEADER_INCOMPLETE",
Ratheesh Kannoth43d64f82021-10-20 08:23:29 +053070 "HEADER_CSUM_BAD",
Dave Hudsonaaf97ca2013-06-13 17:52:29 +010071 "BAD_TOTAL_LENGTH",
72 "NON_V4",
73 "NON_INITIAL_FRAGMENT",
74 "DATAGRAM_INCOMPLETE",
75 "IP_OPTIONS_INCOMPLETE",
76 "UNHANDLED_PROTOCOL"
77};
78
Xiaoping Fan6a1672f2016-08-17 19:58:12 -070079static struct sfe_ipv4 __si;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +010080
81/*
Dave Hudsonaaf97ca2013-06-13 17:52:29 +010082 * sfe_ipv4_gen_ip_csum()
83 * Generate the IP checksum for an IPv4 header.
84 *
85 * Note that this function assumes that we have only 20 bytes of IP header.
86 */
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +053087u16 sfe_ipv4_gen_ip_csum(struct iphdr *iph)
Dave Hudsonaaf97ca2013-06-13 17:52:29 +010088{
Xiaoping Fan6a1672f2016-08-17 19:58:12 -070089 u32 sum;
90 u16 *i = (u16 *)iph;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +010091
92 iph->check = 0;
93
94 /*
95 * Generate the sum.
96 */
97 sum = i[0] + i[1] + i[2] + i[3] + i[4] + i[5] + i[6] + i[7] + i[8] + i[9];
98
99 /*
100 * Fold it to ones-complement form.
101 */
102 sum = (sum & 0xffff) + (sum >> 16);
103 sum = (sum & 0xffff) + (sum >> 16);
104
Xiaoping Fan6a1672f2016-08-17 19:58:12 -0700105 return (u16)sum ^ 0xffff;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100106}
107
108/*
109 * sfe_ipv4_get_connection_match_hash()
110 * Generate the hash used in connection match lookups.
111 */
Xiaoping Fan6a1672f2016-08-17 19:58:12 -0700112static inline unsigned int sfe_ipv4_get_connection_match_hash(struct net_device *dev, u8 protocol,
Dave Hudson87973cd2013-10-22 16:00:04 +0100113 __be32 src_ip, __be16 src_port,
114 __be32 dest_ip, __be16 dest_port)
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100115{
116 size_t dev_addr = (size_t)dev;
Xiaoping Fan6a1672f2016-08-17 19:58:12 -0700117 u32 hash = ((u32)dev_addr) ^ ntohl(src_ip ^ dest_ip) ^ protocol ^ ntohs(src_port ^ dest_port);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100118 return ((hash >> SFE_IPV4_CONNECTION_HASH_SHIFT) ^ hash) & SFE_IPV4_CONNECTION_HASH_MASK;
119}
120
121/*
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +0530122 * sfe_ipv4_find_connection_match_rcu()
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100123 * Get the IPv4 flow match info that corresponds to a particular 5-tuple.
124 *
125 * On entry we must be holding the lock that protects the hash table.
126 */
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +0530127struct sfe_ipv4_connection_match *
128sfe_ipv4_find_connection_match_rcu(struct sfe_ipv4 *si, struct net_device *dev, u8 protocol,
Dave Hudson87973cd2013-10-22 16:00:04 +0100129 __be32 src_ip, __be16 src_port,
130 __be32 dest_ip, __be16 dest_port)
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100131{
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530132 struct sfe_ipv4_connection_match *cm = NULL;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100133 unsigned int conn_match_idx;
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530134 struct hlist_head *lhead;
135
136 WARN_ON_ONCE(!rcu_read_lock_held());
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100137
138 conn_match_idx = sfe_ipv4_get_connection_match_hash(dev, protocol, src_ip, src_port, dest_ip, dest_port);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100139
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530140 lhead = &si->hlist_conn_match_hash_head[conn_match_idx];
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100141
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530142 hlist_for_each_entry_rcu(cm, lhead, hnode) {
143 if (cm->match_src_port != src_port
144 || cm->match_dest_port != dest_port
145 || cm->match_src_ip != src_ip
146 || cm->match_dest_ip != dest_ip
147 || cm->match_protocol != protocol
148 || cm->match_dev != dev) {
149 continue;
150 }
151
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +0530152 this_cpu_inc(si->stats_pcpu->connection_match_hash_hits64);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100153
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530154 break;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100155 }
156
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100157 return cm;
158}
159
160/*
161 * sfe_ipv4_connection_match_update_summary_stats()
162 * Update the summary stats for a connection match entry.
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530163 *
164 * Stats are incremented atomically. So use atomic substraction to update summary
165 * stats.
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100166 */
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530167static inline void sfe_ipv4_connection_match_update_summary_stats(struct sfe_ipv4_connection_match *cm,
168 u32 *packets, u32 *bytes)
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100169{
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530170 u32 packet_count, byte_count;
171
172 packet_count = atomic_read(&cm->rx_packet_count);
173 cm->rx_packet_count64 += packet_count;
174 atomic_sub(packet_count, &cm->rx_packet_count);
175
176 byte_count = atomic_read(&cm->rx_byte_count);
177 cm->rx_byte_count64 += byte_count;
178 atomic_sub(byte_count, &cm->rx_byte_count);
179
180 *packets = packet_count;
181 *bytes = byte_count;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100182}
183
184/*
185 * sfe_ipv4_connection_match_compute_translations()
186 * Compute port and address translations for a connection match entry.
187 */
188static void sfe_ipv4_connection_match_compute_translations(struct sfe_ipv4_connection_match *cm)
189{
190 /*
191 * Before we insert the entry look to see if this is tagged as doing address
192 * translations. If it is then work out the adjustment that we need to apply
193 * to the transport checksum.
194 */
195 if (cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_XLATE_SRC) {
196 /*
197 * Precompute an incremental checksum adjustment so we can
198 * edit packets in this stream very quickly. The algorithm is from RFC1624.
199 */
Xiaoping Fan6a1672f2016-08-17 19:58:12 -0700200 u16 src_ip_hi = cm->match_src_ip >> 16;
201 u16 src_ip_lo = cm->match_src_ip & 0xffff;
202 u32 xlate_src_ip = ~cm->xlate_src_ip;
203 u16 xlate_src_ip_hi = xlate_src_ip >> 16;
204 u16 xlate_src_ip_lo = xlate_src_ip & 0xffff;
205 u16 xlate_src_port = ~cm->xlate_src_port;
206 u32 adj;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100207
208 /*
209 * When we compute this fold it down to a 16-bit offset
210 * as that way we can avoid having to do a double
211 * folding of the twos-complement result because the
212 * addition of 2 16-bit values cannot cause a double
213 * wrap-around!
214 */
215 adj = src_ip_hi + src_ip_lo + cm->match_src_port
216 + xlate_src_ip_hi + xlate_src_ip_lo + xlate_src_port;
217 adj = (adj & 0xffff) + (adj >> 16);
218 adj = (adj & 0xffff) + (adj >> 16);
Xiaoping Fan6a1672f2016-08-17 19:58:12 -0700219 cm->xlate_src_csum_adjustment = (u16)adj;
Nicolas Costaac2979c2014-01-14 10:35:24 -0600220
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100221 }
222
223 if (cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_XLATE_DEST) {
224 /*
225 * Precompute an incremental checksum adjustment so we can
226 * edit packets in this stream very quickly. The algorithm is from RFC1624.
227 */
Xiaoping Fan6a1672f2016-08-17 19:58:12 -0700228 u16 dest_ip_hi = cm->match_dest_ip >> 16;
229 u16 dest_ip_lo = cm->match_dest_ip & 0xffff;
230 u32 xlate_dest_ip = ~cm->xlate_dest_ip;
231 u16 xlate_dest_ip_hi = xlate_dest_ip >> 16;
232 u16 xlate_dest_ip_lo = xlate_dest_ip & 0xffff;
233 u16 xlate_dest_port = ~cm->xlate_dest_port;
234 u32 adj;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100235
236 /*
237 * When we compute this fold it down to a 16-bit offset
238 * as that way we can avoid having to do a double
239 * folding of the twos-complement result because the
240 * addition of 2 16-bit values cannot cause a double
241 * wrap-around!
242 */
243 adj = dest_ip_hi + dest_ip_lo + cm->match_dest_port
244 + xlate_dest_ip_hi + xlate_dest_ip_lo + xlate_dest_port;
245 adj = (adj & 0xffff) + (adj >> 16);
246 adj = (adj & 0xffff) + (adj >> 16);
Xiaoping Fan6a1672f2016-08-17 19:58:12 -0700247 cm->xlate_dest_csum_adjustment = (u16)adj;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100248 }
Xiaoping Fanad755af2015-04-01 16:58:46 -0700249
250 if (cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_XLATE_SRC) {
Xiaoping Fan6a1672f2016-08-17 19:58:12 -0700251 u32 adj = ~cm->match_src_ip + cm->xlate_src_ip;
Xiaoping Fanad755af2015-04-01 16:58:46 -0700252 if (adj < cm->xlate_src_ip) {
253 adj++;
254 }
255
256 adj = (adj & 0xffff) + (adj >> 16);
257 adj = (adj & 0xffff) + (adj >> 16);
Xiaoping Fan6a1672f2016-08-17 19:58:12 -0700258 cm->xlate_src_partial_csum_adjustment = (u16)adj;
Xiaoping Fanad755af2015-04-01 16:58:46 -0700259 }
260
261 if (cm->flags & SFE_IPV4_CONNECTION_MATCH_FLAG_XLATE_DEST) {
Xiaoping Fan6a1672f2016-08-17 19:58:12 -0700262 u32 adj = ~cm->match_dest_ip + cm->xlate_dest_ip;
Xiaoping Fanad755af2015-04-01 16:58:46 -0700263 if (adj < cm->xlate_dest_ip) {
264 adj++;
265 }
266
267 adj = (adj & 0xffff) + (adj >> 16);
268 adj = (adj & 0xffff) + (adj >> 16);
Xiaoping Fan6a1672f2016-08-17 19:58:12 -0700269 cm->xlate_dest_partial_csum_adjustment = (u16)adj;
Xiaoping Fanad755af2015-04-01 16:58:46 -0700270 }
271
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100272}
273
274/*
275 * sfe_ipv4_update_summary_stats()
276 * Update the summary stats.
277 */
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +0530278static void sfe_ipv4_update_summary_stats(struct sfe_ipv4 *si, struct sfe_ipv4_stats *stats)
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100279{
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +0530280 int i = 0;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100281
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +0530282 memset(stats, 0, sizeof(*stats));
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100283
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +0530284 for_each_possible_cpu(i) {
285 const struct sfe_ipv4_stats *s = per_cpu_ptr(si->stats_pcpu, i);
286
287 stats->connection_create_requests64 += s->connection_create_requests64;
288 stats->connection_create_collisions64 += s->connection_create_collisions64;
Ratheesh Kannoth89302a72021-10-20 08:10:37 +0530289 stats->connection_create_failures64 += s->connection_create_failures64;
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +0530290 stats->connection_destroy_requests64 += s->connection_destroy_requests64;
291 stats->connection_destroy_misses64 += s->connection_destroy_misses64;
292 stats->connection_match_hash_hits64 += s->connection_match_hash_hits64;
293 stats->connection_match_hash_reorders64 += s->connection_match_hash_reorders64;
294 stats->connection_flushes64 += s->connection_flushes64;
295 stats->packets_forwarded64 += s->packets_forwarded64;
296 stats->packets_not_forwarded64 += s->packets_not_forwarded64;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100297 }
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +0530298
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100299}
300
301/*
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +0530302 * sfe_ipv4_insert_connection_match()
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100303 * Insert a connection match into the hash.
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100304 */
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +0530305static inline void sfe_ipv4_insert_connection_match(struct sfe_ipv4 *si,
Xiaoping Fan6a1672f2016-08-17 19:58:12 -0700306 struct sfe_ipv4_connection_match *cm)
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100307{
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100308 unsigned int conn_match_idx
309 = sfe_ipv4_get_connection_match_hash(cm->match_dev, cm->match_protocol,
310 cm->match_src_ip, cm->match_src_port,
311 cm->match_dest_ip, cm->match_dest_port);
Xiaoping Fan6a1672f2016-08-17 19:58:12 -0700312
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530313 lockdep_assert_held(&si->lock);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100314
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530315 hlist_add_head_rcu(&cm->hnode, &si->hlist_conn_match_hash_head[conn_match_idx]);
Xiaoping Fand1dc7b22015-01-23 00:43:56 -0800316#ifdef CONFIG_NF_FLOW_COOKIE
Xiaoping Fan640faf42015-08-28 15:50:55 -0700317 if (!si->flow_cookie_enable)
318 return;
319
Xiaoping Fand1dc7b22015-01-23 00:43:56 -0800320 /*
321 * Configure hardware to put a flow cookie in packet of this flow,
322 * then we can accelerate the lookup process when we received this packet.
323 */
324 for (conn_match_idx = 1; conn_match_idx < SFE_FLOW_COOKIE_SIZE; conn_match_idx++) {
325 struct sfe_flow_cookie_entry *entry = &si->sfe_flow_cookie_table[conn_match_idx];
326
327 if ((NULL == entry->match) && time_is_before_jiffies(entry->last_clean_time + HZ)) {
328 flow_cookie_set_func_t func;
329
330 rcu_read_lock();
331 func = rcu_dereference(si->flow_cookie_set_func);
332 if (func) {
Xiaoping Fan59176422015-05-22 15:58:10 -0700333 if (!func(cm->match_protocol, cm->match_src_ip, cm->match_src_port,
Xiaoping Fand1dc7b22015-01-23 00:43:56 -0800334 cm->match_dest_ip, cm->match_dest_port, conn_match_idx)) {
335 entry->match = cm;
336 cm->flow_cookie = conn_match_idx;
337 }
338 }
339 rcu_read_unlock();
340
341 break;
342 }
343 }
344#endif
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100345}
346
347/*
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +0530348 * sfe_ipv4_remove_connection_match()
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100349 * Remove a connection match object from the hash.
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100350 */
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +0530351static inline void sfe_ipv4_remove_connection_match(struct sfe_ipv4 *si, struct sfe_ipv4_connection_match *cm)
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100352{
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530353
354 lockdep_assert_held(&si->lock);
355
Xiaoping Fand1dc7b22015-01-23 00:43:56 -0800356#ifdef CONFIG_NF_FLOW_COOKIE
Xiaoping Fan640faf42015-08-28 15:50:55 -0700357 if (si->flow_cookie_enable) {
358 /*
359 * Tell hardware that we no longer need a flow cookie in packet of this flow
360 */
361 unsigned int conn_match_idx;
Xiaoping Fand1dc7b22015-01-23 00:43:56 -0800362
Xiaoping Fan640faf42015-08-28 15:50:55 -0700363 for (conn_match_idx = 1; conn_match_idx < SFE_FLOW_COOKIE_SIZE; conn_match_idx++) {
364 struct sfe_flow_cookie_entry *entry = &si->sfe_flow_cookie_table[conn_match_idx];
Xiaoping Fand1dc7b22015-01-23 00:43:56 -0800365
Xiaoping Fan640faf42015-08-28 15:50:55 -0700366 if (cm == entry->match) {
367 flow_cookie_set_func_t func;
Xiaoping Fand1dc7b22015-01-23 00:43:56 -0800368
Xiaoping Fan640faf42015-08-28 15:50:55 -0700369 rcu_read_lock();
370 func = rcu_dereference(si->flow_cookie_set_func);
371 if (func) {
372 func(cm->match_protocol, cm->match_src_ip, cm->match_src_port,
373 cm->match_dest_ip, cm->match_dest_port, 0);
374 }
375 rcu_read_unlock();
376
377 cm->flow_cookie = 0;
378 entry->match = NULL;
379 entry->last_clean_time = jiffies;
380 break;
Xiaoping Fand1dc7b22015-01-23 00:43:56 -0800381 }
Xiaoping Fand1dc7b22015-01-23 00:43:56 -0800382 }
383 }
384#endif
385
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530386 hlist_del_init_rcu(&cm->hnode);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100387
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100388}
389
390/*
391 * sfe_ipv4_get_connection_hash()
392 * Generate the hash used in connection lookups.
393 */
Xiaoping Fan6a1672f2016-08-17 19:58:12 -0700394static inline unsigned int sfe_ipv4_get_connection_hash(u8 protocol, __be32 src_ip, __be16 src_port,
Dave Hudson87973cd2013-10-22 16:00:04 +0100395 __be32 dest_ip, __be16 dest_port)
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100396{
Xiaoping Fan6a1672f2016-08-17 19:58:12 -0700397 u32 hash = ntohl(src_ip ^ dest_ip) ^ protocol ^ ntohs(src_port ^ dest_port);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100398 return ((hash >> SFE_IPV4_CONNECTION_HASH_SHIFT) ^ hash) & SFE_IPV4_CONNECTION_HASH_MASK;
399}
400
401/*
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +0530402 * sfe_ipv4_find_connection()
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100403 * Get the IPv4 connection info that corresponds to a particular 5-tuple.
404 *
405 * On entry we must be holding the lock that protects the hash table.
406 */
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +0530407static inline struct sfe_ipv4_connection *sfe_ipv4_find_connection(struct sfe_ipv4 *si, u32 protocol,
Dave Hudson87973cd2013-10-22 16:00:04 +0100408 __be32 src_ip, __be16 src_port,
409 __be32 dest_ip, __be16 dest_port)
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100410{
411 struct sfe_ipv4_connection *c;
412 unsigned int conn_idx = sfe_ipv4_get_connection_hash(protocol, src_ip, src_port, dest_ip, dest_port);
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530413
414 lockdep_assert_held(&si->lock);
415
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100416 c = si->conn_hash[conn_idx];
417
418 /*
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100419 * Will need connection entry for next create/destroy metadata,
420 * So no need to re-order entry for these requests
421 */
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530422 while (c) {
423 if ((c->src_port == src_port)
424 && (c->dest_port == dest_port)
425 && (c->src_ip == src_ip)
426 && (c->dest_ip == dest_ip)
427 && (c->protocol == protocol)) {
428 return c;
429 }
430
431 c = c->next;
432 }
433
434 return NULL;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100435}
436
437/*
Matthew McClintockbe7b47d2013-11-27 13:26:23 -0600438 * sfe_ipv4_mark_rule()
439 * Updates the mark for a current offloaded connection
440 *
441 * Will take hash lock upon entry
442 */
Xiaoping Fand44a5b42015-05-26 17:37:37 -0700443void sfe_ipv4_mark_rule(struct sfe_connection_mark *mark)
Matthew McClintockbe7b47d2013-11-27 13:26:23 -0600444{
445 struct sfe_ipv4 *si = &__si;
446 struct sfe_ipv4_connection *c;
Matthew McClintockdb5ac512014-01-16 17:01:40 -0600447
Xiaoping Fan3c423e32015-07-03 03:09:29 -0700448 spin_lock_bh(&si->lock);
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +0530449 c = sfe_ipv4_find_connection(si, mark->protocol,
Xiaoping Fand44a5b42015-05-26 17:37:37 -0700450 mark->src_ip.ip, mark->src_port,
451 mark->dest_ip.ip, mark->dest_port);
Matthew McClintockbe7b47d2013-11-27 13:26:23 -0600452 if (c) {
Nicolas Costaf53d6fe2014-01-13 16:03:46 -0600453 WARN_ON((0 != c->mark) && (0 == mark->mark));
Matthew McClintockbe7b47d2013-11-27 13:26:23 -0600454 c->mark = mark->mark;
455 }
Xiaoping Fan3c423e32015-07-03 03:09:29 -0700456 spin_unlock_bh(&si->lock);
Xiaoping Fan6a1672f2016-08-17 19:58:12 -0700457
458 if (c) {
459 DEBUG_TRACE("Matching connection found for mark, "
460 "setting from %08x to %08x\n",
461 c->mark, mark->mark);
462 }
Matthew McClintockbe7b47d2013-11-27 13:26:23 -0600463}
464
465/*
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +0530466 * sfe_ipv4_insert_connection()
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100467 * Insert a connection into the hash.
468 *
469 * On entry we must be holding the lock that protects the hash table.
470 */
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +0530471static void sfe_ipv4_insert_connection(struct sfe_ipv4 *si, struct sfe_ipv4_connection *c)
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100472{
473 struct sfe_ipv4_connection **hash_head;
474 struct sfe_ipv4_connection *prev_head;
475 unsigned int conn_idx;
476
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530477 lockdep_assert_held(&si->lock);
478
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100479 /*
480 * Insert entry into the connection hash.
481 */
482 conn_idx = sfe_ipv4_get_connection_hash(c->protocol, c->src_ip, c->src_port,
483 c->dest_ip, c->dest_port);
484 hash_head = &si->conn_hash[conn_idx];
485 prev_head = *hash_head;
486 c->prev = NULL;
487 if (prev_head) {
488 prev_head->prev = c;
489 }
490
491 c->next = prev_head;
492 *hash_head = c;
493
494 /*
495 * Insert entry into the "all connections" list.
496 */
497 if (si->all_connections_tail) {
498 c->all_connections_prev = si->all_connections_tail;
499 si->all_connections_tail->all_connections_next = c;
500 } else {
501 c->all_connections_prev = NULL;
502 si->all_connections_head = c;
503 }
504
505 si->all_connections_tail = c;
506 c->all_connections_next = NULL;
507 si->num_connections++;
508
509 /*
510 * Insert the connection match objects too.
511 */
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +0530512 sfe_ipv4_insert_connection_match(si, c->original_match);
513 sfe_ipv4_insert_connection_match(si, c->reply_match);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100514}
515
516/*
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +0530517 * sfe_ipv4_remove_connection()
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100518 * Remove a sfe_ipv4_connection object from the hash.
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100519 */
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +0530520bool sfe_ipv4_remove_connection(struct sfe_ipv4 *si, struct sfe_ipv4_connection *c)
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100521{
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530522 lockdep_assert_held(&si->lock);
523
524 if (c->removed) {
525 DEBUG_ERROR("%px: Connection has been removed already\n", c);
526 return false;
527 }
528
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100529 /*
530 * Remove the connection match objects.
531 */
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +0530532 sfe_ipv4_remove_connection_match(si, c->reply_match);
533 sfe_ipv4_remove_connection_match(si, c->original_match);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100534
535 /*
536 * Unlink the connection.
537 */
538 if (c->prev) {
539 c->prev->next = c->next;
540 } else {
541 unsigned int conn_idx = sfe_ipv4_get_connection_hash(c->protocol, c->src_ip, c->src_port,
542 c->dest_ip, c->dest_port);
543 si->conn_hash[conn_idx] = c->next;
544 }
545
546 if (c->next) {
547 c->next->prev = c->prev;
548 }
Xiaoping Fan34586472015-07-03 02:20:35 -0700549
550 /*
551 * Unlink connection from all_connections list
552 */
553 if (c->all_connections_prev) {
554 c->all_connections_prev->all_connections_next = c->all_connections_next;
555 } else {
556 si->all_connections_head = c->all_connections_next;
557 }
558
559 if (c->all_connections_next) {
560 c->all_connections_next->all_connections_prev = c->all_connections_prev;
561 } else {
562 si->all_connections_tail = c->all_connections_prev;
563 }
564
Ken Zhudc423672021-09-02 18:27:01 -0700565 /*
566 * If I am the next sync connection, move the sync to my next or head.
567 */
568 if (unlikely(si->wc_next == c)) {
569 si->wc_next = c->all_connections_next;
570 }
571
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530572 c->removed = true;
Xiaoping Fan34586472015-07-03 02:20:35 -0700573 si->num_connections--;
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530574 return true;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100575}
576
577/*
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +0530578 * sfe_ipv4_gen_sync_connection()
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100579 * Sync a connection.
580 *
581 * On entry to this function we expect that the lock for the connection is either
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530582 * already held (while called from sfe_ipv4_periodic_sync() or isn't required
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +0530583 * (while called from sfe_ipv4_flush_connection())
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100584 */
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +0530585static void sfe_ipv4_gen_sync_connection(struct sfe_ipv4 *si, struct sfe_ipv4_connection *c,
Xiaoping Fan99cb4c12015-08-21 19:07:32 -0700586 struct sfe_connection_sync *sis, sfe_sync_reason_t reason,
Xiaoping Fan6a1672f2016-08-17 19:58:12 -0700587 u64 now_jiffies)
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100588{
589 struct sfe_ipv4_connection_match *original_cm;
590 struct sfe_ipv4_connection_match *reply_cm;
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530591 u32 packet_count, byte_count;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100592
593 /*
594 * Fill in the update message.
595 */
Xiaoping Fand44a5b42015-05-26 17:37:37 -0700596 sis->is_v6 = 0;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100597 sis->protocol = c->protocol;
Xiaoping Fand44a5b42015-05-26 17:37:37 -0700598 sis->src_ip.ip = c->src_ip;
Xiaoping Fan99cb4c12015-08-21 19:07:32 -0700599 sis->src_ip_xlate.ip = c->src_ip_xlate;
Xiaoping Fand44a5b42015-05-26 17:37:37 -0700600 sis->dest_ip.ip = c->dest_ip;
Xiaoping Fan99cb4c12015-08-21 19:07:32 -0700601 sis->dest_ip_xlate.ip = c->dest_ip_xlate;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100602 sis->src_port = c->src_port;
Xiaoping Fan99cb4c12015-08-21 19:07:32 -0700603 sis->src_port_xlate = c->src_port_xlate;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100604 sis->dest_port = c->dest_port;
Xiaoping Fan99cb4c12015-08-21 19:07:32 -0700605 sis->dest_port_xlate = c->dest_port_xlate;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100606
607 original_cm = c->original_match;
608 reply_cm = c->reply_match;
609 sis->src_td_max_window = original_cm->protocol_state.tcp.max_win;
610 sis->src_td_end = original_cm->protocol_state.tcp.end;
611 sis->src_td_max_end = original_cm->protocol_state.tcp.max_end;
612 sis->dest_td_max_window = reply_cm->protocol_state.tcp.max_win;
613 sis->dest_td_end = reply_cm->protocol_state.tcp.end;
614 sis->dest_td_max_end = reply_cm->protocol_state.tcp.max_end;
615
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530616 sfe_ipv4_connection_match_update_summary_stats(original_cm, &packet_count, &byte_count);
617 sis->src_new_packet_count = packet_count;
618 sis->src_new_byte_count = byte_count;
Matthew McClintockd0cdb802014-02-24 16:30:35 -0600619
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530620 sfe_ipv4_connection_match_update_summary_stats(reply_cm, &packet_count, &byte_count);
621 sis->dest_new_packet_count = packet_count;
622 sis->dest_new_byte_count = byte_count;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100623
Matthew McClintockd0cdb802014-02-24 16:30:35 -0600624 sis->src_dev = original_cm->match_dev;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100625 sis->src_packet_count = original_cm->rx_packet_count64;
626 sis->src_byte_count = original_cm->rx_byte_count64;
Matthew McClintockd0cdb802014-02-24 16:30:35 -0600627
628 sis->dest_dev = reply_cm->match_dev;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100629 sis->dest_packet_count = reply_cm->rx_packet_count64;
630 sis->dest_byte_count = reply_cm->rx_byte_count64;
631
Xiaoping Fan99cb4c12015-08-21 19:07:32 -0700632 sis->reason = reason;
633
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100634 /*
635 * Get the time increment since our last sync.
636 */
637 sis->delta_jiffies = now_jiffies - c->last_sync_jiffies;
638 c->last_sync_jiffies = now_jiffies;
639}
640
641/*
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +0530642 * sfe_ipv4_free_connection_rcu()
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530643 * Called at RCU qs state to free the connection object.
644 */
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +0530645static void sfe_ipv4_free_connection_rcu(struct rcu_head *head)
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530646{
647 struct sfe_ipv4_connection *c;
648
649 /*
650 * We dont need spin lock as the connection is already removed from link list
651 */
652 c = container_of(head, struct sfe_ipv4_connection, rcu);
653
654 BUG_ON(!c->removed);
655
656 DEBUG_TRACE("%px: connecton has been deleted\n", c);
657
658 /*
659 * Release our hold of the source and dest devices and free the memory
660 * for our connection objects.
661 */
662 dev_put(c->original_dev);
663 dev_put(c->reply_dev);
664 kfree(c->original_match);
665 kfree(c->reply_match);
666 kfree(c);
667}
668
669/*
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +0530670 * sfe_ipv4_flush_connection()
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100671 * Flush a connection and free all associated resources.
672 *
673 * We need to be called with bottom halves disabled locally as we need to acquire
674 * the connection hash lock and release it again. In general we're actually called
675 * from within a BH and so we're fine, but we're also called when connections are
676 * torn down.
677 */
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +0530678void sfe_ipv4_flush_connection(struct sfe_ipv4 *si,
Xiaoping Fan6a1672f2016-08-17 19:58:12 -0700679 struct sfe_ipv4_connection *c,
680 sfe_sync_reason_t reason)
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100681{
Xiaoping Fan6a1672f2016-08-17 19:58:12 -0700682 u64 now_jiffies;
Xiaoping Fand44a5b42015-05-26 17:37:37 -0700683 sfe_sync_rule_callback_t sync_rule_callback;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100684
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530685 BUG_ON(!c->removed);
686
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +0530687 this_cpu_inc(si->stats_pcpu->connection_flushes64);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100688
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530689 rcu_read_lock();
690 sync_rule_callback = rcu_dereference(si->sync_rule_callback);
691
692 /*
693 * Generate a sync message and then sync.
694 */
Dave Hudsondcd08fb2013-11-22 09:25:16 -0600695 if (sync_rule_callback) {
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530696 struct sfe_connection_sync sis;
Dave Hudsondcd08fb2013-11-22 09:25:16 -0600697 now_jiffies = get_jiffies_64();
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +0530698 sfe_ipv4_gen_sync_connection(si, c, &sis, reason, now_jiffies);
Dave Hudsondcd08fb2013-11-22 09:25:16 -0600699 sync_rule_callback(&sis);
700 }
701
702 rcu_read_unlock();
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100703
704 /*
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100705 * Release our hold of the source and dest devices and free the memory
706 * for our connection objects.
707 */
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +0530708 call_rcu(&c->rcu, sfe_ipv4_free_connection_rcu);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100709}
710
711/*
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +0530712 * sfe_ipv4_exception_stats_inc()
713 * Increment exception stats.
714 */
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +0530715void sfe_ipv4_exception_stats_inc(struct sfe_ipv4 *si, enum sfe_ipv4_exception_events reason)
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +0530716{
717 struct sfe_ipv4_stats *stats = this_cpu_ptr(si->stats_pcpu);
718 stats->exception_events64[reason]++;
719 stats->packets_not_forwarded64++;
720}
721
722/*
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100723 * sfe_ipv4_recv()
Matthew McClintocka8ad7962014-01-16 16:49:30 -0600724 * Handle packet receives and forwaring.
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100725 *
726 * Returns 1 if the packet is forwarded or 0 if it isn't.
727 */
Dave Hudsondcd08fb2013-11-22 09:25:16 -0600728int sfe_ipv4_recv(struct net_device *dev, struct sk_buff *skb)
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100729{
730 struct sfe_ipv4 *si = &__si;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100731 unsigned int len;
732 unsigned int tot_len;
733 unsigned int frag_off;
734 unsigned int ihl;
735 bool flush_on_find;
736 bool ip_options;
Ratheesh Kannoth741f7992021-10-20 07:39:52 +0530737 struct iphdr *iph;
Xiaoping Fan6a1672f2016-08-17 19:58:12 -0700738 u32 protocol;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100739
740 /*
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100741 * Check that we have space for an IP header here.
742 */
743 len = skb->len;
Ratheesh Kannoth741f7992021-10-20 07:39:52 +0530744 if (unlikely(!pskb_may_pull(skb, sizeof(struct iphdr)))) {
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +0530745 sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_HEADER_INCOMPLETE);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100746 DEBUG_TRACE("len: %u is too short\n", len);
747 return 0;
748 }
749
750 /*
Ratheesh Kannotha3cf0e02021-12-09 09:44:10 +0530751 * Validate ip csum if necessary. If ip_summed is set to CHECKSUM_UNNECESSARY, it is assumed
752 * that the L3 checksum is validated by the Rx interface or the tunnel interface that has
753 * generated the packet.
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100754 */
Ratheesh Kannoth741f7992021-10-20 07:39:52 +0530755 iph = (struct iphdr *)skb->data;
Ratheesh Kannoth43d64f82021-10-20 08:23:29 +0530756 if (unlikely(skb->ip_summed != CHECKSUM_UNNECESSARY) && (ip_fast_csum((u8 *)iph, iph->ihl))) {
757 sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_HEADER_CSUM_BAD);
758
759 DEBUG_TRACE("Bad IPv4 header csum: 0x%x\n", iph->check);
760 return 0;
761 }
762
763 /*
764 * Check that our "total length" is large enough for an IP header.
765 */
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100766 tot_len = ntohs(iph->tot_len);
Ratheesh Kannoth741f7992021-10-20 07:39:52 +0530767 if (unlikely(tot_len < sizeof(struct iphdr))) {
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100768
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +0530769 sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_BAD_TOTAL_LENGTH);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100770 DEBUG_TRACE("tot_len: %u is too short\n", tot_len);
771 return 0;
772 }
773
774 /*
775 * Is our IP version wrong?
776 */
777 if (unlikely(iph->version != 4)) {
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +0530778 sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_NON_V4);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100779 DEBUG_TRACE("IP version: %u\n", iph->version);
780 return 0;
781 }
782
783 /*
784 * Does our datagram fit inside the skb?
785 */
786 if (unlikely(tot_len > len)) {
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100787
788 DEBUG_TRACE("tot_len: %u, exceeds len: %u\n", tot_len, len);
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +0530789 sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_DATAGRAM_INCOMPLETE);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100790 return 0;
791 }
792
793 /*
794 * Do we have a non-initial fragment?
Nicolas Costaac2979c2014-01-14 10:35:24 -0600795 */
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100796 frag_off = ntohs(iph->frag_off);
797 if (unlikely(frag_off & IP_OFFSET)) {
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +0530798 sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_NON_INITIAL_FRAGMENT);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100799 DEBUG_TRACE("non-initial fragment\n");
800 return 0;
801 }
802
803 /*
804 * If we have a (first) fragment then mark it to cause any connection to flush.
805 */
806 flush_on_find = unlikely(frag_off & IP_MF) ? true : false;
807
808 /*
809 * Do we have any IP options? That's definite a slow path! If we do have IP
810 * options we need to recheck our header size.
811 */
812 ihl = iph->ihl << 2;
Ratheesh Kannoth741f7992021-10-20 07:39:52 +0530813 ip_options = unlikely(ihl != sizeof(struct iphdr)) ? true : false;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100814 if (unlikely(ip_options)) {
815 if (unlikely(len < ihl)) {
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100816
817 DEBUG_TRACE("len: %u is too short for header of size: %u\n", len, ihl);
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +0530818 sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_IP_OPTIONS_INCOMPLETE);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100819 return 0;
820 }
821
822 flush_on_find = true;
823 }
824
825 protocol = iph->protocol;
826 if (IPPROTO_UDP == protocol) {
827 return sfe_ipv4_recv_udp(si, skb, dev, len, iph, ihl, flush_on_find);
828 }
829
830 if (IPPROTO_TCP == protocol) {
831 return sfe_ipv4_recv_tcp(si, skb, dev, len, iph, ihl, flush_on_find);
832 }
833
834 if (IPPROTO_ICMP == protocol) {
835 return sfe_ipv4_recv_icmp(si, skb, dev, len, iph, ihl);
836 }
837
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +0530838 sfe_ipv4_exception_stats_inc(si, SFE_IPV4_EXCEPTION_EVENT_UNHANDLED_PROTOCOL);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100839
840 DEBUG_TRACE("not UDP, TCP or ICMP: %u\n", protocol);
841 return 0;
842}
843
Nicolas Costa436926b2014-01-14 10:36:22 -0600844static void
845sfe_ipv4_update_tcp_state(struct sfe_ipv4_connection *c,
Ratheesh Kannoth89302a72021-10-20 08:10:37 +0530846 struct sfe_ipv4_rule_create_msg *msg)
Nicolas Costa436926b2014-01-14 10:36:22 -0600847{
848 struct sfe_ipv4_connection_match *orig_cm;
849 struct sfe_ipv4_connection_match *repl_cm;
850 struct sfe_ipv4_tcp_connection_match *orig_tcp;
851 struct sfe_ipv4_tcp_connection_match *repl_tcp;
852
853 orig_cm = c->original_match;
854 repl_cm = c->reply_match;
855 orig_tcp = &orig_cm->protocol_state.tcp;
856 repl_tcp = &repl_cm->protocol_state.tcp;
857
858 /* update orig */
Ratheesh Kannoth89302a72021-10-20 08:10:37 +0530859 if (orig_tcp->max_win < msg->tcp_rule.flow_max_window) {
860 orig_tcp->max_win = msg->tcp_rule.flow_max_window;
Nicolas Costa436926b2014-01-14 10:36:22 -0600861 }
Ratheesh Kannoth89302a72021-10-20 08:10:37 +0530862 if ((s32)(orig_tcp->end - msg->tcp_rule.flow_end) < 0) {
863 orig_tcp->end = msg->tcp_rule.flow_end;
Nicolas Costa436926b2014-01-14 10:36:22 -0600864 }
Ratheesh Kannoth89302a72021-10-20 08:10:37 +0530865 if ((s32)(orig_tcp->max_end - msg->tcp_rule.flow_max_end) < 0) {
866 orig_tcp->max_end = msg->tcp_rule.flow_max_end;
Nicolas Costa436926b2014-01-14 10:36:22 -0600867 }
868
869 /* update reply */
Ratheesh Kannoth89302a72021-10-20 08:10:37 +0530870 if (repl_tcp->max_win < msg->tcp_rule.return_max_window) {
871 repl_tcp->max_win = msg->tcp_rule.return_max_window;
Nicolas Costa436926b2014-01-14 10:36:22 -0600872 }
Ratheesh Kannoth89302a72021-10-20 08:10:37 +0530873 if ((s32)(repl_tcp->end - msg->tcp_rule.return_end) < 0) {
874 repl_tcp->end = msg->tcp_rule.return_end;
Nicolas Costa436926b2014-01-14 10:36:22 -0600875 }
Ratheesh Kannoth89302a72021-10-20 08:10:37 +0530876 if ((s32)(repl_tcp->max_end - msg->tcp_rule.return_max_end) < 0) {
877 repl_tcp->max_end = msg->tcp_rule.return_max_end;
Nicolas Costa436926b2014-01-14 10:36:22 -0600878 }
879
880 /* update match flags */
881 orig_cm->flags &= ~SFE_IPV4_CONNECTION_MATCH_FLAG_NO_SEQ_CHECK;
882 repl_cm->flags &= ~SFE_IPV4_CONNECTION_MATCH_FLAG_NO_SEQ_CHECK;
Ratheesh Kannoth89302a72021-10-20 08:10:37 +0530883 if (msg->rule_flags & SFE_RULE_CREATE_FLAG_NO_SEQ_CHECK) {
884
Nicolas Costa436926b2014-01-14 10:36:22 -0600885 orig_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_NO_SEQ_CHECK;
886 repl_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_NO_SEQ_CHECK;
887 }
888}
889
890static void
891sfe_ipv4_update_protocol_state(struct sfe_ipv4_connection *c,
Ratheesh Kannoth89302a72021-10-20 08:10:37 +0530892 struct sfe_ipv4_rule_create_msg *msg)
Nicolas Costa436926b2014-01-14 10:36:22 -0600893{
Ratheesh Kannoth89302a72021-10-20 08:10:37 +0530894 switch (msg->tuple.protocol) {
Nicolas Costa436926b2014-01-14 10:36:22 -0600895 case IPPROTO_TCP:
Ratheesh Kannoth89302a72021-10-20 08:10:37 +0530896 sfe_ipv4_update_tcp_state(c, msg);
Nicolas Costa436926b2014-01-14 10:36:22 -0600897 break;
898 }
899}
900
Ratheesh Kannoth89302a72021-10-20 08:10:37 +0530901void sfe_ipv4_update_rule(struct sfe_ipv4_rule_create_msg *msg)
Nicolas Costa436926b2014-01-14 10:36:22 -0600902{
903 struct sfe_ipv4_connection *c;
904 struct sfe_ipv4 *si = &__si;
905
906 spin_lock_bh(&si->lock);
907
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +0530908 c = sfe_ipv4_find_connection(si,
Ratheesh Kannoth89302a72021-10-20 08:10:37 +0530909 msg->tuple.protocol,
910 msg->tuple.flow_ip,
911 msg->tuple.flow_ident,
912 msg->tuple.return_ip,
913 msg->tuple.return_ident);
Nicolas Costa436926b2014-01-14 10:36:22 -0600914 if (c != NULL) {
Ratheesh Kannoth89302a72021-10-20 08:10:37 +0530915 sfe_ipv4_update_protocol_state(c, msg);
Nicolas Costa436926b2014-01-14 10:36:22 -0600916 }
917
918 spin_unlock_bh(&si->lock);
919}
920
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100921/*
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100922 * sfe_ipv4_create_rule()
923 * Create a forwarding rule.
924 */
Ratheesh Kannoth89302a72021-10-20 08:10:37 +0530925int sfe_ipv4_create_rule(struct sfe_ipv4_rule_create_msg *msg)
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100926{
Dave Hudsondcd08fb2013-11-22 09:25:16 -0600927 struct sfe_ipv4 *si = &__si;
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530928 struct sfe_ipv4_connection *c, *c_old;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100929 struct sfe_ipv4_connection_match *original_cm;
930 struct sfe_ipv4_connection_match *reply_cm;
Matthew McClintockdb5ac512014-01-16 17:01:40 -0600931 struct net_device *dest_dev;
932 struct net_device *src_dev;
Ratheesh Kannoth89302a72021-10-20 08:10:37 +0530933 struct sfe_ipv4_5tuple *tuple = &msg->tuple;
Suruchi Sumanc1a4a612021-10-21 14:50:23 +0530934 s32 flow_interface_num = msg->conn_rule.flow_top_interface_num;
935 s32 return_interface_num = msg->conn_rule.return_top_interface_num;
Matthew McClintockdb5ac512014-01-16 17:01:40 -0600936
Suruchi Sumanc1a4a612021-10-21 14:50:23 +0530937 if (msg->rule_flags & SFE_RULE_CREATE_FLAG_USE_FLOW_BOTTOM_INTERFACE) {
938 flow_interface_num = msg->conn_rule.flow_interface_num;
939 }
940
941 if (msg->rule_flags & SFE_RULE_CREATE_FLAG_USE_RETURN_BOTTOM_INTERFACE) {
942 return_interface_num = msg->conn_rule.return_interface_num;
943 }
944
945 src_dev = dev_get_by_index(&init_net, flow_interface_num);
Ratheesh Kannoth89302a72021-10-20 08:10:37 +0530946 if (!src_dev) {
947 DEBUG_WARN("%px: Unable to find src_dev corresponding to %d\n", msg,
Suruchi Sumanc1a4a612021-10-21 14:50:23 +0530948 flow_interface_num);
Ratheesh Kannoth89302a72021-10-20 08:10:37 +0530949 this_cpu_inc(si->stats_pcpu->connection_create_failures64);
950 return -EINVAL;
951 }
952
Suruchi Sumanc1a4a612021-10-21 14:50:23 +0530953 dest_dev = dev_get_by_index(&init_net, return_interface_num);
Ratheesh Kannoth89302a72021-10-20 08:10:37 +0530954 if (!dest_dev) {
955 DEBUG_WARN("%px: Unable to find dest_dev corresponding to %d\n", msg,
Suruchi Sumanc1a4a612021-10-21 14:50:23 +0530956 return_interface_num);
Ratheesh Kannoth89302a72021-10-20 08:10:37 +0530957 this_cpu_inc(si->stats_pcpu->connection_create_failures64);
958 dev_put(src_dev);
959 return -EINVAL;
960 }
Dave Hudsonaaf97ca2013-06-13 17:52:29 +0100961
Matthew McClintock389b42a2014-09-24 14:05:51 -0500962 if (unlikely((dest_dev->reg_state != NETREG_REGISTERED) ||
963 (src_dev->reg_state != NETREG_REGISTERED))) {
Ratheesh Kannoth89302a72021-10-20 08:10:37 +0530964 dev_put(src_dev);
965 dev_put(dest_dev);
966 DEBUG_WARN("%px: src_dev=%s and dest_dev=%s are unregistered\n", msg,
967 src_dev->name, dest_dev->name);
968 this_cpu_inc(si->stats_pcpu->connection_create_failures64);
Matthew McClintock389b42a2014-09-24 14:05:51 -0500969 return -EINVAL;
970 }
971
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530972 /*
973 * Allocate the various connection tracking objects.
974 */
975 c = (struct sfe_ipv4_connection *)kmalloc(sizeof(struct sfe_ipv4_connection), GFP_ATOMIC);
976 if (unlikely(!c)) {
Ratheesh Kannoth89302a72021-10-20 08:10:37 +0530977 DEBUG_WARN("%px: memory allocation of connection entry failed\n", msg);
978 this_cpu_inc(si->stats_pcpu->connection_create_failures64);
979 dev_put(src_dev);
980 dev_put(dest_dev);
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530981 return -ENOMEM;
982 }
983
984 original_cm = (struct sfe_ipv4_connection_match *)kmalloc(sizeof(struct sfe_ipv4_connection_match), GFP_ATOMIC);
985 if (unlikely(!original_cm)) {
Ratheesh Kannoth89302a72021-10-20 08:10:37 +0530986 DEBUG_WARN("%px: memory allocation of connection match entry failed\n", msg);
987 this_cpu_inc(si->stats_pcpu->connection_create_failures64);
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530988 kfree(c);
Ratheesh Kannoth89302a72021-10-20 08:10:37 +0530989 dev_put(src_dev);
990 dev_put(dest_dev);
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530991 return -ENOMEM;
992 }
993
994 reply_cm = (struct sfe_ipv4_connection_match *)kmalloc(sizeof(struct sfe_ipv4_connection_match), GFP_ATOMIC);
995 if (unlikely(!reply_cm)) {
Ratheesh Kannoth89302a72021-10-20 08:10:37 +0530996 DEBUG_WARN("%px: memory allocation of connection match entry failed\n", msg);
997 this_cpu_inc(si->stats_pcpu->connection_create_failures64);
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530998 kfree(original_cm);
999 kfree(c);
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301000 dev_put(src_dev);
1001 dev_put(dest_dev);
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +05301002 return -ENOMEM;
1003 }
1004
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +05301005 this_cpu_inc(si->stats_pcpu->connection_create_requests64);
1006
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001007 spin_lock_bh(&si->lock);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001008
1009 /*
Nicolas Costa436926b2014-01-14 10:36:22 -06001010 * Check to see if there is already a flow that matches the rule we're
1011 * trying to create. If there is then we can't create a new one.
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001012 */
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +05301013 c_old = sfe_ipv4_find_connection(si,
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301014 msg->tuple.protocol,
1015 msg->tuple.flow_ip,
1016 msg->tuple.flow_ident,
1017 msg->tuple.return_ip,
1018 msg->tuple.return_ident);
1019
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +05301020 if (c_old != NULL) {
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +05301021 this_cpu_inc(si->stats_pcpu->connection_create_collisions64);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001022
1023 /*
Nicolas Costa436926b2014-01-14 10:36:22 -06001024 * If we already have the flow then it's likely that this
1025 * request to create the connection rule contains more
1026 * up-to-date information. Check and update accordingly.
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001027 */
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301028 sfe_ipv4_update_protocol_state(c, msg);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001029 spin_unlock_bh(&si->lock);
1030
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +05301031 kfree(reply_cm);
1032 kfree(original_cm);
1033 kfree(c);
1034
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301035 dev_put(src_dev);
1036 dev_put(dest_dev);
1037
1038 DEBUG_TRACE("connection already exists - p:%d\n"
1039 " s: %s:%pM:%pI4:%u, d: %s:%pM:%pI4:%u\n",
1040 tuple->protocol,
1041 src_dev->name, msg->conn_rule.flow_mac, &tuple->flow_ip, ntohs(tuple->flow_ident),
1042 dest_dev->name, msg->conn_rule.return_mac, &tuple->return_ip, ntohs(tuple->return_ident));
1043
Nicolas Costa514fde02014-01-13 15:50:29 -06001044 return -EADDRINUSE;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001045 }
1046
1047 /*
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001048 * Fill in the "original" direction connection matching object.
1049 * Note that the transmit MAC address is "dest_mac_xlate" because
1050 * we always know both ends of a connection by their translated
1051 * addresses and not their public addresses.
1052 */
Matthew McClintockdb5ac512014-01-16 17:01:40 -06001053 original_cm->match_dev = src_dev;
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301054 original_cm->match_protocol = tuple->protocol;
1055 original_cm->match_src_ip = tuple->flow_ip;
1056 original_cm->match_src_port = tuple->flow_ident;
1057 original_cm->match_dest_ip = tuple->return_ip;
1058 original_cm->match_dest_port = tuple->return_ident;
1059
1060 original_cm->xlate_src_ip = msg->conn_rule.flow_ip_xlate;
1061 original_cm->xlate_src_port = msg->conn_rule.flow_ident_xlate;
1062 original_cm->xlate_dest_ip = msg->conn_rule.return_ip_xlate;
1063 original_cm->xlate_dest_port =msg->conn_rule.return_ident_xlate;
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +05301064 atomic_set(&original_cm->rx_packet_count, 0);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001065 original_cm->rx_packet_count64 = 0;
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +05301066 atomic_set(&original_cm->rx_byte_count, 0);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001067 original_cm->rx_byte_count64 = 0;
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301068
Matthew McClintockdb5ac512014-01-16 17:01:40 -06001069 original_cm->xmit_dev = dest_dev;
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301070 original_cm->xmit_dev_mtu = msg->conn_rule.return_mtu;
1071
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001072 original_cm->connection = c;
1073 original_cm->counter_match = reply_cm;
1074 original_cm->flags = 0;
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301075
1076 if (msg->valid_flags & SFE_RULE_CREATE_QOS_VALID) {
1077 original_cm->priority = msg->qos_rule.flow_qos_tag;
Xiaoping Fane1963d42015-08-25 17:06:19 -07001078 original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_PRIORITY_REMARK;
1079 }
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301080
1081 if (msg->valid_flags & SFE_RULE_CREATE_DSCP_MARKING_VALID) {
1082 original_cm->dscp = msg->dscp_rule.flow_dscp << SFE_IPV4_DSCP_SHIFT;
Xiaoping Fane1963d42015-08-25 17:06:19 -07001083 original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_DSCP_REMARK;
1084 }
Ratheesh Kannotha3cf0e02021-12-09 09:44:10 +05301085
Xiaoping Fand1dc7b22015-01-23 00:43:56 -08001086#ifdef CONFIG_NF_FLOW_COOKIE
1087 original_cm->flow_cookie = 0;
1088#endif
Zhi Chen8748eb32015-06-18 12:58:48 -07001089#ifdef CONFIG_XFRM
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301090 if (msg->valid_flags & SFE_RULE_CREATE_DIRECTION_VALID) {
1091 original_cm->flow_accel = msg->direction_rule.flow_accel;
1092 } else {
1093 original_cm->flow_accel = 1;
1094 }
Zhi Chen8748eb32015-06-18 12:58:48 -07001095#endif
Ratheesh Kannotha3cf0e02021-12-09 09:44:10 +05301096 /*
1097 * If l2_features are disabled and flow uses l2 features such as macvlan/bridge/pppoe/vlan,
1098 * bottom interfaces are expected to be disabled in the flow rule and always top interfaces
1099 * are used. In such cases, do not use HW csum offload. csum offload is used only when we
1100 * are sending directly to the destination interface that supports it.
1101 */
1102 if (likely(dest_dev->features & NETIF_F_HW_CSUM)) {
1103 if ((msg->conn_rule.return_top_interface_num == msg->conn_rule.return_interface_num) ||
1104 (msg->rule_flags & SFE_RULE_CREATE_FLAG_USE_RETURN_BOTTOM_INTERFACE)) {
1105 original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_CSUM_OFFLOAD;
1106 }
1107 }
Matthew McClintockdb5ac512014-01-16 17:01:40 -06001108
1109 /*
Guduri Prathyushaeb31c902021-11-10 20:18:50 +05301110 * Adding PPPoE parameters to original and reply entries based on the direction where
1111 * PPPoE header is valid in ECM rule.
1112 *
1113 * If PPPoE is valid in flow direction (from interface is PPPoE), then
1114 * original cm will have PPPoE at ingress (strip PPPoE header)
1115 * reply cm will have PPPoE at egress (add PPPoE header)
1116 *
1117 * If PPPoE is valid in return direction (to interface is PPPoE), then
1118 * original cm will have PPPoE at egress (add PPPoE header)
1119 * reply cm will have PPPoE at ingress (strip PPPoE header)
1120 */
1121 if (msg->valid_flags & SFE_RULE_CREATE_PPPOE_DECAP_VALID) {
1122 original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_PPPOE_DECAP;
1123 original_cm->pppoe_session_id = msg->pppoe_rule.flow_pppoe_session_id;
1124 ether_addr_copy(original_cm->pppoe_remote_mac, msg->pppoe_rule.flow_pppoe_remote_mac);
1125
1126 reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_PPPOE_ENCAP;
1127 reply_cm->pppoe_session_id = msg->pppoe_rule.flow_pppoe_session_id;
1128 ether_addr_copy(reply_cm->pppoe_remote_mac, msg->pppoe_rule.flow_pppoe_remote_mac);
1129 }
1130
1131 if (msg->valid_flags & SFE_RULE_CREATE_PPPOE_ENCAP_VALID) {
1132 original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_PPPOE_ENCAP;
1133 original_cm->pppoe_session_id = msg->pppoe_rule.return_pppoe_session_id;
1134 ether_addr_copy(original_cm->pppoe_remote_mac, msg->pppoe_rule.return_pppoe_remote_mac);
1135
1136 reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_PPPOE_DECAP;
1137 reply_cm->pppoe_session_id = msg->pppoe_rule.return_pppoe_session_id;
1138 ether_addr_copy(reply_cm->pppoe_remote_mac, msg->pppoe_rule.return_pppoe_remote_mac);
1139 }
1140
1141 /*
Ken Zhubbf49652021-09-12 15:33:09 -07001142 * For the non-arp interface, we don't write L2 HDR.
Matthew McClintockdb5ac512014-01-16 17:01:40 -06001143 */
Ken Zhubbf49652021-09-12 15:33:09 -07001144 if (!(dest_dev->flags & IFF_NOARP)) {
Ratheesh Kannoth29140aa2021-10-20 08:25:02 +05301145
1146 /*
1147 * Check whether the rule has configured a specific source MAC address to use.
1148 * This is needed when virtual L3 interfaces such as br-lan, macvlan, vlan are used during egress
1149 */
1150 if ((msg->valid_flags & SFE_RULE_CREATE_SRC_MAC_VALID) &&
1151 (msg->src_mac_rule.mac_valid_flags & SFE_SRC_MAC_RETURN_VALID)) {
1152 ether_addr_copy((u8 *)original_cm->xmit_src_mac, (u8 *)msg->src_mac_rule.return_src_mac);
1153 } else {
1154 ether_addr_copy((u8 *)original_cm->xmit_src_mac, (u8 *)dest_dev->dev_addr);
1155 }
1156
1157 ether_addr_copy((u8 *)original_cm->xmit_dest_mac, (u8 *)msg->conn_rule.return_mac);
1158
Matthew McClintockdb5ac512014-01-16 17:01:40 -06001159 original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_L2_HDR;
1160
1161 /*
1162 * If our dev writes Ethernet headers then we can write a really fast
1163 * version.
1164 */
1165 if (dest_dev->header_ops) {
1166 if (dest_dev->header_ops->create == eth_header) {
1167 original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_FAST_ETH_HDR;
1168 }
1169 }
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001170 }
1171
1172 /*
1173 * Fill in the "reply" direction connection matching object.
1174 */
Matthew McClintockdb5ac512014-01-16 17:01:40 -06001175 reply_cm->match_dev = dest_dev;
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301176 reply_cm->match_protocol = tuple->protocol;
1177 reply_cm->match_src_ip = msg->conn_rule.return_ip_xlate;
1178 reply_cm->match_src_port = msg->conn_rule.return_ident_xlate;
1179 reply_cm->match_dest_ip = msg->conn_rule.flow_ip_xlate;
1180 reply_cm->match_dest_port = msg->conn_rule.flow_ident_xlate;
1181
1182 reply_cm->xlate_src_ip = tuple->return_ip;
1183 reply_cm->xlate_src_port = tuple->return_ident;
1184 reply_cm->xlate_dest_ip = tuple->flow_ip;
1185 reply_cm->xlate_dest_port = tuple->flow_ident;;
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +05301186
1187 atomic_set(&reply_cm->rx_packet_count, 0);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001188 reply_cm->rx_packet_count64 = 0;
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +05301189 atomic_set(&reply_cm->rx_byte_count, 0);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001190 reply_cm->rx_byte_count64 = 0;
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +05301191
Matthew McClintockdb5ac512014-01-16 17:01:40 -06001192 reply_cm->xmit_dev = src_dev;
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301193 reply_cm->xmit_dev_mtu = msg->conn_rule.flow_mtu;
Ratheesh Kannoth29140aa2021-10-20 08:25:02 +05301194
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001195 reply_cm->connection = c;
1196 reply_cm->counter_match = original_cm;
1197 reply_cm->flags = 0;
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301198 if (msg->valid_flags & SFE_RULE_CREATE_QOS_VALID) {
1199 reply_cm->priority = msg->qos_rule.return_qos_tag;
Xiaoping Fane1963d42015-08-25 17:06:19 -07001200 reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_PRIORITY_REMARK;
1201 }
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301202 if (msg->valid_flags & SFE_RULE_CREATE_DSCP_MARKING_VALID) {
1203 reply_cm->dscp = msg->dscp_rule.return_dscp << SFE_IPV4_DSCP_SHIFT;
Xiaoping Fane1963d42015-08-25 17:06:19 -07001204 reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_DSCP_REMARK;
1205 }
Xiaoping Fand1dc7b22015-01-23 00:43:56 -08001206#ifdef CONFIG_NF_FLOW_COOKIE
1207 reply_cm->flow_cookie = 0;
1208#endif
Zhi Chen8748eb32015-06-18 12:58:48 -07001209#ifdef CONFIG_XFRM
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301210 if (msg->valid_flags & SFE_RULE_CREATE_DIRECTION_VALID) {
1211 reply_cm->flow_accel = msg->direction_rule.return_accel;
1212 } else {
1213 reply_cm->flow_accel = 1;
1214 }
1215
Zhi Chen8748eb32015-06-18 12:58:48 -07001216#endif
Ratheesh Kannotha3cf0e02021-12-09 09:44:10 +05301217 /*
1218 * If l2_features are disabled and flow uses l2 features such as macvlan/bridge/pppoe/vlan,
1219 * bottom interfaces are expected to be disabled in the flow rule and always top interfaces
1220 * are used. In such cases, do not use HW csum offload. csum offload is used only when we
1221 * are sending directly to the destination interface that supports it.
1222 */
1223 if (likely(src_dev->features & NETIF_F_HW_CSUM)) {
1224 if ((msg->conn_rule.flow_top_interface_num == msg->conn_rule.flow_interface_num) ||
1225 (msg->rule_flags & SFE_RULE_CREATE_FLAG_USE_FLOW_BOTTOM_INTERFACE)) {
1226 reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_CSUM_OFFLOAD;
1227 }
1228 }
Matthew McClintockdb5ac512014-01-16 17:01:40 -06001229
1230 /*
Ken Zhubbf49652021-09-12 15:33:09 -07001231 * For the non-arp interface, we don't write L2 HDR.
Matthew McClintockdb5ac512014-01-16 17:01:40 -06001232 */
Ken Zhubbf49652021-09-12 15:33:09 -07001233 if (!(src_dev->flags & IFF_NOARP)) {
Ratheesh Kannoth29140aa2021-10-20 08:25:02 +05301234
1235 /*
1236 * Check whether the rule has configured a specific source MAC address to use.
1237 * This is needed when virtual L3 interfaces such as br-lan, macvlan, vlan are used during egress
1238 */
1239 if ((msg->valid_flags & SFE_RULE_CREATE_SRC_MAC_VALID) &&
1240 (msg->src_mac_rule.mac_valid_flags & SFE_SRC_MAC_FLOW_VALID)) {
1241 ether_addr_copy((u8 *)reply_cm->xmit_src_mac, (u8 *)msg->src_mac_rule.flow_src_mac);
1242 } else {
1243 ether_addr_copy((u8 *)reply_cm->xmit_src_mac, (u8 *)src_dev->dev_addr);
1244 }
1245 ether_addr_copy((u8 *)reply_cm->xmit_dest_mac, (u8 *)msg->conn_rule.flow_mac);
1246
Matthew McClintockdb5ac512014-01-16 17:01:40 -06001247 reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_L2_HDR;
1248
1249 /*
1250 * If our dev writes Ethernet headers then we can write a really fast
1251 * version.
1252 */
1253 if (src_dev->header_ops) {
1254 if (src_dev->header_ops->create == eth_header) {
1255 reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_FAST_ETH_HDR;
1256 }
1257 }
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001258 }
1259
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301260 if ((tuple->return_ip != msg->conn_rule.return_ip_xlate) ||
1261 (tuple->return_ident != msg->conn_rule.return_ident_xlate)) {
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001262 original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_XLATE_DEST;
1263 reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_XLATE_SRC;
1264 }
1265
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301266 if ((tuple->flow_ip != msg->conn_rule.flow_ip_xlate) ||
1267 (tuple->flow_ident != msg->conn_rule.flow_ident_xlate)) {
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001268 original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_XLATE_SRC;
1269 reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_XLATE_DEST;
1270 }
1271
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301272 c->protocol = tuple->protocol;
1273 c->src_ip = tuple->flow_ip;
1274 c->src_ip_xlate = msg->conn_rule.flow_ip_xlate;
1275 c->src_port = tuple->flow_ident;
1276 c->src_port_xlate = msg->conn_rule.flow_ident_xlate;
Matthew McClintockdb5ac512014-01-16 17:01:40 -06001277 c->original_dev = src_dev;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001278 c->original_match = original_cm;
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301279 c->dest_ip = tuple->return_ip;
1280 c->dest_ip_xlate = msg->conn_rule.return_ip_xlate;
1281 c->dest_port = tuple->return_ident;
1282 c->dest_port_xlate = msg->conn_rule.return_ident_xlate;
Matthew McClintockdb5ac512014-01-16 17:01:40 -06001283 c->reply_dev = dest_dev;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001284 c->reply_match = reply_cm;
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301285 c->mark = 0; /* TODO : no mark setting for create rule */
Xiaoping Fan34586472015-07-03 02:20:35 -07001286 c->debug_read_seq = 0;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001287 c->last_sync_jiffies = get_jiffies_64();
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +05301288 c->removed = false;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001289
1290 /*
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001291 * Initialize the protocol-specific information that we track.
1292 */
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301293 switch (tuple->protocol) {
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001294 case IPPROTO_TCP:
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301295 original_cm->protocol_state.tcp.win_scale = msg->tcp_rule.flow_window_scale;
1296 original_cm->protocol_state.tcp.max_win = msg->tcp_rule.flow_max_window ? msg->tcp_rule.flow_max_window : 1;
1297 original_cm->protocol_state.tcp.end = msg->tcp_rule.flow_end;
1298 original_cm->protocol_state.tcp.max_end = msg->tcp_rule.flow_max_end;
1299
1300 reply_cm->protocol_state.tcp.win_scale = msg->tcp_rule.return_window_scale;
1301 reply_cm->protocol_state.tcp.max_win = msg->tcp_rule.return_max_window ? msg->tcp_rule.return_max_window : 1;
1302 reply_cm->protocol_state.tcp.end = msg->tcp_rule.return_end;
1303 reply_cm->protocol_state.tcp.max_end = msg->tcp_rule.return_max_end;
1304
1305 if (msg->rule_flags & SFE_RULE_CREATE_FLAG_NO_SEQ_CHECK) {
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001306 original_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_NO_SEQ_CHECK;
1307 reply_cm->flags |= SFE_IPV4_CONNECTION_MATCH_FLAG_NO_SEQ_CHECK;
1308 }
1309 break;
1310 }
1311
1312 sfe_ipv4_connection_match_compute_translations(original_cm);
1313 sfe_ipv4_connection_match_compute_translations(reply_cm);
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +05301314 sfe_ipv4_insert_connection(si, c);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001315
1316 spin_unlock_bh(&si->lock);
1317
1318 /*
1319 * We have everything we need!
1320 */
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301321 DEBUG_INFO("new connection - p: %d\n"
Tian Yang45f39c82020-10-06 14:07:47 -07001322 " s: %s:%pxM(%pxM):%pI4(%pI4):%u(%u)\n"
1323 " d: %s:%pxM(%pxM):%pI4(%pI4):%u(%u)\n",
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301324 tuple->protocol,
1325 src_dev->name, msg->conn_rule.flow_mac, NULL,
1326 &tuple->flow_ip, &msg->conn_rule.flow_ip_xlate, ntohs(tuple->flow_ident), ntohs(msg->conn_rule.flow_ident_xlate),
1327 dest_dev->name, NULL, msg->conn_rule.return_mac,
1328 &tuple->return_ip, &msg->conn_rule.return_ip_xlate, ntohs(tuple->return_ident), ntohs(msg->conn_rule.return_ident_xlate));
Nicolas Costa514fde02014-01-13 15:50:29 -06001329
1330 return 0;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001331}
1332
1333/*
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001334 * sfe_ipv4_destroy_rule()
1335 * Destroy a forwarding rule.
1336 */
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301337void sfe_ipv4_destroy_rule(struct sfe_ipv4_rule_destroy_msg *msg)
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001338{
Dave Hudsondcd08fb2013-11-22 09:25:16 -06001339 struct sfe_ipv4 *si = &__si;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001340 struct sfe_ipv4_connection *c;
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +05301341 bool ret;
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301342 struct sfe_ipv4_5tuple *tuple = &msg->tuple;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001343
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +05301344 this_cpu_inc(si->stats_pcpu->connection_destroy_requests64);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001345 spin_lock_bh(&si->lock);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001346
1347 /*
1348 * Check to see if we have a flow that matches the rule we're trying
1349 * to destroy. If there isn't then we can't destroy it.
1350 */
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +05301351 c = sfe_ipv4_find_connection(si, tuple->protocol, tuple->flow_ip, tuple->flow_ident,
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301352 tuple->return_ip, tuple->return_ident);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001353 if (!c) {
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001354 spin_unlock_bh(&si->lock);
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +05301355 this_cpu_inc(si->stats_pcpu->connection_destroy_misses64);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001356
1357 DEBUG_TRACE("connection does not exist - p: %d, s: %pI4:%u, d: %pI4:%u\n",
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301358 tuple->protocol, &tuple->flow_ip, ntohs(tuple->flow_ident),
1359 &tuple->return_ip, ntohs(tuple->return_ident));
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001360 return;
1361 }
1362
1363 /*
1364 * Remove our connection details from the hash tables.
1365 */
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +05301366 ret = sfe_ipv4_remove_connection(si, c);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001367 spin_unlock_bh(&si->lock);
1368
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +05301369 if (ret) {
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +05301370 sfe_ipv4_flush_connection(si, c, SFE_SYNC_REASON_DESTROY);
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +05301371 }
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001372
1373 DEBUG_INFO("connection destroyed - p: %d, s: %pI4:%u, d: %pI4:%u\n",
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301374 tuple->protocol, &tuple->flow_ip, ntohs(tuple->flow_ident),
1375 &tuple->return_ip, ntohs(tuple->return_ident));
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001376}
1377
1378/*
Dave Hudsondcd08fb2013-11-22 09:25:16 -06001379 * sfe_ipv4_register_sync_rule_callback()
1380 * Register a callback for rule synchronization.
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001381 */
Xiaoping Fand44a5b42015-05-26 17:37:37 -07001382void sfe_ipv4_register_sync_rule_callback(sfe_sync_rule_callback_t sync_rule_callback)
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001383{
1384 struct sfe_ipv4 *si = &__si;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001385
1386 spin_lock_bh(&si->lock);
Dave Hudsondcd08fb2013-11-22 09:25:16 -06001387 rcu_assign_pointer(si->sync_rule_callback, sync_rule_callback);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001388 spin_unlock_bh(&si->lock);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001389}
1390
1391/*
1392 * sfe_ipv4_get_debug_dev()
1393 */
1394static ssize_t sfe_ipv4_get_debug_dev(struct device *dev,
1395 struct device_attribute *attr,
1396 char *buf)
1397{
1398 struct sfe_ipv4 *si = &__si;
1399 ssize_t count;
1400 int num;
1401
1402 spin_lock_bh(&si->lock);
1403 num = si->debug_dev;
1404 spin_unlock_bh(&si->lock);
1405
1406 count = snprintf(buf, (ssize_t)PAGE_SIZE, "%d\n", num);
1407 return count;
1408}
1409
1410/*
Dave Hudsondcd08fb2013-11-22 09:25:16 -06001411 * sysfs attributes.
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001412 */
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001413static const struct device_attribute sfe_ipv4_debug_dev_attr =
Xiaoping Fane70da412016-02-26 16:47:57 -08001414 __ATTR(debug_dev, S_IWUSR | S_IRUGO, sfe_ipv4_get_debug_dev, NULL);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001415
1416/*
Dave Hudsondcd08fb2013-11-22 09:25:16 -06001417 * sfe_ipv4_destroy_all_rules_for_dev()
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001418 * Destroy all connections that match a particular device.
1419 *
1420 * If we pass dev as NULL then this destroys all connections.
1421 */
Dave Hudsondcd08fb2013-11-22 09:25:16 -06001422void sfe_ipv4_destroy_all_rules_for_dev(struct net_device *dev)
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001423{
Dave Hudsondcd08fb2013-11-22 09:25:16 -06001424 struct sfe_ipv4 *si = &__si;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001425 struct sfe_ipv4_connection *c;
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +05301426 bool ret;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001427
Xiaoping Fan34586472015-07-03 02:20:35 -07001428another_round:
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001429 spin_lock_bh(&si->lock);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001430
Xiaoping Fan34586472015-07-03 02:20:35 -07001431 for (c = si->all_connections_head; c; c = c->all_connections_next) {
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001432 /*
Xiaoping Fan34586472015-07-03 02:20:35 -07001433 * Does this connection relate to the device we are destroying?
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001434 */
1435 if (!dev
1436 || (dev == c->original_dev)
1437 || (dev == c->reply_dev)) {
Xiaoping Fan34586472015-07-03 02:20:35 -07001438 break;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001439 }
Xiaoping Fan34586472015-07-03 02:20:35 -07001440 }
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001441
Xiaoping Fan34586472015-07-03 02:20:35 -07001442 if (c) {
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +05301443 ret = sfe_ipv4_remove_connection(si, c);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001444 }
1445
1446 spin_unlock_bh(&si->lock);
Xiaoping Fan34586472015-07-03 02:20:35 -07001447
1448 if (c) {
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +05301449 if (ret) {
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +05301450 sfe_ipv4_flush_connection(si, c, SFE_SYNC_REASON_DESTROY);
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +05301451 }
Xiaoping Fan34586472015-07-03 02:20:35 -07001452 goto another_round;
1453 }
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001454}
1455
1456/*
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001457 * sfe_ipv4_periodic_sync()
1458 */
Ken Zhu137722d2021-09-23 17:57:36 -07001459static void sfe_ipv4_periodic_sync(struct work_struct *work)
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001460{
Ken Zhu137722d2021-09-23 17:57:36 -07001461 struct sfe_ipv4 *si = container_of((struct delayed_work *)work, struct sfe_ipv4, sync_dwork);
Xiaoping Fan6a1672f2016-08-17 19:58:12 -07001462 u64 now_jiffies;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001463 int quota;
Xiaoping Fand44a5b42015-05-26 17:37:37 -07001464 sfe_sync_rule_callback_t sync_rule_callback;
Ken Zhudc423672021-09-02 18:27:01 -07001465 struct sfe_ipv4_connection *c;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001466
1467 now_jiffies = get_jiffies_64();
1468
Dave Hudsondcd08fb2013-11-22 09:25:16 -06001469 rcu_read_lock();
1470 sync_rule_callback = rcu_dereference(si->sync_rule_callback);
1471 if (!sync_rule_callback) {
1472 rcu_read_unlock();
1473 goto done;
1474 }
1475
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001476 spin_lock_bh(&si->lock);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001477
1478 /*
Ken Zhudc423672021-09-02 18:27:01 -07001479 * If we have reached the end of the connection list, walk from
1480 * the connection head.
1481 */
1482 c = si->wc_next;
1483 if (unlikely(!c)) {
1484 c = si->all_connections_head;
1485 }
1486
1487 /*
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001488 * Get an estimate of the number of connections to parse in this sync.
1489 */
1490 quota = (si->num_connections + 63) / 64;
1491
1492 /*
Ken Zhudc423672021-09-02 18:27:01 -07001493 * Walk the "all connection" list and sync the connection state.
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001494 */
Ken Zhudc423672021-09-02 18:27:01 -07001495 while (likely(c && quota)) {
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001496 struct sfe_ipv4_connection_match *cm;
1497 struct sfe_ipv4_connection_match *counter_cm;
Xiaoping Fand44a5b42015-05-26 17:37:37 -07001498 struct sfe_connection_sync sis;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001499
Ken Zhudc423672021-09-02 18:27:01 -07001500 cm = c->original_match;
1501 counter_cm = c->reply_match;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001502
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001503 /*
Ken Zhudc423672021-09-02 18:27:01 -07001504 * Didn't receive packets in the original direction or reply
1505 * direction, move to the next connection.
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001506 */
Ken Zhudc423672021-09-02 18:27:01 -07001507 if ((!atomic_read(&cm->rx_packet_count)) && !(atomic_read(&counter_cm->rx_packet_count))) {
1508 c = c->all_connections_next;
1509 continue;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001510 }
1511
Ken Zhudc423672021-09-02 18:27:01 -07001512 quota--;
Matthew McClintockaf48f1e2014-01-23 15:29:19 -06001513
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +05301514 sfe_ipv4_gen_sync_connection(si, c, &sis, SFE_SYNC_REASON_STATS, now_jiffies);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001515
Ken Zhudc423672021-09-02 18:27:01 -07001516 si->wc_next = c->all_connections_next;
1517
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001518 /*
1519 * We don't want to be holding the lock when we sync!
1520 */
1521 spin_unlock_bh(&si->lock);
Dave Hudsondcd08fb2013-11-22 09:25:16 -06001522 sync_rule_callback(&sis);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001523 spin_lock_bh(&si->lock);
Ken Zhudc423672021-09-02 18:27:01 -07001524
1525 /*
1526 * c must be set and used in the same lock/unlock window;
1527 * because c could be removed when we don't hold the lock,
1528 * so delay grabbing until after the callback and relock.
1529 */
1530 c = si->wc_next;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001531 }
1532
Ken Zhudc423672021-09-02 18:27:01 -07001533 /*
1534 * At the end of the sync, put the wc_next to the connection we left.
1535 */
1536 si->wc_next = c;
1537
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001538 spin_unlock_bh(&si->lock);
Dave Hudsondcd08fb2013-11-22 09:25:16 -06001539 rcu_read_unlock();
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001540
Dave Hudsondcd08fb2013-11-22 09:25:16 -06001541done:
Ken Zhu137722d2021-09-23 17:57:36 -07001542 schedule_delayed_work_on(si->work_cpu, (struct delayed_work *)work, ((HZ + 99) / 100));
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001543}
1544
1545#define CHAR_DEV_MSG_SIZE 768
1546
1547/*
1548 * sfe_ipv4_debug_dev_read_start()
1549 * Generate part of the XML output.
1550 */
1551static bool sfe_ipv4_debug_dev_read_start(struct sfe_ipv4 *si, char *buffer, char *msg, size_t *length,
1552 int *total_read, struct sfe_ipv4_debug_xml_write_state *ws)
1553{
1554 int bytes_read;
1555
Xiaoping Fan34586472015-07-03 02:20:35 -07001556 si->debug_read_seq++;
1557
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001558 bytes_read = snprintf(msg, CHAR_DEV_MSG_SIZE, "<sfe_ipv4>\n");
1559 if (copy_to_user(buffer + *total_read, msg, CHAR_DEV_MSG_SIZE)) {
1560 return false;
1561 }
1562
1563 *length -= bytes_read;
1564 *total_read += bytes_read;
1565
1566 ws->state++;
1567 return true;
1568}
1569
1570/*
1571 * sfe_ipv4_debug_dev_read_connections_start()
1572 * Generate part of the XML output.
1573 */
1574static bool sfe_ipv4_debug_dev_read_connections_start(struct sfe_ipv4 *si, char *buffer, char *msg, size_t *length,
1575 int *total_read, struct sfe_ipv4_debug_xml_write_state *ws)
1576{
1577 int bytes_read;
1578
1579 bytes_read = snprintf(msg, CHAR_DEV_MSG_SIZE, "\t<connections>\n");
1580 if (copy_to_user(buffer + *total_read, msg, CHAR_DEV_MSG_SIZE)) {
1581 return false;
1582 }
1583
1584 *length -= bytes_read;
1585 *total_read += bytes_read;
1586
1587 ws->state++;
1588 return true;
1589}
1590
1591/*
1592 * sfe_ipv4_debug_dev_read_connections_connection()
1593 * Generate part of the XML output.
1594 */
1595static bool sfe_ipv4_debug_dev_read_connections_connection(struct sfe_ipv4 *si, char *buffer, char *msg, size_t *length,
1596 int *total_read, struct sfe_ipv4_debug_xml_write_state *ws)
1597{
1598 struct sfe_ipv4_connection *c;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001599 struct sfe_ipv4_connection_match *original_cm;
1600 struct sfe_ipv4_connection_match *reply_cm;
1601 int bytes_read;
1602 int protocol;
1603 struct net_device *src_dev;
Dave Hudson87973cd2013-10-22 16:00:04 +01001604 __be32 src_ip;
1605 __be32 src_ip_xlate;
1606 __be16 src_port;
1607 __be16 src_port_xlate;
Xiaoping Fan6a1672f2016-08-17 19:58:12 -07001608 u64 src_rx_packets;
1609 u64 src_rx_bytes;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001610 struct net_device *dest_dev;
Dave Hudson87973cd2013-10-22 16:00:04 +01001611 __be32 dest_ip;
1612 __be32 dest_ip_xlate;
1613 __be16 dest_port;
1614 __be16 dest_port_xlate;
Xiaoping Fan6a1672f2016-08-17 19:58:12 -07001615 u64 dest_rx_packets;
1616 u64 dest_rx_bytes;
1617 u64 last_sync_jiffies;
1618 u32 mark, src_priority, dest_priority, src_dscp, dest_dscp;
Guduri Prathyushaeb31c902021-11-10 20:18:50 +05301619 u32 packet, byte, original_cm_flags;
1620 u16 pppoe_session_id;
1621 u8 pppoe_remote_mac[ETH_ALEN];
Xiaoping Fand1dc7b22015-01-23 00:43:56 -08001622#ifdef CONFIG_NF_FLOW_COOKIE
1623 int src_flow_cookie, dst_flow_cookie;
1624#endif
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001625
1626 spin_lock_bh(&si->lock);
Xiaoping Fan34586472015-07-03 02:20:35 -07001627
1628 for (c = si->all_connections_head; c; c = c->all_connections_next) {
1629 if (c->debug_read_seq < si->debug_read_seq) {
1630 c->debug_read_seq = si->debug_read_seq;
1631 break;
1632 }
1633 }
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001634
1635 /*
Xiaoping Fan34586472015-07-03 02:20:35 -07001636 * If there were no connections then move to the next state.
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001637 */
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +05301638 if (!c || c->removed) {
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001639 spin_unlock_bh(&si->lock);
Xiaoping Fan34586472015-07-03 02:20:35 -07001640 ws->state++;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001641 return true;
1642 }
1643
1644 original_cm = c->original_match;
1645 reply_cm = c->reply_match;
1646
1647 protocol = c->protocol;
1648 src_dev = c->original_dev;
1649 src_ip = c->src_ip;
1650 src_ip_xlate = c->src_ip_xlate;
1651 src_port = c->src_port;
1652 src_port_xlate = c->src_port_xlate;
Xiaoping Fane1963d42015-08-25 17:06:19 -07001653 src_priority = original_cm->priority;
1654 src_dscp = original_cm->dscp >> SFE_IPV4_DSCP_SHIFT;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001655
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +05301656 sfe_ipv4_connection_match_update_summary_stats(original_cm, &packet, &byte);
1657 sfe_ipv4_connection_match_update_summary_stats(reply_cm, &packet, &byte);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001658
1659 src_rx_packets = original_cm->rx_packet_count64;
1660 src_rx_bytes = original_cm->rx_byte_count64;
1661 dest_dev = c->reply_dev;
1662 dest_ip = c->dest_ip;
1663 dest_ip_xlate = c->dest_ip_xlate;
1664 dest_port = c->dest_port;
1665 dest_port_xlate = c->dest_port_xlate;
Xiaoping Fane1963d42015-08-25 17:06:19 -07001666 dest_priority = reply_cm->priority;
1667 dest_dscp = reply_cm->dscp >> SFE_IPV4_DSCP_SHIFT;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001668 dest_rx_packets = reply_cm->rx_packet_count64;
1669 dest_rx_bytes = reply_cm->rx_byte_count64;
1670 last_sync_jiffies = get_jiffies_64() - c->last_sync_jiffies;
Cristian Prundeanu592265e2013-12-26 11:01:22 -06001671 mark = c->mark;
Guduri Prathyushaeb31c902021-11-10 20:18:50 +05301672 original_cm_flags = original_cm->flags;
1673 pppoe_session_id = original_cm->pppoe_session_id;
1674 ether_addr_copy(pppoe_remote_mac, original_cm->pppoe_remote_mac);
1675
Xiaoping Fand1dc7b22015-01-23 00:43:56 -08001676#ifdef CONFIG_NF_FLOW_COOKIE
1677 src_flow_cookie = original_cm->flow_cookie;
1678 dst_flow_cookie = reply_cm->flow_cookie;
1679#endif
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001680 spin_unlock_bh(&si->lock);
1681
1682 bytes_read = snprintf(msg, CHAR_DEV_MSG_SIZE, "\t\t<connection "
1683 "protocol=\"%u\" "
1684 "src_dev=\"%s\" "
1685 "src_ip=\"%pI4\" src_ip_xlate=\"%pI4\" "
1686 "src_port=\"%u\" src_port_xlate=\"%u\" "
Xiaoping Fane1963d42015-08-25 17:06:19 -07001687 "src_priority=\"%u\" src_dscp=\"%u\" "
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001688 "src_rx_pkts=\"%llu\" src_rx_bytes=\"%llu\" "
1689 "dest_dev=\"%s\" "
1690 "dest_ip=\"%pI4\" dest_ip_xlate=\"%pI4\" "
1691 "dest_port=\"%u\" dest_port_xlate=\"%u\" "
Xiaoping Fane1963d42015-08-25 17:06:19 -07001692 "dest_priority=\"%u\" dest_dscp=\"%u\" "
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001693 "dest_rx_pkts=\"%llu\" dest_rx_bytes=\"%llu\" "
Xiaoping Fand1dc7b22015-01-23 00:43:56 -08001694#ifdef CONFIG_NF_FLOW_COOKIE
1695 "src_flow_cookie=\"%d\" dst_flow_cookie=\"%d\" "
1696#endif
Cristian Prundeanu592265e2013-12-26 11:01:22 -06001697 "last_sync=\"%llu\" "
Guduri Prathyushaeb31c902021-11-10 20:18:50 +05301698 "mark=\"%08x\" ",
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001699 protocol,
1700 src_dev->name,
1701 &src_ip, &src_ip_xlate,
Dave Hudson87973cd2013-10-22 16:00:04 +01001702 ntohs(src_port), ntohs(src_port_xlate),
Xiaoping Fane1963d42015-08-25 17:06:19 -07001703 src_priority, src_dscp,
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001704 src_rx_packets, src_rx_bytes,
1705 dest_dev->name,
1706 &dest_ip, &dest_ip_xlate,
Dave Hudson87973cd2013-10-22 16:00:04 +01001707 ntohs(dest_port), ntohs(dest_port_xlate),
Xiaoping Fane1963d42015-08-25 17:06:19 -07001708 dest_priority, dest_dscp,
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001709 dest_rx_packets, dest_rx_bytes,
Xiaoping Fand1dc7b22015-01-23 00:43:56 -08001710#ifdef CONFIG_NF_FLOW_COOKIE
1711 src_flow_cookie, dst_flow_cookie,
1712#endif
Cristian Prundeanu592265e2013-12-26 11:01:22 -06001713 last_sync_jiffies, mark);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001714
Guduri Prathyushaeb31c902021-11-10 20:18:50 +05301715 if (original_cm_flags &= (SFE_IPV4_CONNECTION_MATCH_FLAG_PPPOE_DECAP | SFE_IPV4_CONNECTION_MATCH_FLAG_PPPOE_ENCAP)) {
1716 bytes_read += snprintf(msg + bytes_read, CHAR_DEV_MSG_SIZE, "pppoe session_id=\"%u\" pppoe server MAC=\"%pM\" ",
1717 pppoe_session_id, pppoe_remote_mac);
1718 }
1719
1720 bytes_read += snprintf(msg + bytes_read, CHAR_DEV_MSG_SIZE, "/>\n");
1721
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001722 if (copy_to_user(buffer + *total_read, msg, CHAR_DEV_MSG_SIZE)) {
1723 return false;
1724 }
1725
1726 *length -= bytes_read;
1727 *total_read += bytes_read;
1728
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001729 return true;
1730}
1731
1732/*
1733 * sfe_ipv4_debug_dev_read_connections_end()
1734 * Generate part of the XML output.
1735 */
1736static bool sfe_ipv4_debug_dev_read_connections_end(struct sfe_ipv4 *si, char *buffer, char *msg, size_t *length,
1737 int *total_read, struct sfe_ipv4_debug_xml_write_state *ws)
1738{
1739 int bytes_read;
1740
1741 bytes_read = snprintf(msg, CHAR_DEV_MSG_SIZE, "\t</connections>\n");
1742 if (copy_to_user(buffer + *total_read, msg, CHAR_DEV_MSG_SIZE)) {
1743 return false;
1744 }
1745
1746 *length -= bytes_read;
1747 *total_read += bytes_read;
1748
1749 ws->state++;
1750 return true;
1751}
1752
1753/*
1754 * sfe_ipv4_debug_dev_read_exceptions_start()
1755 * Generate part of the XML output.
1756 */
1757static bool sfe_ipv4_debug_dev_read_exceptions_start(struct sfe_ipv4 *si, char *buffer, char *msg, size_t *length,
1758 int *total_read, struct sfe_ipv4_debug_xml_write_state *ws)
1759{
1760 int bytes_read;
1761
1762 bytes_read = snprintf(msg, CHAR_DEV_MSG_SIZE, "\t<exceptions>\n");
1763 if (copy_to_user(buffer + *total_read, msg, CHAR_DEV_MSG_SIZE)) {
1764 return false;
1765 }
1766
1767 *length -= bytes_read;
1768 *total_read += bytes_read;
1769
1770 ws->state++;
1771 return true;
1772}
1773
1774/*
1775 * sfe_ipv4_debug_dev_read_exceptions_exception()
1776 * Generate part of the XML output.
1777 */
1778static bool sfe_ipv4_debug_dev_read_exceptions_exception(struct sfe_ipv4 *si, char *buffer, char *msg, size_t *length,
1779 int *total_read, struct sfe_ipv4_debug_xml_write_state *ws)
1780{
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +05301781 int i;
1782 u64 val = 0;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001783
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +05301784 for_each_possible_cpu(i) {
1785 const struct sfe_ipv4_stats *s = per_cpu_ptr(si->stats_pcpu, i);
1786 val += s->exception_events64[ws->iter_exception];
1787 }
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001788
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +05301789 if (val) {
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001790 int bytes_read;
1791
1792 bytes_read = snprintf(msg, CHAR_DEV_MSG_SIZE,
1793 "\t\t<exception name=\"%s\" count=\"%llu\" />\n",
1794 sfe_ipv4_exception_events_string[ws->iter_exception],
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +05301795 val);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001796 if (copy_to_user(buffer + *total_read, msg, CHAR_DEV_MSG_SIZE)) {
1797 return false;
1798 }
1799
1800 *length -= bytes_read;
1801 *total_read += bytes_read;
1802 }
1803
1804 ws->iter_exception++;
1805 if (ws->iter_exception >= SFE_IPV4_EXCEPTION_EVENT_LAST) {
1806 ws->iter_exception = 0;
1807 ws->state++;
1808 }
1809
1810 return true;
1811}
1812
1813/*
1814 * sfe_ipv4_debug_dev_read_exceptions_end()
1815 * Generate part of the XML output.
1816 */
1817static bool sfe_ipv4_debug_dev_read_exceptions_end(struct sfe_ipv4 *si, char *buffer, char *msg, size_t *length,
1818 int *total_read, struct sfe_ipv4_debug_xml_write_state *ws)
1819{
1820 int bytes_read;
1821
1822 bytes_read = snprintf(msg, CHAR_DEV_MSG_SIZE, "\t</exceptions>\n");
1823 if (copy_to_user(buffer + *total_read, msg, CHAR_DEV_MSG_SIZE)) {
1824 return false;
1825 }
1826
1827 *length -= bytes_read;
1828 *total_read += bytes_read;
1829
1830 ws->state++;
1831 return true;
1832}
1833
1834/*
1835 * sfe_ipv4_debug_dev_read_stats()
1836 * Generate part of the XML output.
1837 */
1838static bool sfe_ipv4_debug_dev_read_stats(struct sfe_ipv4 *si, char *buffer, char *msg, size_t *length,
1839 int *total_read, struct sfe_ipv4_debug_xml_write_state *ws)
1840{
1841 int bytes_read;
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +05301842 struct sfe_ipv4_stats stats;
1843 unsigned int num_conn;
1844
1845 sfe_ipv4_update_summary_stats(si, &stats);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001846
1847 spin_lock_bh(&si->lock);
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +05301848 num_conn = si->num_connections;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001849 spin_unlock_bh(&si->lock);
1850
1851 bytes_read = snprintf(msg, CHAR_DEV_MSG_SIZE, "\t<stats "
1852 "num_connections=\"%u\" "
Xiaoping Fan59176422015-05-22 15:58:10 -07001853 "pkts_forwarded=\"%llu\" pkts_not_forwarded=\"%llu\" "
1854 "create_requests=\"%llu\" create_collisions=\"%llu\" "
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301855 "create_failures=\"%llu\" "
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001856 "destroy_requests=\"%llu\" destroy_misses=\"%llu\" "
1857 "flushes=\"%llu\" "
1858 "hash_hits=\"%llu\" hash_reorders=\"%llu\" />\n",
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +05301859 num_conn,
1860 stats.packets_forwarded64,
1861 stats.packets_not_forwarded64,
1862 stats.connection_create_requests64,
1863 stats.connection_create_collisions64,
Ratheesh Kannoth89302a72021-10-20 08:10:37 +05301864 stats.connection_create_failures64,
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +05301865 stats.connection_destroy_requests64,
1866 stats.connection_destroy_misses64,
1867 stats.connection_flushes64,
1868 stats.connection_match_hash_hits64,
1869 stats.connection_match_hash_reorders64);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001870 if (copy_to_user(buffer + *total_read, msg, CHAR_DEV_MSG_SIZE)) {
1871 return false;
1872 }
1873
1874 *length -= bytes_read;
1875 *total_read += bytes_read;
1876
1877 ws->state++;
1878 return true;
1879}
1880
1881/*
1882 * sfe_ipv4_debug_dev_read_end()
1883 * Generate part of the XML output.
1884 */
1885static bool sfe_ipv4_debug_dev_read_end(struct sfe_ipv4 *si, char *buffer, char *msg, size_t *length,
1886 int *total_read, struct sfe_ipv4_debug_xml_write_state *ws)
1887{
1888 int bytes_read;
1889
1890 bytes_read = snprintf(msg, CHAR_DEV_MSG_SIZE, "</sfe_ipv4>\n");
1891 if (copy_to_user(buffer + *total_read, msg, CHAR_DEV_MSG_SIZE)) {
1892 return false;
1893 }
1894
1895 *length -= bytes_read;
1896 *total_read += bytes_read;
1897
1898 ws->state++;
1899 return true;
1900}
1901
1902/*
1903 * Array of write functions that write various XML elements that correspond to
1904 * our XML output state machine.
1905 */
Xiaoping Fan6a1672f2016-08-17 19:58:12 -07001906static sfe_ipv4_debug_xml_write_method_t sfe_ipv4_debug_xml_write_methods[SFE_IPV4_DEBUG_XML_STATE_DONE] = {
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001907 sfe_ipv4_debug_dev_read_start,
1908 sfe_ipv4_debug_dev_read_connections_start,
1909 sfe_ipv4_debug_dev_read_connections_connection,
1910 sfe_ipv4_debug_dev_read_connections_end,
1911 sfe_ipv4_debug_dev_read_exceptions_start,
1912 sfe_ipv4_debug_dev_read_exceptions_exception,
1913 sfe_ipv4_debug_dev_read_exceptions_end,
1914 sfe_ipv4_debug_dev_read_stats,
1915 sfe_ipv4_debug_dev_read_end,
1916};
1917
1918/*
1919 * sfe_ipv4_debug_dev_read()
1920 * Send info to userspace upon read request from user
1921 */
1922static ssize_t sfe_ipv4_debug_dev_read(struct file *filp, char *buffer, size_t length, loff_t *offset)
1923{
1924 char msg[CHAR_DEV_MSG_SIZE];
1925 int total_read = 0;
1926 struct sfe_ipv4_debug_xml_write_state *ws;
1927 struct sfe_ipv4 *si = &__si;
1928
1929 ws = (struct sfe_ipv4_debug_xml_write_state *)filp->private_data;
1930 while ((ws->state != SFE_IPV4_DEBUG_XML_STATE_DONE) && (length > CHAR_DEV_MSG_SIZE)) {
1931 if ((sfe_ipv4_debug_xml_write_methods[ws->state])(si, buffer, msg, &length, &total_read, ws)) {
1932 continue;
1933 }
1934 }
1935
1936 return total_read;
1937}
1938
1939/*
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001940 * sfe_ipv4_debug_dev_open()
1941 */
1942static int sfe_ipv4_debug_dev_open(struct inode *inode, struct file *file)
1943{
1944 struct sfe_ipv4_debug_xml_write_state *ws;
1945
1946 ws = (struct sfe_ipv4_debug_xml_write_state *)file->private_data;
1947 if (!ws) {
1948 ws = kzalloc(sizeof(struct sfe_ipv4_debug_xml_write_state), GFP_KERNEL);
1949 if (!ws) {
1950 return -ENOMEM;
1951 }
1952
1953 ws->state = SFE_IPV4_DEBUG_XML_STATE_START;
1954 file->private_data = ws;
1955 }
1956
1957 return 0;
1958}
1959
1960/*
1961 * sfe_ipv4_debug_dev_release()
1962 */
1963static int sfe_ipv4_debug_dev_release(struct inode *inode, struct file *file)
1964{
1965 struct sfe_ipv4_debug_xml_write_state *ws;
1966
1967 ws = (struct sfe_ipv4_debug_xml_write_state *)file->private_data;
1968 if (ws) {
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001969 /*
1970 * We've finished with our output so free the write state.
1971 */
1972 kfree(ws);
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +05301973 file->private_data = NULL;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001974 }
1975
1976 return 0;
1977}
1978
1979/*
1980 * File operations used in the debug char device
1981 */
1982static struct file_operations sfe_ipv4_debug_dev_fops = {
1983 .read = sfe_ipv4_debug_dev_read,
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01001984 .open = sfe_ipv4_debug_dev_open,
1985 .release = sfe_ipv4_debug_dev_release
1986};
1987
Xiaoping Fand1dc7b22015-01-23 00:43:56 -08001988#ifdef CONFIG_NF_FLOW_COOKIE
1989/*
1990 * sfe_register_flow_cookie_cb
1991 * register a function in SFE to let SFE use this function to configure flow cookie for a flow
1992 *
1993 * Hardware driver which support flow cookie should register a callback function in SFE. Then SFE
1994 * can use this function to configure flow cookie for a flow.
1995 * return: 0, success; !=0, fail
1996 */
1997int sfe_register_flow_cookie_cb(flow_cookie_set_func_t cb)
1998{
1999 struct sfe_ipv4 *si = &__si;
2000
2001 BUG_ON(!cb);
2002
2003 if (si->flow_cookie_set_func) {
2004 return -1;
2005 }
2006
2007 rcu_assign_pointer(si->flow_cookie_set_func, cb);
2008 return 0;
2009}
2010
2011/*
2012 * sfe_unregister_flow_cookie_cb
2013 * unregister function which is used to configure flow cookie for a flow
2014 *
2015 * return: 0, success; !=0, fail
2016 */
2017int sfe_unregister_flow_cookie_cb(flow_cookie_set_func_t cb)
2018{
2019 struct sfe_ipv4 *si = &__si;
2020
2021 RCU_INIT_POINTER(si->flow_cookie_set_func, NULL);
2022 return 0;
2023}
Xiaoping Fan640faf42015-08-28 15:50:55 -07002024
2025/*
2026 * sfe_ipv4_get_flow_cookie()
2027 */
2028static ssize_t sfe_ipv4_get_flow_cookie(struct device *dev,
2029 struct device_attribute *attr,
2030 char *buf)
2031{
2032 struct sfe_ipv4 *si = &__si;
Xiaoping Fan01c67cc2015-11-09 11:31:57 -08002033 return snprintf(buf, (ssize_t)PAGE_SIZE, "%d\n", si->flow_cookie_enable);
Xiaoping Fan640faf42015-08-28 15:50:55 -07002034}
2035
2036/*
2037 * sfe_ipv4_set_flow_cookie()
2038 */
2039static ssize_t sfe_ipv4_set_flow_cookie(struct device *dev,
2040 struct device_attribute *attr,
2041 const char *buf, size_t size)
2042{
2043 struct sfe_ipv4 *si = &__si;
Ken Zhu137722d2021-09-23 17:57:36 -07002044 si->flow_cookie_enable = simple_strtol(buf, NULL, 0);
Xiaoping Fan640faf42015-08-28 15:50:55 -07002045
2046 return size;
2047}
2048
2049/*
2050 * sysfs attributes.
2051 */
2052static const struct device_attribute sfe_ipv4_flow_cookie_attr =
Xiaoping Fane70da412016-02-26 16:47:57 -08002053 __ATTR(flow_cookie_enable, S_IWUSR | S_IRUGO, sfe_ipv4_get_flow_cookie, sfe_ipv4_set_flow_cookie);
Xiaoping Fand1dc7b22015-01-23 00:43:56 -08002054#endif /*CONFIG_NF_FLOW_COOKIE*/
2055
Ken Zhu137722d2021-09-23 17:57:36 -07002056/*
2057 * sfe_ipv4_get_cpu()
2058 */
2059static ssize_t sfe_ipv4_get_cpu(struct device *dev,
2060 struct device_attribute *attr,
2061 char *buf)
2062{
2063 struct sfe_ipv4 *si = &__si;
2064 return snprintf(buf, (ssize_t)PAGE_SIZE, "%d\n", si->work_cpu);
2065}
2066
2067/*
2068 * sfe_ipv4_set_cpu()
2069 */
2070static ssize_t sfe_ipv4_set_cpu(struct device *dev,
2071 struct device_attribute *attr,
2072 const char *buf, size_t size)
2073{
2074 struct sfe_ipv4 *si = &__si;
2075 int work_cpu;
2076 work_cpu = simple_strtol(buf, NULL, 0);
2077 if ((work_cpu >= 0) && (work_cpu <= NR_CPUS)) {
2078 si->work_cpu = work_cpu;
2079 } else {
2080 dev_err(dev, "%s is not in valid range[0,%d]", buf, NR_CPUS);
2081 }
2082 return size;
2083}
2084/*
2085 * sysfs attributes.
2086 */
2087static const struct device_attribute sfe_ipv4_cpu_attr =
2088 __ATTR(stats_work_cpu, S_IWUSR | S_IRUGO, sfe_ipv4_get_cpu, sfe_ipv4_set_cpu);
2089
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +05302090 /*
2091 * sfe_ipv4_conn_match_hash_init()
2092 * Initialize conn match hash lists
2093 */
2094static void sfe_ipv4_conn_match_hash_init(struct sfe_ipv4 *si, int len)
2095{
2096 struct hlist_head *hash_list = si->hlist_conn_match_hash_head;
2097 int i;
2098
2099 for (i = 0; i < len; i++) {
2100 INIT_HLIST_HEAD(&hash_list[i]);
2101 }
2102}
2103
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01002104/*
Dave Hudson87973cd2013-10-22 16:00:04 +01002105 * sfe_ipv4_init()
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01002106 */
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +05302107int sfe_ipv4_init(void)
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01002108{
2109 struct sfe_ipv4 *si = &__si;
2110 int result = -1;
2111
Dave Hudsondcd08fb2013-11-22 09:25:16 -06002112 DEBUG_INFO("SFE IPv4 init\n");
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01002113
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +05302114 sfe_ipv4_conn_match_hash_init(si, ARRAY_SIZE(si->hlist_conn_match_hash_head));
2115
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +05302116 si->stats_pcpu = alloc_percpu_gfp(struct sfe_ipv4_stats, GFP_KERNEL | __GFP_ZERO);
2117 if (!si->stats_pcpu) {
2118 DEBUG_ERROR("failed to allocate stats memory for sfe_ipv4\n");
2119 goto exit0;
2120 }
2121
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01002122 /*
2123 * Create sys/sfe_ipv4
2124 */
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +05302125 si->sys_ipv4 = kobject_create_and_add("sfe_ipv4", NULL);
2126 if (!si->sys_ipv4) {
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01002127 DEBUG_ERROR("failed to register sfe_ipv4\n");
2128 goto exit1;
2129 }
2130
2131 /*
2132 * Create files, one for each parameter supported by this module.
2133 */
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +05302134 result = sysfs_create_file(si->sys_ipv4, &sfe_ipv4_debug_dev_attr.attr);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01002135 if (result) {
2136 DEBUG_ERROR("failed to register debug dev file: %d\n", result);
Xiaoping Fan640faf42015-08-28 15:50:55 -07002137 goto exit2;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01002138 }
2139
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +05302140 result = sysfs_create_file(si->sys_ipv4, &sfe_ipv4_cpu_attr.attr);
Ken Zhu137722d2021-09-23 17:57:36 -07002141 if (result) {
2142 DEBUG_ERROR("failed to register debug dev file: %d\n", result);
2143 goto exit3;
2144 }
2145
Xiaoping Fan640faf42015-08-28 15:50:55 -07002146#ifdef CONFIG_NF_FLOW_COOKIE
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +05302147 result = sysfs_create_file(si->sys_ipv4, &sfe_ipv4_flow_cookie_attr.attr);
Xiaoping Fan640faf42015-08-28 15:50:55 -07002148 if (result) {
2149 DEBUG_ERROR("failed to register flow cookie enable file: %d\n", result);
Ken Zhu137722d2021-09-23 17:57:36 -07002150 goto exit4;
Xiaoping Fan640faf42015-08-28 15:50:55 -07002151 }
2152#endif /* CONFIG_NF_FLOW_COOKIE */
2153
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01002154 /*
2155 * Register our debug char device.
2156 */
2157 result = register_chrdev(0, "sfe_ipv4", &sfe_ipv4_debug_dev_fops);
2158 if (result < 0) {
2159 DEBUG_ERROR("Failed to register chrdev: %d\n", result);
Ken Zhu137722d2021-09-23 17:57:36 -07002160 goto exit5;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01002161 }
2162
2163 si->debug_dev = result;
Ken Zhu137722d2021-09-23 17:57:36 -07002164 si->work_cpu = WORK_CPU_UNBOUND;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01002165
2166 /*
Ken Zhu137722d2021-09-23 17:57:36 -07002167 * Create a work to handle periodic statistics.
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01002168 */
Ken Zhu137722d2021-09-23 17:57:36 -07002169 INIT_DELAYED_WORK(&(si->sync_dwork), sfe_ipv4_periodic_sync);
2170 schedule_delayed_work_on(si->work_cpu, &(si->sync_dwork), ((HZ + 99) / 100));
2171
Dave Hudson87973cd2013-10-22 16:00:04 +01002172 spin_lock_init(&si->lock);
Dave Hudson87973cd2013-10-22 16:00:04 +01002173 return 0;
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01002174
Ken Zhu137722d2021-09-23 17:57:36 -07002175exit5:
Xiaoping Fan640faf42015-08-28 15:50:55 -07002176#ifdef CONFIG_NF_FLOW_COOKIE
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +05302177 sysfs_remove_file(si->sys_ipv4, &sfe_ipv4_flow_cookie_attr.attr);
Xiaoping Fan640faf42015-08-28 15:50:55 -07002178
Ken Zhu137722d2021-09-23 17:57:36 -07002179exit4:
Xiaoping Fan640faf42015-08-28 15:50:55 -07002180#endif /* CONFIG_NF_FLOW_COOKIE */
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +05302181 sysfs_remove_file(si->sys_ipv4, &sfe_ipv4_cpu_attr.attr);
Ken Zhu137722d2021-09-23 17:57:36 -07002182exit3:
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +05302183 sysfs_remove_file(si->sys_ipv4, &sfe_ipv4_debug_dev_attr.attr);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01002184
Xiaoping Fan640faf42015-08-28 15:50:55 -07002185exit2:
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +05302186 kobject_put(si->sys_ipv4);
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01002187
2188exit1:
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +05302189 free_percpu(si->stats_pcpu);
2190
2191exit0:
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01002192 return result;
2193}
2194
2195/*
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01002196 * sfe_ipv4_exit()
2197 */
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +05302198void sfe_ipv4_exit(void)
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01002199{
Dave Hudson87973cd2013-10-22 16:00:04 +01002200 struct sfe_ipv4 *si = &__si;
2201
Dave Hudsondcd08fb2013-11-22 09:25:16 -06002202 DEBUG_INFO("SFE IPv4 exit\n");
Dave Hudson87973cd2013-10-22 16:00:04 +01002203 /*
2204 * Destroy all connections.
2205 */
Dave Hudsondcd08fb2013-11-22 09:25:16 -06002206 sfe_ipv4_destroy_all_rules_for_dev(NULL);
Dave Hudson87973cd2013-10-22 16:00:04 +01002207
Ken Zhu137722d2021-09-23 17:57:36 -07002208 cancel_delayed_work_sync(&si->sync_dwork);
Dave Hudson87973cd2013-10-22 16:00:04 +01002209
Dave Hudson87973cd2013-10-22 16:00:04 +01002210 unregister_chrdev(si->debug_dev, "sfe_ipv4");
2211
Xiaoping Fan640faf42015-08-28 15:50:55 -07002212#ifdef CONFIG_NF_FLOW_COOKIE
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +05302213 sysfs_remove_file(si->sys_ipv4, &sfe_ipv4_flow_cookie_attr.attr);
Xiaoping Fan640faf42015-08-28 15:50:55 -07002214#endif /* CONFIG_NF_FLOW_COOKIE */
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +05302215 sysfs_remove_file(si->sys_ipv4, &sfe_ipv4_debug_dev_attr.attr);
2216 sysfs_remove_file(si->sys_ipv4, &sfe_ipv4_cpu_attr.attr);
Dave Hudson87973cd2013-10-22 16:00:04 +01002217
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +05302218 kobject_put(si->sys_ipv4);
Dave Hudson87973cd2013-10-22 16:00:04 +01002219
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +05302220 free_percpu(si->stats_pcpu);
2221
Dave Hudsonaaf97ca2013-06-13 17:52:29 +01002222}
2223
Xiaoping Fand1dc7b22015-01-23 00:43:56 -08002224#ifdef CONFIG_NF_FLOW_COOKIE
2225EXPORT_SYMBOL(sfe_register_flow_cookie_cb);
2226EXPORT_SYMBOL(sfe_unregister_flow_cookie_cb);
2227#endif