blob: e614cbca74224045df6bd0daa2fc5c518b1e24fb [file] [log] [blame]
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +05301/*
2 * sfe_ipv4.h
3 * Shortcut forwarding engine header file for IPv4.
4 *
5 * Copyright (c) 2013-2016, 2019-2020, The Linux Foundation. All rights reserved.
Guduri Prathyusha647fe3e2021-11-22 19:17:51 +05306 * Copyright (c) 2021,2022 Qualcomm Innovation Center, Inc. All rights reserved.
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +05307 *
8 * Permission to use, copy, modify, and/or distribute this software for any
9 * purpose with or without fee is hereby granted, provided that the above
10 * copyright notice and this permission notice appear in all copies.
11 *
12 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19 */
20
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +053021#define SFE_IPV4_DSCP_MASK 0x3
22#define SFE_IPV4_DSCP_SHIFT 2
23
24/*
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +053025 * Specifies the lower bound on ACK numbers carried in the TCP header
26 */
27#define SFE_IPV4_TCP_MAX_ACK_WINDOW 65520
28
29/*
30 * IPv4 TCP connection match additional data.
31 */
32struct sfe_ipv4_tcp_connection_match {
33 u8 win_scale; /* Window scale */
34 u32 max_win; /* Maximum window size seen */
35 u32 end; /* Sequence number of the next byte to send (seq + segment length) */
36 u32 max_end; /* Sequence number of the last byte to ack */
37};
38
39/*
40 * Bit flags for IPv4 connection matching entry.
41 */
42#define SFE_IPV4_CONNECTION_MATCH_FLAG_XLATE_SRC (1<<0)
43 /* Perform source translation */
44#define SFE_IPV4_CONNECTION_MATCH_FLAG_XLATE_DEST (1<<1)
45 /* Perform destination translation */
46#define SFE_IPV4_CONNECTION_MATCH_FLAG_NO_SEQ_CHECK (1<<2)
47 /* Ignore TCP sequence numbers */
48#define SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_FAST_ETH_HDR (1<<3)
49 /* Fast Ethernet header write */
50#define SFE_IPV4_CONNECTION_MATCH_FLAG_WRITE_L2_HDR (1<<4)
51 /* Fast Ethernet header write */
52#define SFE_IPV4_CONNECTION_MATCH_FLAG_PRIORITY_REMARK (1<<5)
53 /* remark priority of SKB */
54#define SFE_IPV4_CONNECTION_MATCH_FLAG_DSCP_REMARK (1<<6)
55 /* remark DSCP of packet */
Ratheesh Kannotha3cf0e02021-12-09 09:44:10 +053056#define SFE_IPV4_CONNECTION_MATCH_FLAG_CSUM_OFFLOAD (1<<7)
57 /* checksum offload.*/
Guduri Prathyushaeb31c902021-11-10 20:18:50 +053058#define SFE_IPV4_CONNECTION_MATCH_FLAG_PPPOE_DECAP (1<<8)
59 /* Indicates that PPPoE should be decapsulated */
60#define SFE_IPV4_CONNECTION_MATCH_FLAG_PPPOE_ENCAP (1<<9)
61 /* Indicates that PPPoE should be encapsulated */
Ratheesh Kannoth71fc51e2022-01-05 10:02:47 +053062#define SFE_IPV4_CONNECTION_MATCH_FLAG_BRIDGE_FLOW (1<<10)
63 /* Bridge flow */
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +053064
65/*
66 * IPv4 connection matching structure.
67 */
68struct sfe_ipv4_connection_match {
69 /*
70 * References to other objects.
71 */
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +053072 struct hlist_node hnode;
73
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +053074 struct sfe_ipv4_connection *connection;
75 struct sfe_ipv4_connection_match *counter_match;
76 /* Matches the flow in the opposite direction as the one in *connection */
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +053077 /*
78 * Characteristics that identify flows that match this rule.
79 */
80 struct net_device *match_dev; /* Network device */
81 u8 match_protocol; /* Protocol */
82 __be32 match_src_ip; /* Source IP address */
83 __be32 match_dest_ip; /* Destination IP address */
84 __be16 match_src_port; /* Source port/connection ident */
85 __be16 match_dest_port; /* Destination port/connection ident */
86
Amitesh Anand63be37d2021-12-24 20:51:48 +053087 struct udp_sock *up; /* Stores UDP sock information; valid only in decap path */
88
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +053089 /*
90 * Control the operations of the match.
91 */
92 u32 flags; /* Bit flags */
93#ifdef CONFIG_NF_FLOW_COOKIE
94 u32 flow_cookie; /* used flow cookie, for debug */
95#endif
96#ifdef CONFIG_XFRM
97 u32 flow_accel; /* The flow accelerated or not */
98#endif
99
100 /*
101 * Connection state that we track once we match.
102 */
103 union { /* Protocol-specific state */
104 struct sfe_ipv4_tcp_connection_match tcp;
105 } protocol_state;
106 /*
107 * Stats recorded in a sync period. These stats will be added to
108 * rx_packet_count64/rx_byte_count64 after a sync period.
109 */
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530110 atomic_t rx_packet_count;
111 atomic_t rx_byte_count;
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +0530112
113 /*
114 * Packet translation information.
115 */
116 __be32 xlate_src_ip; /* Address after source translation */
117 __be16 xlate_src_port; /* Port/connection ident after source translation */
118 u16 xlate_src_csum_adjustment;
119 /* Transport layer checksum adjustment after source translation */
120 u16 xlate_src_partial_csum_adjustment;
121 /* Transport layer pseudo header checksum adjustment after source translation */
122
123 __be32 xlate_dest_ip; /* Address after destination translation */
124 __be16 xlate_dest_port; /* Port/connection ident after destination translation */
125 u16 xlate_dest_csum_adjustment;
126 /* Transport layer checksum adjustment after destination translation */
127 u16 xlate_dest_partial_csum_adjustment;
128 /* Transport layer pseudo header checksum adjustment after destination translation */
129
130 /*
131 * QoS information
132 */
133 u32 priority;
134 u32 dscp;
135
136 /*
137 * Packet transmit information.
138 */
139 struct net_device *xmit_dev; /* Network device on which to transmit */
140 unsigned short int xmit_dev_mtu;
141 /* Interface MTU */
142 u16 xmit_dest_mac[ETH_ALEN / 2];
143 /* Destination MAC address to use when forwarding */
144 u16 xmit_src_mac[ETH_ALEN / 2];
145 /* Source MAC address to use when forwarding */
146
147 /*
148 * Summary stats.
149 */
150 u64 rx_packet_count64;
151 u64 rx_byte_count64;
Guduri Prathyushaeb31c902021-11-10 20:18:50 +0530152
153 /*
154 * PPPoE information
155 */
156 u16 pppoe_session_id;
157 u8 pppoe_remote_mac[ETH_ALEN];
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +0530158};
159
160/*
161 * Per-connection data structure.
162 */
163struct sfe_ipv4_connection {
164 struct sfe_ipv4_connection *next;
165 /* Pointer to the next entry in a hash chain */
166 struct sfe_ipv4_connection *prev;
167 /* Pointer to the previous entry in a hash chain */
168 int protocol; /* IP protocol number */
169 __be32 src_ip; /* Src IP addr pre-translation */
170 __be32 src_ip_xlate; /* Src IP addr post-translation */
171 __be32 dest_ip; /* Dest IP addr pre-translation */
172 __be32 dest_ip_xlate; /* Dest IP addr post-translation */
173 __be16 src_port; /* Src port pre-translation */
174 __be16 src_port_xlate; /* Src port post-translation */
175 __be16 dest_port; /* Dest port pre-translation */
176 __be16 dest_port_xlate; /* Dest port post-translation */
177 struct sfe_ipv4_connection_match *original_match;
178 /* Original direction matching structure */
179 struct net_device *original_dev;
180 /* Original direction source device */
181 struct sfe_ipv4_connection_match *reply_match;
182 /* Reply direction matching structure */
183 struct net_device *reply_dev; /* Reply direction source device */
184 u64 last_sync_jiffies; /* Jiffies count for the last sync */
185 struct sfe_ipv4_connection *all_connections_next;
186 /* Pointer to the next entry in the list of all connections */
187 struct sfe_ipv4_connection *all_connections_prev;
188 /* Pointer to the previous entry in the list of all connections */
189 u32 mark; /* mark for outgoing packet */
190 u32 debug_read_seq; /* sequence number for debug dump */
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530191 bool removed; /* Indicates the connection is removed */
192 struct rcu_head rcu; /* delay rcu free */
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +0530193};
194
195/*
196 * IPv4 connections and hash table size information.
197 */
198#define SFE_IPV4_CONNECTION_HASH_SHIFT 12
199#define SFE_IPV4_CONNECTION_HASH_SIZE (1 << SFE_IPV4_CONNECTION_HASH_SHIFT)
200#define SFE_IPV4_CONNECTION_HASH_MASK (SFE_IPV4_CONNECTION_HASH_SIZE - 1)
201
202enum sfe_ipv4_exception_events {
203 SFE_IPV4_EXCEPTION_EVENT_UDP_HEADER_INCOMPLETE,
204 SFE_IPV4_EXCEPTION_EVENT_UDP_NO_CONNECTION,
205 SFE_IPV4_EXCEPTION_EVENT_UDP_IP_OPTIONS_OR_INITIAL_FRAGMENT,
206 SFE_IPV4_EXCEPTION_EVENT_UDP_SMALL_TTL,
207 SFE_IPV4_EXCEPTION_EVENT_UDP_NEEDS_FRAGMENTATION,
208 SFE_IPV4_EXCEPTION_EVENT_TCP_HEADER_INCOMPLETE,
209 SFE_IPV4_EXCEPTION_EVENT_TCP_NO_CONNECTION_SLOW_FLAGS,
210 SFE_IPV4_EXCEPTION_EVENT_TCP_NO_CONNECTION_FAST_FLAGS,
211 SFE_IPV4_EXCEPTION_EVENT_TCP_IP_OPTIONS_OR_INITIAL_FRAGMENT,
212 SFE_IPV4_EXCEPTION_EVENT_TCP_SMALL_TTL,
213 SFE_IPV4_EXCEPTION_EVENT_TCP_NEEDS_FRAGMENTATION,
214 SFE_IPV4_EXCEPTION_EVENT_TCP_FLAGS,
215 SFE_IPV4_EXCEPTION_EVENT_TCP_SEQ_EXCEEDS_RIGHT_EDGE,
216 SFE_IPV4_EXCEPTION_EVENT_TCP_SMALL_DATA_OFFS,
217 SFE_IPV4_EXCEPTION_EVENT_TCP_BAD_SACK,
218 SFE_IPV4_EXCEPTION_EVENT_TCP_BIG_DATA_OFFS,
219 SFE_IPV4_EXCEPTION_EVENT_TCP_SEQ_BEFORE_LEFT_EDGE,
220 SFE_IPV4_EXCEPTION_EVENT_TCP_ACK_EXCEEDS_RIGHT_EDGE,
221 SFE_IPV4_EXCEPTION_EVENT_TCP_ACK_BEFORE_LEFT_EDGE,
222 SFE_IPV4_EXCEPTION_EVENT_ICMP_HEADER_INCOMPLETE,
223 SFE_IPV4_EXCEPTION_EVENT_ICMP_UNHANDLED_TYPE,
224 SFE_IPV4_EXCEPTION_EVENT_ICMP_IPV4_HEADER_INCOMPLETE,
225 SFE_IPV4_EXCEPTION_EVENT_ICMP_IPV4_NON_V4,
226 SFE_IPV4_EXCEPTION_EVENT_ICMP_IPV4_IP_OPTIONS_INCOMPLETE,
227 SFE_IPV4_EXCEPTION_EVENT_ICMP_IPV4_UDP_HEADER_INCOMPLETE,
228 SFE_IPV4_EXCEPTION_EVENT_ICMP_IPV4_TCP_HEADER_INCOMPLETE,
229 SFE_IPV4_EXCEPTION_EVENT_ICMP_IPV4_UNHANDLED_PROTOCOL,
230 SFE_IPV4_EXCEPTION_EVENT_ICMP_NO_CONNECTION,
231 SFE_IPV4_EXCEPTION_EVENT_ICMP_FLUSHED_CONNECTION,
232 SFE_IPV4_EXCEPTION_EVENT_HEADER_INCOMPLETE,
Ratheesh Kannoth43d64f82021-10-20 08:23:29 +0530233 SFE_IPV4_EXCEPTION_EVENT_HEADER_CSUM_BAD,
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +0530234 SFE_IPV4_EXCEPTION_EVENT_BAD_TOTAL_LENGTH,
235 SFE_IPV4_EXCEPTION_EVENT_NON_V4,
236 SFE_IPV4_EXCEPTION_EVENT_NON_INITIAL_FRAGMENT,
237 SFE_IPV4_EXCEPTION_EVENT_DATAGRAM_INCOMPLETE,
238 SFE_IPV4_EXCEPTION_EVENT_IP_OPTIONS_INCOMPLETE,
239 SFE_IPV4_EXCEPTION_EVENT_UNHANDLED_PROTOCOL,
Guduri Prathyusha79a5fee2021-11-11 17:59:10 +0530240 SFE_IPV4_EXCEPTION_EVENT_PPPOE_HEADER_ENCAP_FAILED,
Guduri Prathyusha647fe3e2021-11-22 19:17:51 +0530241 SFE_IPV4_EXCEPTION_EVENT_INVALID_PPPOE_SESSION,
242 SFE_IPV4_EXCEPTION_EVENT_INCORRECT_PPPOE_PARSING,
243 SFE_IPV4_EXCEPTION_EVENT_PPPOE_NOT_SET_IN_CME,
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +0530244 SFE_IPV4_EXCEPTION_EVENT_LAST
245};
246
247/*
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +0530248 * per CPU stats
249 */
250struct sfe_ipv4_stats {
251 /*
252 * Stats recorded in a sync period. These stats will be added to
253 * connection_xxx64 after a sync period.
254 */
255 u64 connection_create_requests64;
256 /* Number of IPv4 connection create requests */
257 u64 connection_create_collisions64;
258 /* Number of IPv4 connection create requests that collided with existing hash table entries */
Ratheesh Kannoth89302a72021-10-20 08:10:37 +0530259 u64 connection_create_failures64;
260 /* Number of IPv4 connection create requests that failed */
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +0530261 u64 connection_destroy_requests64;
262 /* Number of IPv4 connection destroy requests */
263 u64 connection_destroy_misses64;
264 /* Number of IPv4 connection destroy requests that missed our hash table */
265 u64 connection_match_hash_hits64;
266 /* Number of IPv4 connection match hash hits */
267 u64 connection_match_hash_reorders64;
268 /* Number of IPv4 connection match hash reorders */
269 u64 connection_flushes64; /* Number of IPv4 connection flushes */
Amitesh Anand63be37d2021-12-24 20:51:48 +0530270 u64 packets_dropped64; /* Number of IPv4 packets dropped */
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +0530271 u64 packets_forwarded64; /* Number of IPv4 packets forwarded */
272 u64 packets_not_forwarded64; /* Number of IPv4 packets not forwarded */
273 u64 exception_events64[SFE_IPV4_EXCEPTION_EVENT_LAST];
Guduri Prathyusha79a5fee2021-11-11 17:59:10 +0530274 u64 pppoe_encap_packets_forwarded64; /* Number of IPv4 PPPOE encap packets forwarded */
Guduri Prathyusha647fe3e2021-11-22 19:17:51 +0530275 u64 pppoe_decap_packets_forwarded64; /* Number of IPv4 PPPOE decap packets forwarded */
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +0530276};
277
278/*
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +0530279 * Per-module structure.
280 */
281struct sfe_ipv4 {
282 spinlock_t lock; /* Lock for SMP correctness */
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +0530283 struct sfe_ipv4_connection *all_connections_head;
284 /* Head of the list of all connections */
285 struct sfe_ipv4_connection *all_connections_tail;
286 /* Tail of the list of all connections */
287 unsigned int num_connections; /* Number of connections */
Ken Zhu137722d2021-09-23 17:57:36 -0700288 struct delayed_work sync_dwork; /* Work to sync the statistics */
289 unsigned int work_cpu; /* The core to run stats sync on */
290
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +0530291 sfe_sync_rule_callback_t __rcu sync_rule_callback;
292 /* Callback function registered by a connection manager for stats syncing */
293 struct sfe_ipv4_connection *conn_hash[SFE_IPV4_CONNECTION_HASH_SIZE];
294 /* Connection hash table */
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530295
296 struct hlist_head hlist_conn_match_hash_head[SFE_IPV4_CONNECTION_HASH_SIZE];
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +0530297 /* Connection match hash table */
Ratheesh Kannoth94fc5b82021-10-20 07:45:06 +0530298
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +0530299#ifdef CONFIG_NF_FLOW_COOKIE
300 struct sfe_flow_cookie_entry sfe_flow_cookie_table[SFE_FLOW_COOKIE_SIZE];
301 /* flow cookie table*/
302 flow_cookie_set_func_t flow_cookie_set_func;
303 /* function used to configure flow cookie in hardware*/
304 int flow_cookie_enable;
305 /* Enable/disable flow cookie at runtime */
306#endif
307
Ratheesh Kannoth3aeb2892021-10-20 07:57:15 +0530308 struct sfe_ipv4_stats __percpu *stats_pcpu;
309 /* Per CPU statistics. */
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +0530310
Ken Zhudc423672021-09-02 18:27:01 -0700311 struct sfe_ipv4_connection *wc_next; /* Connection list walk pointer for stats sync */
312
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +0530313 /*
314 * Control state.
315 */
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +0530316 struct kobject *sys_ipv4; /* sysfs linkage */
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +0530317 int debug_dev; /* Major number of the debug char device */
318 u32 debug_read_seq; /* sequence number for debug dump */
319};
320
321/*
322 * Enumeration of the XML output.
323 */
324enum sfe_ipv4_debug_xml_states {
325 SFE_IPV4_DEBUG_XML_STATE_START,
326 SFE_IPV4_DEBUG_XML_STATE_CONNECTIONS_START,
327 SFE_IPV4_DEBUG_XML_STATE_CONNECTIONS_CONNECTION,
328 SFE_IPV4_DEBUG_XML_STATE_CONNECTIONS_END,
329 SFE_IPV4_DEBUG_XML_STATE_EXCEPTIONS_START,
330 SFE_IPV4_DEBUG_XML_STATE_EXCEPTIONS_EXCEPTION,
331 SFE_IPV4_DEBUG_XML_STATE_EXCEPTIONS_END,
332 SFE_IPV4_DEBUG_XML_STATE_STATS,
333 SFE_IPV4_DEBUG_XML_STATE_END,
334 SFE_IPV4_DEBUG_XML_STATE_DONE
335};
336
337/*
338 * XML write state.
339 */
340struct sfe_ipv4_debug_xml_write_state {
341 enum sfe_ipv4_debug_xml_states state;
342 /* XML output file state machine state */
343 int iter_exception; /* Next exception iterator */
344};
345
346typedef bool (*sfe_ipv4_debug_xml_write_method_t)(struct sfe_ipv4 *si, char *buffer, char *msg, size_t *length,
347 int *total_read, struct sfe_ipv4_debug_xml_write_state *ws);
348
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +0530349u16 sfe_ipv4_gen_ip_csum(struct iphdr *iph);
350void sfe_ipv4_exception_stats_inc(struct sfe_ipv4 *si, enum sfe_ipv4_exception_events reason);
351bool sfe_ipv4_remove_connection(struct sfe_ipv4 *si, struct sfe_ipv4_connection *c);
352void sfe_ipv4_flush_connection(struct sfe_ipv4 *si, struct sfe_ipv4_connection *c, sfe_sync_reason_t reason);
353
354struct sfe_ipv4_connection_match *
355sfe_ipv4_find_connection_match_rcu(struct sfe_ipv4 *si, struct net_device *dev, u8 protocol,
356 __be32 src_ip, __be16 src_port,
357 __be32 dest_ip, __be16 dest_port);
358
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +0530359void sfe_ipv4_exit(void);
360int sfe_ipv4_init(void);