blob: f737f7970eeea56b2f217e511ae38317c43898e8 [file] [log] [blame]
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +05301/*
2 * sfe_ipv6.h
3 * Shortcut forwarding engine header file for IPv6.
4 *
5 * Copyright (c) 2015-2016, 2019-2020, The Linux Foundation. All rights reserved.
Guduri Prathyusha647fe3e2021-11-22 19:17:51 +05306 * Copyright (c) 2021,2022 Qualcomm Innovation Center, Inc. All rights reserved.
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +05307 *
8 * Permission to use, copy, modify, and/or distribute this software for any
9 * purpose with or without fee is hereby granted, provided that the above
10 * copyright notice and this permission notice appear in all copies.
11 *
12 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19 */
20
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +053021#define CHAR_DEV_MSG_SIZE 768
22
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +053023#define SFE_IPV6_DSCP_MASK 0xf03f
24#define SFE_IPV6_DSCP_SHIFT 2
25
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +053026#define SFE_IPV6_FRAG_OFFSET 0xfff8
27
28/*
29 * generic IPv6 extension header
30 */
31struct sfe_ipv6_ext_hdr {
32 __u8 next_hdr;
33 __u8 hdr_len;
34 __u8 padding[6];
Ratheesh Kannoth741f7992021-10-20 07:39:52 +053035};
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +053036
37/*
38 * Specifies the lower bound on ACK numbers carried in the TCP header
39 */
40#define SFE_IPV6_TCP_MAX_ACK_WINDOW 65520
41
42/*
43 * IPv6 TCP connection match additional data.
44 */
45struct sfe_ipv6_tcp_connection_match {
46 u8 win_scale; /* Window scale */
47 u32 max_win; /* Maximum window size seen */
48 u32 end; /* Sequence number of the next byte to send (seq + segment length) */
49 u32 max_end; /* Sequence number of the last byte to ack */
50};
51
52/*
53 * Bit flags for IPv6 connection matching entry.
54 */
55#define SFE_IPV6_CONNECTION_MATCH_FLAG_XLATE_SRC (1<<0)
56 /* Perform source translation */
57#define SFE_IPV6_CONNECTION_MATCH_FLAG_XLATE_DEST (1<<1)
58 /* Perform destination translation */
59#define SFE_IPV6_CONNECTION_MATCH_FLAG_NO_SEQ_CHECK (1<<2)
60 /* Ignore TCP sequence numbers */
61#define SFE_IPV6_CONNECTION_MATCH_FLAG_WRITE_FAST_ETH_HDR (1<<3)
62 /* Fast Ethernet header write */
63#define SFE_IPV6_CONNECTION_MATCH_FLAG_WRITE_L2_HDR (1<<4)
64 /* Fast Ethernet header write */
65#define SFE_IPV6_CONNECTION_MATCH_FLAG_PRIORITY_REMARK (1<<5)
66 /* remark priority of SKB */
67#define SFE_IPV6_CONNECTION_MATCH_FLAG_DSCP_REMARK (1<<6)
68 /* remark DSCP of packet */
Ratheesh Kannotha3cf0e02021-12-09 09:44:10 +053069#define SFE_IPV6_CONNECTION_MATCH_FLAG_CSUM_OFFLOAD (1<<7)
70 /* checksum offload.*/
Guduri Prathyushaeb31c902021-11-10 20:18:50 +053071#define SFE_IPV6_CONNECTION_MATCH_FLAG_PPPOE_DECAP (1<<8)
72 /* Indicates that PPPoE should be decapsulated */
73#define SFE_IPV6_CONNECTION_MATCH_FLAG_PPPOE_ENCAP (1<<9)
74 /* Indicates that PPPoE should be encapsulated */
Ratheesh Kannoth71fc51e2022-01-05 10:02:47 +053075#define SFE_IPV6_CONNECTION_MATCH_FLAG_BRIDGE_FLOW (1<<10)
76 /* Bridge flow */
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +053077
78/*
79 * IPv6 connection matching structure.
80 */
81struct sfe_ipv6_connection_match {
82 /*
83 * References to other objects.
84 */
Ratheesh Kannotha212fc52021-10-20 07:50:32 +053085 struct hlist_node hnode;
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +053086 struct sfe_ipv6_connection *connection;
87 struct sfe_ipv6_connection_match *counter_match;
88 /* Matches the flow in the opposite direction as the one in connection */
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +053089 /*
90 * Characteristics that identify flows that match this rule.
91 */
92 struct net_device *match_dev; /* Network device */
93 u8 match_protocol; /* Protocol */
94 struct sfe_ipv6_addr match_src_ip[1]; /* Source IP address */
95 struct sfe_ipv6_addr match_dest_ip[1]; /* Destination IP address */
96 __be16 match_src_port; /* Source port/connection ident */
97 __be16 match_dest_port; /* Destination port/connection ident */
98
Suruchi Suman23a279d2021-11-16 15:13:09 +053099 struct udp_sock *up; /* Stores UDP sock information; valid only in decap path */
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +0530100 /*
101 * Control the operations of the match.
102 */
103 u32 flags; /* Bit flags */
104#ifdef CONFIG_NF_FLOW_COOKIE
105 u32 flow_cookie; /* used flow cookie, for debug */
106#endif
107#ifdef CONFIG_XFRM
108 u32 flow_accel; /* The flow accelerated or not */
109#endif
110
111 /*
112 * Connection state that we track once we match.
113 */
114 union { /* Protocol-specific state */
115 struct sfe_ipv6_tcp_connection_match tcp;
116 } protocol_state;
117 /*
118 * Stats recorded in a sync period. These stats will be added to
119 * rx_packet_count64/rx_byte_count64 after a sync period.
120 */
Ratheesh Kannotha212fc52021-10-20 07:50:32 +0530121 atomic_t rx_packet_count;
122 atomic_t rx_byte_count;
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +0530123
124 /*
125 * Packet translation information.
126 */
127 struct sfe_ipv6_addr xlate_src_ip[1]; /* Address after source translation */
128 __be16 xlate_src_port; /* Port/connection ident after source translation */
129 u16 xlate_src_csum_adjustment;
130 /* Transport layer checksum adjustment after source translation */
131 struct sfe_ipv6_addr xlate_dest_ip[1]; /* Address after destination translation */
132 __be16 xlate_dest_port; /* Port/connection ident after destination translation */
133 u16 xlate_dest_csum_adjustment;
134 /* Transport layer checksum adjustment after destination translation */
135
136 /*
137 * QoS information
138 */
139 u32 priority;
140 u32 dscp;
141
142 /*
143 * Packet transmit information.
144 */
145 struct net_device *xmit_dev; /* Network device on which to transmit */
146 unsigned short int xmit_dev_mtu;
147 /* Interface MTU */
148 u16 xmit_dest_mac[ETH_ALEN / 2];
149 /* Destination MAC address to use when forwarding */
150 u16 xmit_src_mac[ETH_ALEN / 2];
151 /* Source MAC address to use when forwarding */
152
153 /*
154 * Summary stats.
155 */
156 u64 rx_packet_count64;
157 u64 rx_byte_count64;
Guduri Prathyushaeb31c902021-11-10 20:18:50 +0530158
159 /*
160 * PPPoE information.
161 */
162 u16 pppoe_session_id;
163 u8 pppoe_remote_mac[ETH_ALEN];
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +0530164};
165
166/*
167 * Per-connection data structure.
168 */
169struct sfe_ipv6_connection {
170 struct sfe_ipv6_connection *next;
171 /* Pointer to the next entry in a hash chain */
172 struct sfe_ipv6_connection *prev;
173 /* Pointer to the previous entry in a hash chain */
174 int protocol; /* IP protocol number */
175 struct sfe_ipv6_addr src_ip[1]; /* Src IP addr pre-translation */
176 struct sfe_ipv6_addr src_ip_xlate[1]; /* Src IP addr post-translation */
177 struct sfe_ipv6_addr dest_ip[1]; /* Dest IP addr pre-translation */
178 struct sfe_ipv6_addr dest_ip_xlate[1]; /* Dest IP addr post-translation */
179 __be16 src_port; /* Src port pre-translation */
180 __be16 src_port_xlate; /* Src port post-translation */
181 __be16 dest_port; /* Dest port pre-translation */
182 __be16 dest_port_xlate; /* Dest port post-translation */
183 struct sfe_ipv6_connection_match *original_match;
184 /* Original direction matching structure */
185 struct net_device *original_dev;
186 /* Original direction source device */
187 struct sfe_ipv6_connection_match *reply_match;
188 /* Reply direction matching structure */
189 struct net_device *reply_dev; /* Reply direction source device */
190 u64 last_sync_jiffies; /* Jiffies count for the last sync */
191 struct sfe_ipv6_connection *all_connections_next;
192 /* Pointer to the next entry in the list of all connections */
193 struct sfe_ipv6_connection *all_connections_prev;
194 /* Pointer to the previous entry in the list of all connections */
Ratheesh Kannotha212fc52021-10-20 07:50:32 +0530195 bool removed; /* Indicates the connection is removed */
196 struct rcu_head rcu; /* delay rcu free */
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +0530197 u32 mark; /* mark for outgoing packet */
198 u32 debug_read_seq; /* sequence number for debug dump */
199};
200
201/*
202 * IPv6 connections and hash table size information.
203 */
204#define SFE_IPV6_CONNECTION_HASH_SHIFT 12
205#define SFE_IPV6_CONNECTION_HASH_SIZE (1 << SFE_IPV6_CONNECTION_HASH_SHIFT)
206#define SFE_IPV6_CONNECTION_HASH_MASK (SFE_IPV6_CONNECTION_HASH_SIZE - 1)
207
208enum sfe_ipv6_exception_events {
209 SFE_IPV6_EXCEPTION_EVENT_UDP_HEADER_INCOMPLETE,
210 SFE_IPV6_EXCEPTION_EVENT_UDP_NO_CONNECTION,
211 SFE_IPV6_EXCEPTION_EVENT_UDP_IP_OPTIONS_OR_INITIAL_FRAGMENT,
212 SFE_IPV6_EXCEPTION_EVENT_UDP_SMALL_TTL,
213 SFE_IPV6_EXCEPTION_EVENT_UDP_NEEDS_FRAGMENTATION,
214 SFE_IPV6_EXCEPTION_EVENT_TCP_HEADER_INCOMPLETE,
215 SFE_IPV6_EXCEPTION_EVENT_TCP_NO_CONNECTION_SLOW_FLAGS,
216 SFE_IPV6_EXCEPTION_EVENT_TCP_NO_CONNECTION_FAST_FLAGS,
217 SFE_IPV6_EXCEPTION_EVENT_TCP_IP_OPTIONS_OR_INITIAL_FRAGMENT,
218 SFE_IPV6_EXCEPTION_EVENT_TCP_SMALL_TTL,
219 SFE_IPV6_EXCEPTION_EVENT_TCP_NEEDS_FRAGMENTATION,
220 SFE_IPV6_EXCEPTION_EVENT_TCP_FLAGS,
221 SFE_IPV6_EXCEPTION_EVENT_TCP_SEQ_EXCEEDS_RIGHT_EDGE,
222 SFE_IPV6_EXCEPTION_EVENT_TCP_SMALL_DATA_OFFS,
223 SFE_IPV6_EXCEPTION_EVENT_TCP_BAD_SACK,
224 SFE_IPV6_EXCEPTION_EVENT_TCP_BIG_DATA_OFFS,
225 SFE_IPV6_EXCEPTION_EVENT_TCP_SEQ_BEFORE_LEFT_EDGE,
226 SFE_IPV6_EXCEPTION_EVENT_TCP_ACK_EXCEEDS_RIGHT_EDGE,
227 SFE_IPV6_EXCEPTION_EVENT_TCP_ACK_BEFORE_LEFT_EDGE,
228 SFE_IPV6_EXCEPTION_EVENT_ICMP_HEADER_INCOMPLETE,
229 SFE_IPV6_EXCEPTION_EVENT_ICMP_UNHANDLED_TYPE,
230 SFE_IPV6_EXCEPTION_EVENT_ICMP_IPV6_HEADER_INCOMPLETE,
231 SFE_IPV6_EXCEPTION_EVENT_ICMP_IPV6_NON_V6,
232 SFE_IPV6_EXCEPTION_EVENT_ICMP_IPV6_IP_OPTIONS_INCOMPLETE,
233 SFE_IPV6_EXCEPTION_EVENT_ICMP_IPV6_UDP_HEADER_INCOMPLETE,
234 SFE_IPV6_EXCEPTION_EVENT_ICMP_IPV6_TCP_HEADER_INCOMPLETE,
235 SFE_IPV6_EXCEPTION_EVENT_ICMP_IPV6_UNHANDLED_PROTOCOL,
236 SFE_IPV6_EXCEPTION_EVENT_ICMP_NO_CONNECTION,
237 SFE_IPV6_EXCEPTION_EVENT_ICMP_FLUSHED_CONNECTION,
238 SFE_IPV6_EXCEPTION_EVENT_HEADER_INCOMPLETE,
239 SFE_IPV6_EXCEPTION_EVENT_BAD_TOTAL_LENGTH,
240 SFE_IPV6_EXCEPTION_EVENT_NON_V6,
241 SFE_IPV6_EXCEPTION_EVENT_NON_INITIAL_FRAGMENT,
242 SFE_IPV6_EXCEPTION_EVENT_DATAGRAM_INCOMPLETE,
243 SFE_IPV6_EXCEPTION_EVENT_IP_OPTIONS_INCOMPLETE,
244 SFE_IPV6_EXCEPTION_EVENT_UNHANDLED_PROTOCOL,
245 SFE_IPV6_EXCEPTION_EVENT_FLOW_COOKIE_ADD_FAIL,
Guduri Prathyusha79a5fee2021-11-11 17:59:10 +0530246 SFE_IPV6_EXCEPTION_EVENT_PPPOE_HEADER_ENCAP_FAILED,
Guduri Prathyusha647fe3e2021-11-22 19:17:51 +0530247 SFE_IPV6_EXCEPTION_EVENT_INVALID_PPPOE_SESSION,
248 SFE_IPV6_EXCEPTION_EVENT_INCORRECT_PPPOE_PARSING,
249 SFE_IPV6_EXCEPTION_EVENT_PPPOE_NOT_SET_IN_CME,
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +0530250 SFE_IPV6_EXCEPTION_EVENT_LAST
251};
252
253/*
Ratheesh Kannoth1ed95462021-10-20 07:57:45 +0530254 * Per CPU stats
255 */
256struct sfe_ipv6_stats {
257 /*
258 * Stats recorded in a sync period. These stats will be added to
259 * connection_xxx64 after a sync period.
260 */
261 u64 connection_create_requests64;
262 /* Number of IPv6 connection create requests */
263 u64 connection_create_collisions64;
264 /* Number of IPv6 connection create requests that collided with existing hash table entries */
Ratheesh Kannoth89302a72021-10-20 08:10:37 +0530265 u64 connection_create_failures64;
266 /* Number of IPv6 connection create requests failures. */
267
Ratheesh Kannoth1ed95462021-10-20 07:57:45 +0530268 u64 connection_destroy_requests64;
269 /* Number of IPv6 connection destroy requests */
270 u64 connection_destroy_misses64;
271 /* Number of IPv6 connection destroy requests that missed our hash table */
272 u64 connection_match_hash_hits64;
273 /* Number of IPv6 connection match hash hits */
274 u64 connection_match_hash_reorders64;
275 /* Number of IPv6 connection match hash reorders */
276 u64 connection_flushes64; /* Number of IPv6 connection flushes */
Suruchi Suman23a279d2021-11-16 15:13:09 +0530277 u64 packets_dropped64; /* Number of IPv4 packets dropped */
Ratheesh Kannoth1ed95462021-10-20 07:57:45 +0530278 u64 packets_forwarded64; /* Number of IPv6 packets forwarded */
279 u64 packets_not_forwarded64; /* Number of IPv6 packets not forwarded */
280 u64 exception_events64[SFE_IPV6_EXCEPTION_EVENT_LAST];
Guduri Prathyusha79a5fee2021-11-11 17:59:10 +0530281 u64 pppoe_encap_packets_forwarded64; /* Number of IPv6 PPPOE encap packets forwarded */
Guduri Prathyusha647fe3e2021-11-22 19:17:51 +0530282 u64 pppoe_decap_packets_forwarded64; /* Number of IPv6 PPPOE decap packets forwarded */
Ratheesh Kannoth1ed95462021-10-20 07:57:45 +0530283};
284
285/*
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +0530286 * Per-module structure.
287 */
288struct sfe_ipv6 {
289 spinlock_t lock; /* Lock for SMP correctness */
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +0530290 struct sfe_ipv6_connection *all_connections_head;
291 /* Head of the list of all connections */
292 struct sfe_ipv6_connection *all_connections_tail;
293 /* Tail of the list of all connections */
294 unsigned int num_connections; /* Number of connections */
Ken Zhu137722d2021-09-23 17:57:36 -0700295 struct delayed_work sync_dwork; /* Work to sync the statistics */
296 unsigned int work_cpu; /* The core to run stats sync on */
297
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +0530298 sfe_sync_rule_callback_t __rcu sync_rule_callback;
299 /* Callback function registered by a connection manager for stats syncing */
300 struct sfe_ipv6_connection *conn_hash[SFE_IPV6_CONNECTION_HASH_SIZE];
301 /* Connection hash table */
Ratheesh Kannotha212fc52021-10-20 07:50:32 +0530302 struct hlist_head hlist_conn_match_hash_head[SFE_IPV6_CONNECTION_HASH_SIZE];
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +0530303#ifdef CONFIG_NF_FLOW_COOKIE
304 struct sfe_ipv6_flow_cookie_entry sfe_flow_cookie_table[SFE_FLOW_COOKIE_SIZE];
305 /* flow cookie table*/
306 sfe_ipv6_flow_cookie_set_func_t flow_cookie_set_func;
307 /* function used to configure flow cookie in hardware*/
308 int flow_cookie_enable;
309 /* Enable/disable flow cookie at runtime */
310#endif
311
Ratheesh Kannoth1ed95462021-10-20 07:57:45 +0530312 struct sfe_ipv6_stats __percpu *stats_pcpu;
313 /* Common SFE counters. */
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +0530314
Ken Zhu32b95392021-09-03 13:52:04 -0700315 struct sfe_ipv6_connection *wc_next;
316 /* The next walk point in the all connection list*/
317
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +0530318 /*
319 * Control state.
320 */
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +0530321 struct kobject *sys_ipv6; /* sysfs linkage */
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +0530322 int debug_dev; /* Major number of the debug char device */
323 u32 debug_read_seq; /* sequence number for debug dump */
324};
325
326/*
327 * Enumeration of the XML output.
328 */
329enum sfe_ipv6_debug_xml_states {
330 SFE_IPV6_DEBUG_XML_STATE_START,
331 SFE_IPV6_DEBUG_XML_STATE_CONNECTIONS_START,
332 SFE_IPV6_DEBUG_XML_STATE_CONNECTIONS_CONNECTION,
333 SFE_IPV6_DEBUG_XML_STATE_CONNECTIONS_END,
334 SFE_IPV6_DEBUG_XML_STATE_EXCEPTIONS_START,
335 SFE_IPV6_DEBUG_XML_STATE_EXCEPTIONS_EXCEPTION,
336 SFE_IPV6_DEBUG_XML_STATE_EXCEPTIONS_END,
337 SFE_IPV6_DEBUG_XML_STATE_STATS,
338 SFE_IPV6_DEBUG_XML_STATE_END,
339 SFE_IPV6_DEBUG_XML_STATE_DONE
340};
341
342/*
343 * XML write state.
344 */
345struct sfe_ipv6_debug_xml_write_state {
346 enum sfe_ipv6_debug_xml_states state;
347 /* XML output file state machine state */
348 int iter_exception; /* Next exception iterator */
349};
350
351typedef bool (*sfe_ipv6_debug_xml_write_method_t)(struct sfe_ipv6 *si, char *buffer, char *msg, size_t *length,
352 int *total_read, struct sfe_ipv6_debug_xml_write_state *ws);
353
Ratheesh Kannoth6307bec2021-11-25 08:26:39 +0530354/*
355 * sfe_ipv6_is_ext_hdr()
356 * check if we recognize ipv6 extension header
357 */
358static inline bool sfe_ipv6_is_ext_hdr(u8 hdr)
359{
360 return (hdr == NEXTHDR_HOP) ||
361 (hdr == NEXTHDR_ROUTING) ||
362 (hdr == NEXTHDR_FRAGMENT) ||
363 (hdr == NEXTHDR_AUTH) ||
364 (hdr == NEXTHDR_DEST) ||
365 (hdr == NEXTHDR_MOBILITY);
366}
367
368/*
369 * sfe_ipv6_change_dsfield()
370 * change dscp field in IPv6 packet
371 */
372static inline void sfe_ipv6_change_dsfield(struct ipv6hdr *iph, u8 dscp)
373{
374 __be16 *p = (__be16 *)iph;
375
376 *p = ((*p & htons(SFE_IPV6_DSCP_MASK)) | htons((u16)dscp << 4));
377}
378
379void sfe_ipv6_exception_stats_inc(struct sfe_ipv6 *si, enum sfe_ipv6_exception_events reason);
380
381struct sfe_ipv6_connection_match *
382sfe_ipv6_find_connection_match_rcu(struct sfe_ipv6 *si, struct net_device *dev, u8 protocol,
383 struct sfe_ipv6_addr *src_ip, __be16 src_port,
384 struct sfe_ipv6_addr *dest_ip, __be16 dest_port);
385
386bool sfe_ipv6_remove_connection(struct sfe_ipv6 *si, struct sfe_ipv6_connection *c);
387
388void sfe_ipv6_flush_connection(struct sfe_ipv6 *si,
389 struct sfe_ipv6_connection *c,
390 sfe_sync_reason_t reason);
391
Ratheesh Kannoth24fb1db2021-10-20 07:28:06 +0530392void sfe_ipv6_exit(void);
393int sfe_ipv6_init(void);