blob: 825099a276d353a987a9ac7b53a73131a34edf68 [file] [log] [blame]
Dave Barach68b0fb02017-02-28 15:15:56 -05001/*
Florin Coras222e1f412019-02-16 20:47:32 -08002 * Copyright (c) 2016-2019 Cisco and/or its affiliates.
Dave Barach68b0fb02017-02-28 15:15:56 -05003 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16#ifndef _vnet_tcp_h_
17#define _vnet_tcp_h_
18
19#include <vnet/vnet.h>
20#include <vnet/ip/ip.h>
21#include <vnet/tcp/tcp_packet.h>
22#include <vnet/tcp/tcp_timer.h>
23#include <vnet/session/transport.h>
24#include <vnet/session/session.h>
Florin Corase69f4952017-03-07 10:06:24 -080025#include <vnet/tcp/tcp_debug.h>
Dave Barach68b0fb02017-02-28 15:15:56 -050026
Florin Coras3af90fc2017-05-03 21:09:42 -070027#define TCP_TICK 0.001 /**< TCP tick period (s) */
28#define THZ (u32) (1/TCP_TICK) /**< TCP tick frequency */
Dave Barach68b0fb02017-02-28 15:15:56 -050029#define TCP_TSTAMP_RESOLUTION TCP_TICK /**< Time stamp resolution */
30#define TCP_PAWS_IDLE 24 * 24 * 60 * 60 * THZ /**< 24 days */
Florin Corasf6359c82017-06-19 12:26:09 -040031#define TCP_FIB_RECHECK_PERIOD 1 * THZ /**< Recheck every 1s */
Dave Barach68b0fb02017-02-28 15:15:56 -050032#define TCP_MAX_OPTION_SPACE 40
Florin Corasaa01abb2018-11-12 09:13:10 -080033#define TCP_CC_DATA_SZ 24
Simon Zhang1146ff42019-09-02 22:54:00 +080034#define TCP_MAX_GSO_SZ 65536
Florin Coras36ebcff2019-09-12 18:36:44 -070035#define TCP_RXT_MAX_BURST 10
Dave Barach68b0fb02017-02-28 15:15:56 -050036
Florin Coras6792ec02017-03-13 03:49:51 -070037#define TCP_DUPACK_THRESHOLD 3
Florin Coras6792ec02017-03-13 03:49:51 -070038#define TCP_IW_N_SEGMENTS 10
Florin Corasf03a59a2017-06-09 21:07:32 -070039#define TCP_ALWAYS_ACK 1 /**< On/off delayed acks */
Florin Coras93992a92017-05-24 18:03:56 -070040#define TCP_USE_SACKS 1 /**< Disable only for testing */
Dave Barach68b0fb02017-02-28 15:15:56 -050041
42/** TCP FSM state definitions as per RFC793. */
43#define foreach_tcp_fsm_state \
44 _(CLOSED, "CLOSED") \
45 _(LISTEN, "LISTEN") \
46 _(SYN_SENT, "SYN_SENT") \
47 _(SYN_RCVD, "SYN_RCVD") \
48 _(ESTABLISHED, "ESTABLISHED") \
49 _(CLOSE_WAIT, "CLOSE_WAIT") \
50 _(FIN_WAIT_1, "FIN_WAIT_1") \
51 _(LAST_ACK, "LAST_ACK") \
52 _(CLOSING, "CLOSING") \
53 _(FIN_WAIT_2, "FIN_WAIT_2") \
54 _(TIME_WAIT, "TIME_WAIT")
55
56typedef enum _tcp_state
57{
58#define _(sym, str) TCP_STATE_##sym,
59 foreach_tcp_fsm_state
60#undef _
61 TCP_N_STATES
62} tcp_state_t;
63
64format_function_t format_tcp_state;
Dave Barach1f75cfd2017-04-14 16:46:44 -040065format_function_t format_tcp_flags;
Florin Coras45d34962017-04-25 00:05:27 -070066format_function_t format_tcp_sacks;
Florin Coras3eb50622017-07-13 01:24:57 -040067format_function_t format_tcp_rcv_sacks;
Dave Barach68b0fb02017-02-28 15:15:56 -050068
69/** TCP timers */
70#define foreach_tcp_timer \
71 _(RETRANSMIT, "RETRANSMIT") \
72 _(DELACK, "DELAYED ACK") \
73 _(PERSIST, "PERSIST") \
Florin Corasd79b41e2017-03-04 05:37:52 -080074 _(WAITCLOSE, "WAIT CLOSE") \
75 _(RETRANSMIT_SYN, "RETRANSMIT SYN") \
Dave Barach68b0fb02017-02-28 15:15:56 -050076
77typedef enum _tcp_timers
78{
79#define _(sym, str) TCP_TIMER_##sym,
80 foreach_tcp_timer
81#undef _
82 TCP_N_TIMERS
83} tcp_timers_e;
84
85typedef void (timer_expiration_handler) (u32 index);
86
87extern timer_expiration_handler tcp_timer_delack_handler;
88extern timer_expiration_handler tcp_timer_retransmit_handler;
Florin Coras3e350af2017-03-30 02:54:28 -070089extern timer_expiration_handler tcp_timer_persist_handler;
Dave Barach68b0fb02017-02-28 15:15:56 -050090extern timer_expiration_handler tcp_timer_retransmit_syn_handler;
91
92#define TCP_TIMER_HANDLE_INVALID ((u32) ~0)
93
Florin Coras9094b5c2019-08-12 14:17:47 -070094#define TCP_TIMER_TICK 0.1 /**< Timer tick in seconds */
95#define TCP_TO_TIMER_TICK TCP_TICK*10 /**< Factor for converting
96 ticks to timer ticks */
Dave Barach68b0fb02017-02-28 15:15:56 -050097
98#define TCP_RTO_MAX 60 * THZ /* Min max RTO (60s) as per RFC6298 */
Florin Corasf03a59a2017-06-09 21:07:32 -070099#define TCP_RTO_MIN 0.2 * THZ /* Min RTO (200ms) - lower than standard */
Dave Barach68b0fb02017-02-28 15:15:56 -0500100#define TCP_RTT_MAX 30 * THZ /* 30s (probably too much) */
101#define TCP_RTO_SYN_RETRIES 3 /* SYN retries without doubling RTO */
102#define TCP_RTO_INIT 1 * THZ /* Initial retransmit timer */
Florin Coras548f7572019-06-07 15:31:06 -0700103#define TCP_RTO_BOFF_MAX 8 /* Max number of retries before reset */
Florin Coras75c48c12019-08-02 15:17:21 -0700104#define TCP_ESTABLISH_TIME (60 * THZ) /* Connection establish timeout */
Dave Barach68b0fb02017-02-28 15:15:56 -0500105
Florin Corasbbcfaac2019-10-10 13:52:04 -0700106/** Connection configuration flags */
107#define foreach_tcp_cfg_flag \
108 _(RATE_SAMPLE, "Rate sampling") \
109 _(NO_CSUM_OFFLOAD, "No csum offload") \
110 _(NO_TSO, "TSO off") \
111 _(TSO, "TSO") \
112
113typedef enum tcp_cfg_flag_bits_
114{
115#define _(sym, str) TCP_CFG_F_##sym##_BIT,
116 foreach_tcp_cfg_flag
117#undef _
118 TCP_CFG_N_FLAG_BITS
119} tcp_cfg_flag_bits_e;
120
121typedef enum tcp_cfg_flag_
122{
123#define _(sym, str) TCP_CFG_F_##sym = 1 << TCP_CFG_F_##sym##_BIT,
124 foreach_tcp_cfg_flag
125#undef _
126 TCP_CFG_N_FLAGS
127} tcp_cfg_flags_e;
128
Dave Barach68b0fb02017-02-28 15:15:56 -0500129/** TCP connection flags */
130#define foreach_tcp_connection_flag \
Dave Barach68b0fb02017-02-28 15:15:56 -0500131 _(SNDACK, "Send ACK") \
Florin Corasd79b41e2017-03-04 05:37:52 -0800132 _(FINSNT, "FIN sent") \
Florin Corasa096f2d2017-09-28 23:49:42 -0400133 _(RECOVERY, "Recovery") \
134 _(FAST_RECOVERY, "Fast Recovery") \
Florin Corasb11175d2018-11-09 14:34:08 -0800135 _(DCNT_PENDING, "Disconnect pending") \
Florin Corasb2215d62017-08-01 16:56:58 -0700136 _(HALF_OPEN_DONE, "Half-open completed") \
Florin Corasca1c8f32018-05-23 21:01:30 -0700137 _(FINPNDG, "FIN pending") \
Florin Coras36ebcff2019-09-12 18:36:44 -0700138 _(RXT_PENDING, "Retransmit pending") \
Florin Corasbbcfaac2019-10-10 13:52:04 -0700139 _(FRXT_FIRST, "Retransmit first") \
140 _(DEQ_PENDING, "Dequeue pending ") \
Florin Corase96bf632018-12-18 22:44:27 -0800141 _(PSH_PENDING, "PSH pending") \
Florin Corasf65074e2019-03-31 17:17:11 -0700142 _(FINRCVD, "FIN received") \
Florin Coras52814732019-06-12 15:38:19 -0700143 _(TRACK_BURST, "Track burst") \
Vladimir Kropylev398afbd2019-06-25 00:06:52 +0300144 _(ZERO_RWND_SENT, "Zero RWND sent") \
Dave Barach68b0fb02017-02-28 15:15:56 -0500145
Florin Corasbbcfaac2019-10-10 13:52:04 -0700146typedef enum tcp_connection_flag_bits_
Dave Barach68b0fb02017-02-28 15:15:56 -0500147{
148#define _(sym, str) TCP_CONN_##sym##_BIT,
149 foreach_tcp_connection_flag
150#undef _
151 TCP_CONN_N_FLAG_BITS
152} tcp_connection_flag_bits_e;
153
Florin Corasbbcfaac2019-10-10 13:52:04 -0700154typedef enum tcp_connection_flag_
Dave Barach68b0fb02017-02-28 15:15:56 -0500155{
156#define _(sym, str) TCP_CONN_##sym = 1 << TCP_CONN_##sym##_BIT,
157 foreach_tcp_connection_flag
158#undef _
159 TCP_CONN_N_FLAGS
160} tcp_connection_flags_e;
161
Florin Coras3eb50622017-07-13 01:24:57 -0400162#define TCP_SCOREBOARD_TRACE (0)
Florin Corase55a6d72018-10-31 23:09:22 -0700163#define TCP_MAX_SACK_BLOCKS 256 /**< Max number of SACK blocks stored */
Dave Barach68b0fb02017-02-28 15:15:56 -0500164#define TCP_INVALID_SACK_HOLE_INDEX ((u32)~0)
165
Florin Coras3eb50622017-07-13 01:24:57 -0400166typedef struct _scoreboard_trace_elt
167{
168 u32 start;
169 u32 end;
170 u32 ack;
171 u32 snd_una_max;
172 u32 group;
173} scoreboard_trace_elt_t;
174
Dave Barach68b0fb02017-02-28 15:15:56 -0500175typedef struct _sack_scoreboard_hole
176{
177 u32 next; /**< Index for next entry in linked list */
178 u32 prev; /**< Index for previous entry in linked list */
179 u32 start; /**< Start sequence number */
180 u32 end; /**< End sequence number */
Florin Coras93992a92017-05-24 18:03:56 -0700181 u8 is_lost; /**< Mark hole as lost */
Dave Barach68b0fb02017-02-28 15:15:56 -0500182} sack_scoreboard_hole_t;
183
184typedef struct _sack_scoreboard
185{
186 sack_scoreboard_hole_t *holes; /**< Pool of holes */
Florin Coras6792ec02017-03-13 03:49:51 -0700187 u32 head; /**< Index of first entry */
188 u32 tail; /**< Index of last entry */
Dave Barach68b0fb02017-02-28 15:15:56 -0500189 u32 sacked_bytes; /**< Number of bytes sacked in sb */
Florin Coras6792ec02017-03-13 03:49:51 -0700190 u32 last_sacked_bytes; /**< Number of bytes last sacked */
Florin Coras52814732019-06-12 15:38:19 -0700191 u32 last_bytes_delivered; /**< Sack bytes delivered to app */
Florin Corasbe237bf2019-09-27 08:16:40 -0700192 u32 rxt_sacked; /**< Rxt bytes last delivered */
Florin Coras93992a92017-05-24 18:03:56 -0700193 u32 high_sacked; /**< Highest byte sacked (fack) */
194 u32 high_rxt; /**< Highest retransmitted sequence */
195 u32 rescue_rxt; /**< Rescue sequence number */
196 u32 lost_bytes; /**< Bytes lost as per RFC6675 */
Florin Corasb3f58e12019-07-05 18:31:54 -0700197 u32 last_lost_bytes; /**< Number of bytes last lost */
Florin Coras93992a92017-05-24 18:03:56 -0700198 u32 cur_rxt_hole; /**< Retransmitting from this hole */
Florin Coras558e3e02019-09-06 12:56:58 -0700199 u8 is_reneging;
Florin Coras3eb50622017-07-13 01:24:57 -0400200
201#if TCP_SCOREBOARD_TRACE
202 scoreboard_trace_elt_t *trace;
203#endif
204
Dave Barach68b0fb02017-02-28 15:15:56 -0500205} sack_scoreboard_t;
206
Florin Coras3eb50622017-07-13 01:24:57 -0400207#if TCP_SCOREBOARD_TRACE
208#define tcp_scoreboard_trace_add(_tc, _ack) \
209{ \
210 static u64 _group = 0; \
211 sack_scoreboard_t *_sb = &_tc->sack_sb; \
212 sack_block_t *_sack, *_sacks; \
213 scoreboard_trace_elt_t *_elt; \
214 int i; \
215 _group++; \
216 _sacks = _tc->rcv_opts.sacks; \
217 for (i = 0; i < vec_len (_sacks); i++) \
218 { \
219 _sack = &_sacks[i]; \
220 vec_add2 (_sb->trace, _elt, 1); \
221 _elt->start = _sack->start; \
222 _elt->end = _sack->end; \
223 _elt->ack = _elt->end == _ack ? _ack : 0; \
224 _elt->snd_una_max = _elt->end == _ack ? _tc->snd_una_max : 0; \
225 _elt->group = _group; \
226 } \
227}
228#else
229#define tcp_scoreboard_trace_add(_tc, _ack)
230#endif
231
Florin Coras0dbd5172018-06-25 16:19:34 -0700232sack_scoreboard_hole_t *scoreboard_next_rxt_hole (sack_scoreboard_t * sb,
233 sack_scoreboard_hole_t *
234 start, u8 have_sent_1_smss,
235 u8 * can_rescue,
236 u8 * snd_limited);
237sack_scoreboard_hole_t *scoreboard_get_hole (sack_scoreboard_t * sb,
238 u32 index);
239
240sack_scoreboard_hole_t *scoreboard_next_hole (sack_scoreboard_t * sb,
241 sack_scoreboard_hole_t * hole);
242sack_scoreboard_hole_t *scoreboard_prev_hole (sack_scoreboard_t * sb,
243 sack_scoreboard_hole_t * hole);
244sack_scoreboard_hole_t *scoreboard_first_hole (sack_scoreboard_t * sb);
245sack_scoreboard_hole_t *scoreboard_last_hole (sack_scoreboard_t * sb);
Florin Coras36ebcff2019-09-12 18:36:44 -0700246
Florin Coras0dbd5172018-06-25 16:19:34 -0700247void scoreboard_clear (sack_scoreboard_t * sb);
Florin Coras36ebcff2019-09-12 18:36:44 -0700248void scoreboard_clear_reneging (sack_scoreboard_t * sb, u32 start, u32 end);
Florin Coras0dbd5172018-06-25 16:19:34 -0700249void scoreboard_init (sack_scoreboard_t * sb);
Florin Corasbe237bf2019-09-27 08:16:40 -0700250void scoreboard_init_rxt (sack_scoreboard_t * sb, u32 snd_una);
Florin Coras0dbd5172018-06-25 16:19:34 -0700251u8 *format_tcp_scoreboard (u8 * s, va_list * args);
252
Florin Coras52814732019-06-12 15:38:19 -0700253#define TCP_BTS_INVALID_INDEX ((u32)~0)
254
255typedef enum tcp_bts_flags_
256{
257 TCP_BTS_IS_RXT = 1,
258 TCP_BTS_IS_APP_LIMITED = 1 << 1,
259} __clib_packed tcp_bts_flags_t;
260
261typedef struct tcp_bt_sample_
262{
263 u32 next; /**< Next sample index in list */
264 u32 prev; /**< Previous sample index in list */
265 u32 min_seq; /**< Min seq number in sample */
266 u32 max_seq; /**< Max seq number. Set for rxt samples */
Florin Coras85fc1302019-06-26 09:12:34 -0700267 u64 delivered; /**< Total delivered bytes for sample */
Florin Coras52814732019-06-12 15:38:19 -0700268 f64 delivered_time; /**< Delivered time when sample taken */
Florin Coras85fc1302019-06-26 09:12:34 -0700269 f64 tx_time; /**< Transmit time for the burst */
Florin Coras7436b432019-09-10 23:26:27 -0700270 f64 first_tx_time; /**< Connection first tx time at tx */
Florin Coras52814732019-06-12 15:38:19 -0700271 tcp_bts_flags_t flags; /**< Sample flag */
272} tcp_bt_sample_t;
273
274typedef struct tcp_rate_sample_
275{
Florin Coras85fc1302019-06-26 09:12:34 -0700276 u64 prior_delivered; /**< Delivered of sample used for rate, i.e.,
277 total bytes delivered at prior_time */
278 f64 prior_time; /**< Delivered time of sample used for rate */
279 f64 interval_time; /**< Time to ack the bytes delivered */
280 f64 rtt_time; /**< RTT for sample */
Florin Coras85fc1302019-06-26 09:12:34 -0700281 u32 delivered; /**< Bytes delivered in interval_time */
282 u32 acked_and_sacked; /**< Bytes acked + sacked now */
283 u32 lost; /**< Bytes lost now */
Florin Coras52814732019-06-12 15:38:19 -0700284 tcp_bts_flags_t flags; /**< Rate sample flags from bt sample */
285} tcp_rate_sample_t;
286
287typedef struct tcp_byte_tracker_
288{
289 tcp_bt_sample_t *samples; /**< Pool of samples */
290 rb_tree_t sample_lookup; /**< Rbtree for sample lookup by min_seq */
291 u32 head; /**< Head of samples linked list */
292 u32 tail; /**< Tail of samples linked list */
293 u32 last_ooo; /**< Cached last ooo sample */
294} tcp_byte_tracker_t;
295
Dave Barach68b0fb02017-02-28 15:15:56 -0500296typedef enum _tcp_cc_algorithm_type
297{
298 TCP_CC_NEWRENO,
Florin Coras2e31cc32018-09-25 14:00:34 -0700299 TCP_CC_CUBIC,
Florin Coras4e116fb2019-06-10 08:33:50 -0700300 TCP_CC_LAST = TCP_CC_CUBIC
Dave Barach68b0fb02017-02-28 15:15:56 -0500301} tcp_cc_algorithm_type_e;
302
303typedef struct _tcp_cc_algorithm tcp_cc_algorithm_t;
304
305typedef enum _tcp_cc_ack_t
306{
307 TCP_CC_ACK,
308 TCP_CC_DUPACK,
309 TCP_CC_PARTIALACK
310} tcp_cc_ack_t;
311
Florin Corasca2831a2019-07-04 17:05:59 -0700312typedef enum tcp_cc_event_
313{
314 TCP_CC_EVT_START_TX,
315} tcp_cc_event_t;
316
Florin Corasedfe0ee2019-07-29 18:13:25 -0700317/*
318 * As per RFC4898 tcpEStatsStackSoftErrors
319 */
320typedef struct tcp_errors_
321{
322 u32 below_data_wnd; /**< All data in seg is below snd_una */
323 u32 above_data_wnd; /**< Some data in segment is above snd_wnd */
324 u32 below_ack_wnd; /**< Acks for data below snd_una */
325 u32 above_ack_wnd; /**< Acks for data not sent */
326} tcp_errors_t;
327
Dave Barach68b0fb02017-02-28 15:15:56 -0500328typedef struct _tcp_connection
329{
Florin Coras2e31cc32018-09-25 14:00:34 -0700330 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
Dave Barach68b0fb02017-02-28 15:15:56 -0500331 transport_connection_t connection; /**< Common transport data. First! */
332
333 u8 state; /**< TCP state as per tcp_state_t */
Florin Corasbbcfaac2019-10-10 13:52:04 -0700334 u8 cfg_flags; /**< Connection configuration flags */
Dave Barach68b0fb02017-02-28 15:15:56 -0500335 u16 flags; /**< Connection flags (see tcp_conn_flags_e) */
336 u32 timers[TCP_N_TIMERS]; /**< Timer handles into timer wheel */
337
Florin Corasedfe0ee2019-07-29 18:13:25 -0700338 u64 segs_in; /** RFC4022/4898 tcpHCInSegs/tcpEStatsPerfSegsIn */
339 u64 bytes_in; /** RFC4898 tcpEStatsPerfHCDataOctetsIn */
340 u64 segs_out; /** RFC4898 tcpEStatsPerfSegsOut */
341 u64 bytes_out; /** RFC4898 tcpEStatsPerfHCDataOctetsOut */
Dave Barach68b0fb02017-02-28 15:15:56 -0500342
343 /** Send sequence variables RFC793 */
344 u32 snd_una; /**< oldest unacknowledged sequence number */
345 u32 snd_una_max; /**< newest unacknowledged sequence number + 1*/
346 u32 snd_wnd; /**< send window */
347 u32 snd_wl1; /**< seq number used for last snd.wnd update */
348 u32 snd_wl2; /**< ack number used for last snd.wnd update */
349 u32 snd_nxt; /**< next seq number to be sent */
Florin Corasf03a59a2017-06-09 21:07:32 -0700350 u16 snd_mss; /**< Effective send max seg (data) size */
Dave Barach68b0fb02017-02-28 15:15:56 -0500351
Florin Corasedfe0ee2019-07-29 18:13:25 -0700352 u64 data_segs_in; /** RFC4898 tcpEStatsPerfDataSegsIn */
353 u64 data_segs_out; /** RFC4898 tcpEStatsPerfDataSegsOut */
354
Dave Barach68b0fb02017-02-28 15:15:56 -0500355 /** Receive sequence variables RFC793 */
356 u32 rcv_nxt; /**< next sequence number expected */
357 u32 rcv_wnd; /**< receive window we expect */
358
359 u32 rcv_las; /**< rcv_nxt at last ack sent/rcv_wnd update */
360 u32 iss; /**< initial sent sequence */
361 u32 irs; /**< initial remote sequence */
362
363 /* Options */
Florin Corasc8343412017-05-04 14:25:50 -0700364 u8 snd_opts_len; /**< Tx options len */
Florin Coras0dbd5172018-06-25 16:19:34 -0700365 u8 rcv_wscale; /**< Window scale to advertise to peer */
366 u8 snd_wscale; /**< Window scale to use when sending */
367 u32 tsval_recent; /**< Last timestamp received */
368 u32 tsval_recent_age; /**< When last updated tstamp_recent*/
369 tcp_options_t snd_opts; /**< Tx options for connection */
370 tcp_options_t rcv_opts; /**< Rx options for connection */
Dave Barach68b0fb02017-02-28 15:15:56 -0500371
372 sack_block_t *snd_sacks; /**< Vector of SACKs to send. XXX Fixed size? */
Florin Corase5b17912019-02-21 16:46:24 -0800373 u8 snd_sack_pos; /**< Position in vec of first block to send */
Florin Corasb691f762019-02-22 09:07:20 -0800374 sack_block_t *snd_sacks_fl; /**< Vector for building new list */
Dave Barach68b0fb02017-02-28 15:15:56 -0500375 sack_scoreboard_t sack_sb; /**< SACK "scoreboard" that tracks holes */
376
Florin Corasedfe0ee2019-07-29 18:13:25 -0700377 u16 rcv_dupacks; /**< Number of recent DUPACKs received */
378 u32 dupacks_in; /**< RFC4898 tcpEStatsStackDupAcksIn*/
Florin Coras7ac053b2018-11-05 15:57:21 -0800379 u8 pending_dupacks; /**< Number of DUPACKs to be sent */
Florin Corasedfe0ee2019-07-29 18:13:25 -0700380 u32 dupacks_out; /**< RFC4898 tcpEStatsPathDupAcksOut */
Dave Barach68b0fb02017-02-28 15:15:56 -0500381
382 /* Congestion control */
383 u32 cwnd; /**< Congestion window */
Florin Coras62166002018-04-18 16:40:55 -0700384 u32 cwnd_acc_bytes; /**< Bytes accumulated for cwnd increment */
Dave Barach68b0fb02017-02-28 15:15:56 -0500385 u32 ssthresh; /**< Slow-start threshold */
386 u32 prev_ssthresh; /**< ssthresh before congestion */
Florin Coras93992a92017-05-24 18:03:56 -0700387 u32 prev_cwnd; /**< ssthresh before congestion */
Dave Barach68b0fb02017-02-28 15:15:56 -0500388 u32 bytes_acked; /**< Bytes acknowledged by current segment */
Florin Coras9ece3c02018-11-05 11:06:53 -0800389 u32 burst_acked; /**< Bytes acknowledged in current burst */
Florin Coras36ebcff2019-09-12 18:36:44 -0700390 u32 snd_rxt_bytes; /**< Retransmitted bytes during current cc event */
Florin Coras93992a92017-05-24 18:03:56 -0700391 u32 snd_rxt_ts; /**< Timestamp when first packet is retransmitted */
Florin Coras36ebcff2019-09-12 18:36:44 -0700392 u32 prr_delivered; /**< RFC6937 bytes delivered during current event */
Florin Corasbe237bf2019-09-27 08:16:40 -0700393 u32 prr_start; /**< snd_una when prr starts */
Florin Coras36ebcff2019-09-12 18:36:44 -0700394 u32 rxt_delivered; /**< Rxt bytes delivered during current cc event */
Florin Corasbe237bf2019-09-27 08:16:40 -0700395 u32 rxt_head; /**< snd_una last time we re rxted the head */
Florin Corasc31dc312019-10-06 14:06:14 -0700396 u32 prev_dsegs_out; /**< Number of dsegs after last ack */
Florin Corase69f4952017-03-07 10:06:24 -0800397 u32 tsecr_last_ack; /**< Timestamp echoed to us in last healthy ACK */
Florin Coras6792ec02017-03-13 03:49:51 -0700398 u32 snd_congestion; /**< snd_una_max when congestion is detected */
Florin Coras52814732019-06-12 15:38:19 -0700399 u32 tx_fifo_size; /**< Tx fifo size. Used to constrain cwnd */
Dave Barach68b0fb02017-02-28 15:15:56 -0500400 tcp_cc_algorithm_t *cc_algo; /**< Congestion control algorithm */
Florin Coras2e31cc32018-09-25 14:00:34 -0700401 u8 cc_data[TCP_CC_DATA_SZ]; /**< Congestion control algo private data */
Dave Barach68b0fb02017-02-28 15:15:56 -0500402
Florin Corasedfe0ee2019-07-29 18:13:25 -0700403 u32 fr_occurences; /**< fast-retransmit occurrences RFC4898
404 tcpEStatsStackFastRetran */
405 u32 tr_occurences; /**< timer-retransmit occurrences */
406 u64 bytes_retrans; /**< RFC4898 tcpEStatsPerfOctetsRetrans */
407 u64 segs_retrans; /**< RFC4898 tcpEStatsPerfSegsRetrans*/
408
Dave Barach68b0fb02017-02-28 15:15:56 -0500409 /* RTT and RTO */
410 u32 rto; /**< Retransmission timeout */
411 u32 rto_boff; /**< Index for RTO backoff */
412 u32 srtt; /**< Smoothed RTT */
413 u32 rttvar; /**< Smoothed mean RTT difference. Approximates variance */
Dave Barach68b0fb02017-02-28 15:15:56 -0500414 u32 rtt_seq; /**< Sequence number for tracked ACK */
Florin Corasd67f1122018-05-21 17:47:40 -0700415 f64 rtt_ts; /**< Timestamp for tracked ACK */
416 f64 mrtt_us; /**< High precision mrtt from tracked acks */
Dave Barach68b0fb02017-02-28 15:15:56 -0500417
Florin Coras42ceddb2018-12-12 10:56:01 -0800418 u32 psh_seq; /**< Add psh header for seg that includes this */
Florin Coras07beade2019-06-20 12:18:31 -0700419 u32 next_node_index; /**< Can be used to control next node in output */
420 u32 next_node_opaque; /**< Opaque to pass to next node */
Florin Coras52814732019-06-12 15:38:19 -0700421 u32 limited_transmit; /**< snd_nxt when limited transmit starts */
422 u32 sw_if_index; /**< Interface for the connection */
423
424 /* Delivery rate estimation */
425 u64 delivered; /**< Total bytes delivered to peer */
426 u64 app_limited; /**< Delivered when app-limited detected */
427 f64 delivered_time; /**< Time last bytes were acked */
Florin Coras7436b432019-09-10 23:26:27 -0700428 f64 first_tx_time; /**< Send time for recently delivered/sent */
Florin Coras52814732019-06-12 15:38:19 -0700429 tcp_byte_tracker_t *bt; /**< Tx byte tracker */
430
Florin Corasedfe0ee2019-07-29 18:13:25 -0700431 tcp_errors_t errors; /**< Soft connection errors */
432
433 f64 start_ts; /**< Timestamp when connection initialized */
Florin Coras52814732019-06-12 15:38:19 -0700434 u32 last_fib_check; /**< Last time we checked fib route for peer */
435 u16 mss; /**< Our max seg size that includes options */
Florin Corasedfe0ee2019-07-29 18:13:25 -0700436 u32 timestamp_delta; /**< Offset for timestamp */
Dave Barach68b0fb02017-02-28 15:15:56 -0500437} tcp_connection_t;
438
Florin Corasaa01abb2018-11-12 09:13:10 -0800439/* *INDENT-OFF* */
Dave Barach68b0fb02017-02-28 15:15:56 -0500440struct _tcp_cc_algorithm
441{
Florin Corasaa01abb2018-11-12 09:13:10 -0800442 const char *name;
443 uword (*unformat_cfg) (unformat_input_t * input);
Florin Corasa3c32652019-07-03 17:47:22 -0700444 void (*init) (tcp_connection_t * tc);
445 void (*cleanup) (tcp_connection_t * tc);
Florin Coras52814732019-06-12 15:38:19 -0700446 void (*rcv_ack) (tcp_connection_t * tc, tcp_rate_sample_t *rs);
447 void (*rcv_cong_ack) (tcp_connection_t * tc, tcp_cc_ack_t ack,
448 tcp_rate_sample_t *rs);
Dave Barach68b0fb02017-02-28 15:15:56 -0500449 void (*congestion) (tcp_connection_t * tc);
Florin Corasa3c32652019-07-03 17:47:22 -0700450 void (*loss) (tcp_connection_t * tc);
Dave Barach68b0fb02017-02-28 15:15:56 -0500451 void (*recovered) (tcp_connection_t * tc);
Florin Coraseff6b822019-07-03 19:51:02 -0700452 void (*undo_recovery) (tcp_connection_t * tc);
Florin Corasca2831a2019-07-04 17:05:59 -0700453 void (*event) (tcp_connection_t *tc, tcp_cc_event_t evt);
Florin Corasd2067242019-08-16 10:33:49 -0700454 u64 (*get_pacing_rate) (tcp_connection_t *tc);
Dave Barach68b0fb02017-02-28 15:15:56 -0500455};
Florin Corasaa01abb2018-11-12 09:13:10 -0800456/* *INDENT-ON* */
Dave Barach68b0fb02017-02-28 15:15:56 -0500457
458#define tcp_fastrecovery_on(tc) (tc)->flags |= TCP_CONN_FAST_RECOVERY
459#define tcp_fastrecovery_off(tc) (tc)->flags &= ~TCP_CONN_FAST_RECOVERY
Florin Coras3e350af2017-03-30 02:54:28 -0700460#define tcp_recovery_on(tc) (tc)->flags |= TCP_CONN_RECOVERY
461#define tcp_recovery_off(tc) (tc)->flags &= ~TCP_CONN_RECOVERY
Dave Barach68b0fb02017-02-28 15:15:56 -0500462#define tcp_in_fastrecovery(tc) ((tc)->flags & TCP_CONN_FAST_RECOVERY)
Florin Coras3e350af2017-03-30 02:54:28 -0700463#define tcp_in_recovery(tc) ((tc)->flags & (TCP_CONN_RECOVERY))
Dave Barach68b0fb02017-02-28 15:15:56 -0500464#define tcp_in_slowstart(tc) (tc->cwnd < tc->ssthresh)
Florin Corasb11175d2018-11-09 14:34:08 -0800465#define tcp_disconnect_pending(tc) ((tc)->flags & TCP_CONN_DCNT_PENDING)
466#define tcp_disconnect_pending_on(tc) ((tc)->flags |= TCP_CONN_DCNT_PENDING)
467#define tcp_disconnect_pending_off(tc) ((tc)->flags &= ~TCP_CONN_DCNT_PENDING)
Florin Corasbf4d5ce2018-10-19 16:26:24 -0700468#define tcp_fastrecovery_first(tc) ((tc)->flags & TCP_CONN_FRXT_FIRST)
469#define tcp_fastrecovery_first_on(tc) ((tc)->flags |= TCP_CONN_FRXT_FIRST)
470#define tcp_fastrecovery_first_off(tc) ((tc)->flags &= ~TCP_CONN_FRXT_FIRST)
Dave Barach68b0fb02017-02-28 15:15:56 -0500471
Florin Coras3e350af2017-03-30 02:54:28 -0700472#define tcp_in_cong_recovery(tc) ((tc)->flags & \
473 (TCP_CONN_FAST_RECOVERY | TCP_CONN_RECOVERY))
474
Florin Corasbbcfaac2019-10-10 13:52:04 -0700475#define tcp_csum_offload(tc) (!((tc)->cfg_flags & TCP_CFG_F_NO_CSUM_OFFLOAD))
476
Florin Coras3e350af2017-03-30 02:54:28 -0700477always_inline void
478tcp_cong_recovery_off (tcp_connection_t * tc)
479{
480 tc->flags &= ~(TCP_CONN_FAST_RECOVERY | TCP_CONN_RECOVERY);
Florin Corasbf4d5ce2018-10-19 16:26:24 -0700481 tcp_fastrecovery_first_off (tc);
Florin Coras3e350af2017-03-30 02:54:28 -0700482}
483
Florin Corasd1ba3d22019-08-08 09:44:47 -0700484#define tcp_zero_rwnd_sent(tc) ((tc)->flags & TCP_CONN_ZERO_RWND_SENT)
Vladimir Kropylev398afbd2019-06-25 00:06:52 +0300485#define tcp_zero_rwnd_sent_on(tc) (tc)->flags |= TCP_CONN_ZERO_RWND_SENT
486#define tcp_zero_rwnd_sent_off(tc) (tc)->flags &= ~TCP_CONN_ZERO_RWND_SENT
487
Dave Barach68b0fb02017-02-28 15:15:56 -0500488typedef enum _tcp_error
489{
490#define tcp_error(n,s) TCP_ERROR_##n,
491#include <vnet/tcp/tcp_error.def>
492#undef tcp_error
493 TCP_N_ERROR,
494} tcp_error_t;
495
496typedef struct _tcp_lookup_dispatch
497{
498 u8 next, error;
499} tcp_lookup_dispatch_t;
500
Florin Coras2c414432018-06-19 09:58:04 -0700501typedef struct tcp_worker_ctx_
502{
503 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
Florin Corasb11175d2018-11-09 14:34:08 -0800504 /** worker time */
505 u32 time_now;
506
507 /** worker timer wheel */
508 tw_timer_wheel_16t_2w_512sl_t timer_wheel;
509
510 /** tx buffer free list */
511 u32 *tx_buffers;
512
513 /** tx frames for tcp 4/6 output nodes */
514 vlib_frame_t *tx_frames[2];
515
516 /** tx frames for ip 4/6 lookup nodes */
517 vlib_frame_t *ip_lookup_tx_frames[2];
518
Florin Corasb11175d2018-11-09 14:34:08 -0800519 /** vector of pending ack dequeues */
Florin Coras9ece3c02018-11-05 11:06:53 -0800520 u32 *pending_deq_acked;
Florin Corasb11175d2018-11-09 14:34:08 -0800521
Florin Corasb11175d2018-11-09 14:34:08 -0800522 /** vector of pending disconnect notifications */
523 u32 *pending_disconnects;
524
525 /** convenience pointer to this thread's vlib main */
526 vlib_main_t *vm;
Florin Corasbf4d5ce2018-10-19 16:26:24 -0700527
Florin Corasb26743d2018-06-26 09:31:04 -0700528 CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
Florin Corasb11175d2018-11-09 14:34:08 -0800529
530 /** cached 'on the wire' options for bursts */
531 u8 cached_opts[40];
Florin Corasbf4d5ce2018-10-19 16:26:24 -0700532
Florin Coras2c414432018-06-19 09:58:04 -0700533} tcp_worker_ctx_t;
534
Florin Coras18e0d4f2019-01-02 12:22:02 -0800535typedef struct tcp_iss_seed_
536{
537 u64 first;
538 u64 second;
539} tcp_iss_seed_t;
540
Florin Coras9094b5c2019-08-12 14:17:47 -0700541typedef struct tcp_configuration_
542{
543 /** Max rx fifo size for a session (in bytes). It is used in to compute the
544 * rfc 7323 window scaling factor */
545 u32 max_rx_fifo;
546
547 /** Min rx fifo for a session (in bytes) */
548 u32 min_rx_fifo;
549
550 /** Default MTU to be used when establishing connections */
551 u16 default_mtu;
552
553 /** Initial CWND multiplier, which multiplies MSS to determine initial CWND.
554 * Set 0 to determine the initial CWND by another way */
555 u16 initial_cwnd_multiplier;
556
557 /** Enable tx pacing for new connections */
558 u8 enable_tx_pacing;
559
Florin Corasbbcfaac2019-10-10 13:52:04 -0700560 /** Allow use of TSO whenever available */
561 u8 allow_tso;
562
Florin Coras9094b5c2019-08-12 14:17:47 -0700563 /** Default congestion control algorithm type */
564 tcp_cc_algorithm_type_e cc_algo;
565
Florin Coras017dc452019-08-30 11:06:35 -0700566 /** Min rwnd, as number of snd_mss segments, for update ack to be sent after
567 * a zero rwnd advertisement */
568 u32 rwnd_min_update_ack;
569
Florin Coras9094b5c2019-08-12 14:17:47 -0700570 /** Delayed ack time (disabled) */
571 u16 delack_time;
572
573 /** Timer ticks to wait for close from app */
574 u16 closewait_time;
575
576 /** Timer ticks to wait in time-wait. Also known as 2MSL */
577 u16 timewait_time;
578
579 /** Timer ticks to wait in fin-wait1 to send fin and rcv fin-ack */
580 u16 finwait1_time;
581
582 /** Timer ticks to wait in last ack for ack */
583 u16 lastack_time;
584
585 /** Timer ticks to wait in fin-wait2 for fin */
586 u16 finwait2_time;
587
588 /** Timer ticks to wait in closing for fin ack */
589 u16 closing_time;
590
591 /** Timer ticks to wait before cleaning up the connection */
592 u16 cleanup_time;
593
594 /** Number of preallocated connections */
595 u32 preallocated_connections;
596
597 /** Number of preallocated half-open connections */
598 u32 preallocated_half_open_connections;
599
600 /** Vectors of src addresses. Optional unless one needs > 63K active-opens */
601 ip4_address_t *ip4_src_addrs;
602 ip6_address_t *ip6_src_addrs;
603
604 /** Fault-injection. Debug only */
605 f64 buffer_fail_fraction;
606} tcp_configuration_t;
607
Dave Barach68b0fb02017-02-28 15:15:56 -0500608typedef struct _tcp_main
609{
610 /* Per-worker thread tcp connection pools */
611 tcp_connection_t **connections;
612
613 /* Pool of listeners. */
614 tcp_connection_t *listener_pool;
615
616 /** Dispatch table by state and flags */
617 tcp_lookup_dispatch_t dispatch_table[TCP_N_STATES][64];
618
619 u8 log2_tstamp_clocks_per_tick;
620 f64 tstamp_ticks_per_clock;
621
Florin Coras2c414432018-06-19 09:58:04 -0700622 /** per-worker context */
623 tcp_worker_ctx_t *wrk_ctx;
Dave Barach68b0fb02017-02-28 15:15:56 -0500624
Florin Coras9094b5c2019-08-12 14:17:47 -0700625 /** Pool of half-open connections on which we've sent a SYN */
Dave Barach68b0fb02017-02-28 15:15:56 -0500626 tcp_connection_t *half_open_connections;
Florin Coras6534b7a2017-07-18 05:38:03 -0400627 clib_spinlock_t half_open_lock;
Dave Barach68b0fb02017-02-28 15:15:56 -0500628
Florin Corasca031862018-09-24 13:58:05 -0700629 /** vlib buffer size */
630 u32 bytes_per_buffer;
631
Florin Coras9094b5c2019-08-12 14:17:47 -0700632 /** Seed used to generate random iss */
Florin Coras18e0d4f2019-01-02 12:22:02 -0800633 tcp_iss_seed_t iss_seed;
634
Florin Coras9094b5c2019-08-12 14:17:47 -0700635 /** Congestion control algorithms registered */
Florin Coras4e116fb2019-06-10 08:33:50 -0700636 tcp_cc_algorithm_t *cc_algos;
637
Florin Corasfbf278a2019-03-26 14:05:38 -0700638 /** Hash table of cc algorithms by name */
639 uword *cc_algo_by_name;
640
Florin Coras4e116fb2019-06-10 08:33:50 -0700641 /** Last cc algo registered */
642 tcp_cc_algorithm_type_e cc_last_type;
643
Florin Coras9094b5c2019-08-12 14:17:47 -0700644 /** Flag that indicates if stack is on or off */
Florin Corasa0b34a72017-03-07 01:20:52 -0800645 u8 is_enabled;
646
Florin Coras9094b5c2019-08-12 14:17:47 -0700647 /** Flag that indicates if v4 punting is enabled */
Pierre Pfister7fe51f32017-09-20 08:48:36 +0200648 u8 punt_unknown4;
Florin Coras9094b5c2019-08-12 14:17:47 -0700649
650 /** Flag that indicates if v6 punting is enabled */
Pierre Pfister7fe51f32017-09-20 08:48:36 +0200651 u8 punt_unknown6;
Florin Corasf988e692017-11-27 04:34:14 -0500652
Florin Coras9094b5c2019-08-12 14:17:47 -0700653 /** Rotor for v4 source addresses */
654 u32 last_v4_addr_rotor;
Florin Coras2e31cc32018-09-25 14:00:34 -0700655
Florin Coras9094b5c2019-08-12 14:17:47 -0700656 /** Rotor for v6 source addresses */
657 u32 last_v6_addr_rotor;
658
659 /** Protocol configuration */
660 tcp_configuration_t cfg;
Dave Barach68b0fb02017-02-28 15:15:56 -0500661} tcp_main_t;
662
663extern tcp_main_t tcp_main;
664extern vlib_node_registration_t tcp4_input_node;
665extern vlib_node_registration_t tcp6_input_node;
666extern vlib_node_registration_t tcp4_output_node;
667extern vlib_node_registration_t tcp6_output_node;
Filip Tehlare275bed2019-03-06 00:06:56 -0800668extern vlib_node_registration_t tcp4_established_node;
669extern vlib_node_registration_t tcp6_established_node;
670extern vlib_node_registration_t tcp4_syn_sent_node;
671extern vlib_node_registration_t tcp6_syn_sent_node;
672extern vlib_node_registration_t tcp4_rcv_process_node;
673extern vlib_node_registration_t tcp6_rcv_process_node;
674extern vlib_node_registration_t tcp4_listen_node;
675extern vlib_node_registration_t tcp6_listen_node;
Dave Barach68b0fb02017-02-28 15:15:56 -0500676
Florin Coras9094b5c2019-08-12 14:17:47 -0700677#define tcp_cfg tcp_main.cfg
Florin Coras1f421012019-07-26 10:18:51 -0700678#define tcp_node_index(node_id, is_ip4) \
679 ((is_ip4) ? tcp4_##node_id##_node.index : tcp6_##node_id##_node.index)
680
Dave Barach68b0fb02017-02-28 15:15:56 -0500681always_inline tcp_main_t *
682vnet_get_tcp_main ()
683{
684 return &tcp_main;
685}
686
Florin Corasbe72ae62018-11-01 11:23:03 -0700687always_inline tcp_worker_ctx_t *
688tcp_get_worker (u32 thread_index)
689{
690 return &tcp_main.wrk_ctx[thread_index];
691}
692
Florin Coras82b13a82017-04-25 11:58:06 -0700693always_inline tcp_header_t *
694tcp_buffer_hdr (vlib_buffer_t * b)
695{
696 ASSERT ((signed) b->current_data >= (signed) -VLIB_BUFFER_PRE_DATA_SIZE);
697 return (tcp_header_t *) (b->data + b->current_data
698 + vnet_buffer (b)->tcp.hdr_offset);
699}
700
Florin Corasf988e692017-11-27 04:34:14 -0500701#if (VLIB_BUFFER_TRACE_TRAJECTORY)
702#define tcp_trajectory_add_start(b, start) \
703{ \
704 (*vlib_buffer_trace_trajectory_cb) (b, start); \
705}
706#else
707#define tcp_trajectory_add_start(b, start)
708#endif
709
Florin Corasa0b34a72017-03-07 01:20:52 -0800710clib_error_t *vnet_tcp_enable_disable (vlib_main_t * vm, u8 is_en);
711
Pierre Pfister7fe51f32017-09-20 08:48:36 +0200712void tcp_punt_unknown (vlib_main_t * vm, u8 is_ip4, u8 is_add);
713
Dave Barach68b0fb02017-02-28 15:15:56 -0500714always_inline tcp_connection_t *
715tcp_connection_get (u32 conn_index, u32 thread_index)
716{
Florin Coras68810622017-07-24 17:40:28 -0700717 if (PREDICT_FALSE
718 (pool_is_free_index (tcp_main.connections[thread_index], conn_index)))
Florin Corasd79b41e2017-03-04 05:37:52 -0800719 return 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500720 return pool_elt_at_index (tcp_main.connections[thread_index], conn_index);
721}
722
723always_inline tcp_connection_t *
724tcp_connection_get_if_valid (u32 conn_index, u32 thread_index)
725{
726 if (tcp_main.connections[thread_index] == 0)
727 return 0;
728 if (pool_is_free_index (tcp_main.connections[thread_index], conn_index))
729 return 0;
730 return pool_elt_at_index (tcp_main.connections[thread_index], conn_index);
731}
732
Florin Coras3eb50622017-07-13 01:24:57 -0400733always_inline tcp_connection_t *
734tcp_get_connection_from_transport (transport_connection_t * tconn)
735{
736 return (tcp_connection_t *) tconn;
737}
738
Florin Coras54ddf432018-12-21 13:54:09 -0800739always_inline void
740tcp_connection_set_state (tcp_connection_t * tc, tcp_state_t state)
741{
742 tc->state = state;
Florin Corasa436a422019-08-20 07:09:31 -0700743 TCP_EVT (TCP_EVT_STATE_CHANGE, tc);
Florin Coras54ddf432018-12-21 13:54:09 -0800744}
745
Dave Barach68b0fb02017-02-28 15:15:56 -0500746void tcp_connection_close (tcp_connection_t * tc);
747void tcp_connection_cleanup (tcp_connection_t * tc);
748void tcp_connection_del (tcp_connection_t * tc);
Florin Coras68810622017-07-24 17:40:28 -0700749int tcp_half_open_connection_cleanup (tcp_connection_t * tc);
Florin Coras8124cb72018-12-16 20:57:29 -0800750tcp_connection_t *tcp_connection_alloc (u8 thread_index);
Florin Coras12f69362019-08-16 09:44:00 -0700751tcp_connection_t *tcp_connection_alloc_w_base (u8 thread_index,
752 tcp_connection_t * base);
Florin Coras8124cb72018-12-16 20:57:29 -0800753void tcp_connection_free (tcp_connection_t * tc);
Florin Corasd79b41e2017-03-04 05:37:52 -0800754void tcp_connection_reset (tcp_connection_t * tc);
Dave Barach3bbcfab2017-08-15 19:03:44 -0400755int tcp_configure_v4_source_address_range (vlib_main_t * vm,
756 ip4_address_t * start,
757 ip4_address_t * end, u32 table_id);
758int tcp_configure_v6_source_address_range (vlib_main_t * vm,
759 ip6_address_t * start,
760 ip6_address_t * end, u32 table_id);
761void tcp_api_reference (void);
Florin Corase69f4952017-03-07 10:06:24 -0800762u8 *format_tcp_connection (u8 * s, va_list * args);
Florin Coras3eb50622017-07-13 01:24:57 -0400763
Dave Barach68b0fb02017-02-28 15:15:56 -0500764always_inline tcp_connection_t *
765tcp_listener_get (u32 tli)
766{
767 return pool_elt_at_index (tcp_main.listener_pool, tli);
768}
769
770always_inline tcp_connection_t *
771tcp_half_open_connection_get (u32 conn_index)
772{
Florin Coras68810622017-07-24 17:40:28 -0700773 tcp_connection_t *tc = 0;
774 clib_spinlock_lock_if_init (&tcp_main.half_open_lock);
775 if (!pool_is_free_index (tcp_main.half_open_connections, conn_index))
776 tc = pool_elt_at_index (tcp_main.half_open_connections, conn_index);
777 clib_spinlock_unlock_if_init (&tcp_main.half_open_lock);
778 return tc;
Dave Barach68b0fb02017-02-28 15:15:56 -0500779}
780
Florin Corasd79b41e2017-03-04 05:37:52 -0800781void tcp_make_fin (tcp_connection_t * tc, vlib_buffer_t * b);
Dave Barach68b0fb02017-02-28 15:15:56 -0500782void tcp_make_synack (tcp_connection_t * ts, vlib_buffer_t * b);
Florin Coras1f152cd2017-08-18 19:28:03 -0700783void tcp_send_reset_w_pkt (tcp_connection_t * tc, vlib_buffer_t * pkt,
Florin Corasd4c49be2019-02-07 00:15:53 -0800784 u32 thread_index, u8 is_ip4);
Florin Coras1f152cd2017-08-18 19:28:03 -0700785void tcp_send_reset (tcp_connection_t * tc);
Dave Barach68b0fb02017-02-28 15:15:56 -0500786void tcp_send_syn (tcp_connection_t * tc);
Florin Coras7ac053b2018-11-05 15:57:21 -0800787void tcp_send_synack (tcp_connection_t * tc);
Dave Barach68b0fb02017-02-28 15:15:56 -0500788void tcp_send_fin (tcp_connection_t * tc);
Florin Corascb711a42019-10-16 19:28:17 -0700789void tcp_send_ack (tcp_connection_t * tc);
Florin Corasb26743d2018-06-26 09:31:04 -0700790void tcp_update_burst_snd_vars (tcp_connection_t * tc);
Florin Coras93992a92017-05-24 18:03:56 -0700791void tcp_update_rto (tcp_connection_t * tc);
Florin Corasbe72ae62018-11-01 11:23:03 -0700792void tcp_flush_frame_to_output (tcp_worker_ctx_t * wrk, u8 is_ip4);
793void tcp_flush_frames_to_output (tcp_worker_ctx_t * wrk);
Vladimir Kropylev398afbd2019-06-25 00:06:52 +0300794void tcp_send_window_update_ack (tcp_connection_t * tc);
Florin Coras7ac053b2018-11-05 15:57:21 -0800795
Florin Coras26dd6de2019-07-23 23:54:47 -0700796void tcp_program_ack (tcp_connection_t * tc);
797void tcp_program_dupack (tcp_connection_t * tc);
Florin Coras36ebcff2019-09-12 18:36:44 -0700798void tcp_program_retransmit (tcp_connection_t * tc);
Florin Coras26dd6de2019-07-23 23:54:47 -0700799
Florin Coras52814732019-06-12 15:38:19 -0700800/*
801 * Rate estimation
802 */
803
804/**
805 * Byte tracker initialize
806 *
807 * @param tc connection for which the byte tracker should be allocated and
808 * initialized
809 */
810void tcp_bt_init (tcp_connection_t * tc);
811/**
812 * Byte tracker cleanup
813 *
814 * @param tc connection for which the byte tracker should be cleaned up
815 */
816void tcp_bt_cleanup (tcp_connection_t * tc);
817/**
818 * Flush byte tracker samples
819 *
820 * @param tc tcp connection for which samples should be flushed
821 */
822void tcp_bt_flush_samples (tcp_connection_t * tc);
823/**
824 * Track a tcp tx burst
825 *
826 * @param tc tcp connection
827 */
828void tcp_bt_track_tx (tcp_connection_t * tc);
829/**
830 * Track a tcp retransmission
831 *
832 * @param tc tcp connection
833 * @param start start sequence number
834 * @param end end sequence number
835 */
836void tcp_bt_track_rxt (tcp_connection_t * tc, u32 start, u32 end);
837/**
838 * Generate a delivery rate sample from recently acked bytes
839 *
840 * @param tc tcp connection
841 * @param rs resulting rate sample
842 */
843void tcp_bt_sample_delivery_rate (tcp_connection_t * tc,
844 tcp_rate_sample_t * rs);
845/**
846 * Check if sample to be generated is app limited
847 *
848 * @param tc tcp connection
849 */
850void tcp_bt_check_app_limited (tcp_connection_t * tc);
851/**
852 * Check if the byte tracker is in sane state
853 *
854 * Should be used only for testing
855 *
856 * @param bt byte tracker
857 */
858int tcp_bt_is_sane (tcp_byte_tracker_t * bt);
859
Dave Barach68b0fb02017-02-28 15:15:56 -0500860always_inline u32
861tcp_end_seq (tcp_header_t * th, u32 len)
862{
863 return th->seq_number + tcp_is_syn (th) + tcp_is_fin (th) + len;
864}
865
866/* Modulo arithmetic for TCP sequence numbers */
867#define seq_lt(_s1, _s2) ((i32)((_s1)-(_s2)) < 0)
868#define seq_leq(_s1, _s2) ((i32)((_s1)-(_s2)) <= 0)
869#define seq_gt(_s1, _s2) ((i32)((_s1)-(_s2)) > 0)
870#define seq_geq(_s1, _s2) ((i32)((_s1)-(_s2)) >= 0)
Florin Corasf03a59a2017-06-09 21:07:32 -0700871#define seq_max(_s1, _s2) (seq_gt((_s1), (_s2)) ? (_s1) : (_s2))
Dave Barach68b0fb02017-02-28 15:15:56 -0500872
873/* Modulo arithmetic for timestamps */
874#define timestamp_lt(_t1, _t2) ((i32)((_t1)-(_t2)) < 0)
875#define timestamp_leq(_t1, _t2) ((i32)((_t1)-(_t2)) <= 0)
876
Florin Coras93992a92017-05-24 18:03:56 -0700877/**
878 * Our estimate of the number of bytes that have left the network
879 */
880always_inline u32
881tcp_bytes_out (const tcp_connection_t * tc)
882{
883 if (tcp_opts_sack_permitted (&tc->rcv_opts))
884 return tc->sack_sb.sacked_bytes + tc->sack_sb.lost_bytes;
885 else
886 return tc->rcv_dupacks * tc->snd_mss;
887}
888
889/**
890 * Our estimate of the number of bytes in flight (pipe size)
891 */
Dave Barach68b0fb02017-02-28 15:15:56 -0500892always_inline u32
893tcp_flight_size (const tcp_connection_t * tc)
894{
Florin Coras6792ec02017-03-13 03:49:51 -0700895 int flight_size;
896
Florin Coras47596832019-03-12 18:58:54 -0700897 flight_size = (int) (tc->snd_nxt - tc->snd_una) - tcp_bytes_out (tc)
Florin Coras36ebcff2019-09-12 18:36:44 -0700898 + tc->snd_rxt_bytes - tc->rxt_delivered;
Florin Coras6792ec02017-03-13 03:49:51 -0700899
Florin Coras36ebcff2019-09-12 18:36:44 -0700900 ASSERT (flight_size >= 0);
Florin Coras6792ec02017-03-13 03:49:51 -0700901
902 return flight_size;
Dave Barach68b0fb02017-02-28 15:15:56 -0500903}
904
905/**
906 * Initial cwnd as per RFC5681
907 */
908always_inline u32
909tcp_initial_cwnd (const tcp_connection_t * tc)
910{
Florin Coras9094b5c2019-08-12 14:17:47 -0700911 if (tcp_cfg.initial_cwnd_multiplier > 0)
912 return tcp_cfg.initial_cwnd_multiplier * tc->snd_mss;
Vladimir Kropylev4f731f82019-06-26 12:56:21 +0300913
Dave Barach68b0fb02017-02-28 15:15:56 -0500914 if (tc->snd_mss > 2190)
915 return 2 * tc->snd_mss;
916 else if (tc->snd_mss > 1095)
917 return 3 * tc->snd_mss;
918 else
919 return 4 * tc->snd_mss;
920}
921
Florin Coras2e31cc32018-09-25 14:00:34 -0700922/*
923 * Accumulate acked bytes for cwnd increase
924 *
925 * Once threshold bytes are accumulated, snd_mss bytes are added
926 * to the cwnd.
927 */
928always_inline void
929tcp_cwnd_accumulate (tcp_connection_t * tc, u32 thresh, u32 bytes)
930{
931 tc->cwnd_acc_bytes += bytes;
932 if (tc->cwnd_acc_bytes >= thresh)
933 {
934 u32 inc = tc->cwnd_acc_bytes / thresh;
935 tc->cwnd_acc_bytes -= inc * thresh;
936 tc->cwnd += inc * tc->snd_mss;
937 tc->cwnd = clib_min (tc->cwnd, tc->tx_fifo_size);
938 }
939}
940
Dave Barach68b0fb02017-02-28 15:15:56 -0500941always_inline u32
942tcp_loss_wnd (const tcp_connection_t * tc)
943{
Florin Coras36ebcff2019-09-12 18:36:44 -0700944 /* Whatever we have in flight + the packet we're about to send */
945 return tcp_flight_size (tc) + tc->snd_mss;
Dave Barach68b0fb02017-02-28 15:15:56 -0500946}
947
948always_inline u32
Florin Coras1f152cd2017-08-18 19:28:03 -0700949tcp_available_snd_wnd (const tcp_connection_t * tc)
Dave Barach68b0fb02017-02-28 15:15:56 -0500950{
951 return clib_min (tc->cwnd, tc->snd_wnd);
952}
953
954always_inline u32
Florin Coras1f152cd2017-08-18 19:28:03 -0700955tcp_available_output_snd_space (const tcp_connection_t * tc)
956{
957 u32 available_wnd = tcp_available_snd_wnd (tc);
958 int flight_size = (int) (tc->snd_nxt - tc->snd_una);
959
960 if (available_wnd <= flight_size)
961 return 0;
962
963 return available_wnd - flight_size;
964}
965
966/**
967 * Estimate of how many bytes we can still push into the network
968 */
969always_inline u32
Florin Corasca1c8f32018-05-23 21:01:30 -0700970tcp_available_cc_snd_space (const tcp_connection_t * tc)
Dave Barach68b0fb02017-02-28 15:15:56 -0500971{
Florin Coras1f152cd2017-08-18 19:28:03 -0700972 u32 available_wnd = tcp_available_snd_wnd (tc);
Dave Barach68b0fb02017-02-28 15:15:56 -0500973 u32 flight_size = tcp_flight_size (tc);
974
975 if (available_wnd <= flight_size)
976 return 0;
977
978 return available_wnd - flight_size;
979}
980
Florin Corasb2215d62017-08-01 16:56:58 -0700981always_inline u8
982tcp_is_lost_fin (tcp_connection_t * tc)
983{
Florin Coras36ebcff2019-09-12 18:36:44 -0700984 if ((tc->flags & TCP_CONN_FINSNT) && (tc->snd_una_max - tc->snd_una == 1))
Florin Corasb2215d62017-08-01 16:56:58 -0700985 return 1;
986 return 0;
987}
988
Florin Coras45ca73f2018-09-27 09:19:29 -0700989u32 tcp_snd_space (tcp_connection_t * tc);
Florin Corasbe237bf2019-09-27 08:16:40 -0700990int tcp_fastrecovery_prr_snd_space (tcp_connection_t * tc);
Dave Barach68b0fb02017-02-28 15:15:56 -0500991
Florin Corasf6359c82017-06-19 12:26:09 -0400992fib_node_index_t tcp_lookup_rmt_in_fib (tcp_connection_t * tc);
993
Florin Coras45d34962017-04-25 00:05:27 -0700994/* Made public for unit testing only */
995void tcp_update_sack_list (tcp_connection_t * tc, u32 start, u32 end);
Florin Corasca1c8f32018-05-23 21:01:30 -0700996u32 tcp_sack_list_bytes (tcp_connection_t * tc);
Florin Coras45d34962017-04-25 00:05:27 -0700997
Dave Barach68b0fb02017-02-28 15:15:56 -0500998always_inline u32
999tcp_time_now (void)
1000{
Florin Coras2c414432018-06-19 09:58:04 -07001001 return tcp_main.wrk_ctx[vlib_get_thread_index ()].time_now;
Florin Coras82d3ec82017-08-14 08:10:42 -07001002}
1003
Florin Corasbe72ae62018-11-01 11:23:03 -07001004always_inline u32
1005tcp_time_now_w_thread (u32 thread_index)
1006{
1007 return tcp_main.wrk_ctx[thread_index].time_now;
1008}
1009
Vladimir Kropylev1cfcb782019-07-02 11:25:26 +03001010/**
1011 * Generate timestamp for tcp connection
1012 */
1013always_inline u32
1014tcp_tstamp (tcp_connection_t * tc)
1015{
1016 return (tcp_main.wrk_ctx[tc->c_thread_index].time_now -
1017 tc->timestamp_delta);
1018}
1019
Florin Corasd67f1122018-05-21 17:47:40 -07001020always_inline f64
1021tcp_time_now_us (u32 thread_index)
1022{
1023 return transport_time_now (thread_index);
1024}
1025
Florin Coras82d3ec82017-08-14 08:10:42 -07001026always_inline u32
Florin Corasbe72ae62018-11-01 11:23:03 -07001027tcp_set_time_now (tcp_worker_ctx_t * wrk)
Florin Coras82d3ec82017-08-14 08:10:42 -07001028{
Florin Corasbe72ae62018-11-01 11:23:03 -07001029 wrk->time_now = clib_cpu_time_now () * tcp_main.tstamp_ticks_per_clock;
1030 return wrk->time_now;
Dave Barach68b0fb02017-02-28 15:15:56 -05001031}
1032
Florin Coras14ed6df2019-03-06 21:13:42 -08001033u32 tcp_session_push_header (transport_connection_t * tconn,
1034 vlib_buffer_t * b);
Florin Coras26dd6de2019-07-23 23:54:47 -07001035int tcp_session_custom_tx (void *conn, u32 max_burst_size);
Dave Barach68b0fb02017-02-28 15:15:56 -05001036
1037void tcp_connection_timers_init (tcp_connection_t * tc);
1038void tcp_connection_timers_reset (tcp_connection_t * tc);
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001039void tcp_init_snd_vars (tcp_connection_t * tc);
Dave Barach68b0fb02017-02-28 15:15:56 -05001040void tcp_connection_init_vars (tcp_connection_t * tc);
Florin Corasc44a5582018-11-01 16:30:54 -07001041void tcp_connection_tx_pacer_update (tcp_connection_t * tc);
1042void tcp_connection_tx_pacer_reset (tcp_connection_t * tc, u32 window,
1043 u32 start_bucket);
Florin Corasd67f1122018-05-21 17:47:40 -07001044
1045always_inline void
Florin Coras52814732019-06-12 15:38:19 -07001046tcp_cc_rcv_ack (tcp_connection_t * tc, tcp_rate_sample_t * rs)
Florin Corasd67f1122018-05-21 17:47:40 -07001047{
Florin Coras52814732019-06-12 15:38:19 -07001048 tc->cc_algo->rcv_ack (tc, rs);
Florin Corasd67f1122018-05-21 17:47:40 -07001049 tc->tsecr_last_ack = tc->rcv_opts.tsecr;
1050}
Dave Barach68b0fb02017-02-28 15:15:56 -05001051
Florin Coras52814732019-06-12 15:38:19 -07001052static inline void
1053tcp_cc_rcv_cong_ack (tcp_connection_t * tc, tcp_cc_ack_t ack_type,
1054 tcp_rate_sample_t * rs)
1055{
1056 tc->cc_algo->rcv_cong_ack (tc, ack_type, rs);
1057}
1058
Florin Corasa3c32652019-07-03 17:47:22 -07001059static inline void
Florin Coras36ebcff2019-09-12 18:36:44 -07001060tcp_cc_congestion (tcp_connection_t * tc)
1061{
1062 tc->cc_algo->congestion (tc);
1063}
1064
1065static inline void
Florin Corasa3c32652019-07-03 17:47:22 -07001066tcp_cc_loss (tcp_connection_t * tc)
1067{
1068 tc->cc_algo->loss (tc);
1069}
1070
Florin Coraseff6b822019-07-03 19:51:02 -07001071static inline void
1072tcp_cc_recovered (tcp_connection_t * tc)
1073{
1074 tc->cc_algo->recovered (tc);
1075}
1076
1077static inline void
1078tcp_cc_undo_recovery (tcp_connection_t * tc)
1079{
1080 if (tc->cc_algo->undo_recovery)
1081 tc->cc_algo->undo_recovery (tc);
1082}
1083
Florin Corasca2831a2019-07-04 17:05:59 -07001084static inline void
1085tcp_cc_event (tcp_connection_t * tc, tcp_cc_event_t evt)
1086{
1087 if (tc->cc_algo->event)
1088 tc->cc_algo->event (tc, evt);
1089}
1090
Florin Corasd2067242019-08-16 10:33:49 -07001091static inline u64
1092tcp_cc_get_pacing_rate (tcp_connection_t * tc)
1093{
1094 if (tc->cc_algo->get_pacing_rate)
1095 return tc->cc_algo->get_pacing_rate (tc);
1096
1097 f64 srtt = clib_min ((f64) tc->srtt * TCP_TICK, tc->mrtt_us);
Florin Coras36ebcff2019-09-12 18:36:44 -07001098
Florin Corasd2067242019-08-16 10:33:49 -07001099 /* TODO should constrain to interface's max throughput but
1100 * we don't have link speeds for sw ifs ..*/
Florin Coras36ebcff2019-09-12 18:36:44 -07001101 return ((f64) tc->cwnd / srtt);
Florin Corasd2067242019-08-16 10:33:49 -07001102}
1103
Dave Barach68b0fb02017-02-28 15:15:56 -05001104always_inline void
Dave Barach68b0fb02017-02-28 15:15:56 -05001105tcp_timer_set (tcp_connection_t * tc, u8 timer_id, u32 interval)
1106{
Dave Barach2c25a622017-06-26 11:35:07 -04001107 ASSERT (tc->c_thread_index == vlib_get_thread_index ());
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001108 ASSERT (tc->timers[timer_id] == TCP_TIMER_HANDLE_INVALID);
Florin Coras2c414432018-06-19 09:58:04 -07001109 tc->timers[timer_id] =
1110 tw_timer_start_16t_2w_512sl (&tcp_main.
1111 wrk_ctx[tc->c_thread_index].timer_wheel,
1112 tc->c_c_index, timer_id, interval);
Dave Barach68b0fb02017-02-28 15:15:56 -05001113}
1114
1115always_inline void
Dave Barach68b0fb02017-02-28 15:15:56 -05001116tcp_timer_reset (tcp_connection_t * tc, u8 timer_id)
1117{
Dave Barach2c25a622017-06-26 11:35:07 -04001118 ASSERT (tc->c_thread_index == vlib_get_thread_index ());
Dave Barach68b0fb02017-02-28 15:15:56 -05001119 if (tc->timers[timer_id] == TCP_TIMER_HANDLE_INVALID)
1120 return;
1121
Florin Coras2c414432018-06-19 09:58:04 -07001122 tw_timer_stop_16t_2w_512sl (&tcp_main.
1123 wrk_ctx[tc->c_thread_index].timer_wheel,
Dave Barach68b0fb02017-02-28 15:15:56 -05001124 tc->timers[timer_id]);
1125 tc->timers[timer_id] = TCP_TIMER_HANDLE_INVALID;
1126}
1127
1128always_inline void
1129tcp_timer_update (tcp_connection_t * tc, u8 timer_id, u32 interval)
1130{
Dave Barach2c25a622017-06-26 11:35:07 -04001131 ASSERT (tc->c_thread_index == vlib_get_thread_index ());
Dave Barach68b0fb02017-02-28 15:15:56 -05001132 if (tc->timers[timer_id] != TCP_TIMER_HANDLE_INVALID)
Florin Corasadb5bd52018-06-22 15:29:38 -07001133 tw_timer_update_16t_2w_512sl (&tcp_main.
1134 wrk_ctx[tc->c_thread_index].timer_wheel,
1135 tc->timers[timer_id], interval);
1136 else
1137 tc->timers[timer_id] =
1138 tw_timer_start_16t_2w_512sl (&tcp_main.
1139 wrk_ctx[tc->c_thread_index].timer_wheel,
1140 tc->c_c_index, timer_id, interval);
Dave Barach68b0fb02017-02-28 15:15:56 -05001141}
1142
Florin Coras6792ec02017-03-13 03:49:51 -07001143always_inline void
1144tcp_retransmit_timer_set (tcp_connection_t * tc)
1145{
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001146 ASSERT (tc->snd_una != tc->snd_una_max);
Florin Coras6792ec02017-03-13 03:49:51 -07001147 tcp_timer_set (tc, TCP_TIMER_RETRANSMIT,
1148 clib_max (tc->rto * TCP_TO_TIMER_TICK, 1));
1149}
1150
1151always_inline void
Florin Coras6792ec02017-03-13 03:49:51 -07001152tcp_retransmit_timer_reset (tcp_connection_t * tc)
1153{
1154 tcp_timer_reset (tc, TCP_TIMER_RETRANSMIT);
1155}
1156
Florin Coras3e350af2017-03-30 02:54:28 -07001157always_inline void
Florin Coras93992a92017-05-24 18:03:56 -07001158tcp_retransmit_timer_force_update (tcp_connection_t * tc)
1159{
1160 tcp_timer_update (tc, TCP_TIMER_RETRANSMIT,
1161 clib_max (tc->rto * TCP_TO_TIMER_TICK, 1));
1162}
1163
1164always_inline void
Florin Coras3e350af2017-03-30 02:54:28 -07001165tcp_persist_timer_set (tcp_connection_t * tc)
1166{
1167 /* Reuse RTO. It's backed off in handler */
1168 tcp_timer_set (tc, TCP_TIMER_PERSIST,
Florin Coras9094b5c2019-08-12 14:17:47 -07001169 clib_max (tc->rto * TCP_TO_TIMER_TICK, 1));
Florin Coras3e350af2017-03-30 02:54:28 -07001170}
1171
1172always_inline void
1173tcp_persist_timer_update (tcp_connection_t * tc)
1174{
1175 tcp_timer_update (tc, TCP_TIMER_PERSIST,
Florin Coras9094b5c2019-08-12 14:17:47 -07001176 clib_max (tc->rto * TCP_TO_TIMER_TICK, 1));
Florin Coras3e350af2017-03-30 02:54:28 -07001177}
1178
1179always_inline void
1180tcp_persist_timer_reset (tcp_connection_t * tc)
1181{
1182 tcp_timer_reset (tc, TCP_TIMER_PERSIST);
1183}
1184
Florin Coras93992a92017-05-24 18:03:56 -07001185always_inline void
1186tcp_retransmit_timer_update (tcp_connection_t * tc)
1187{
Florin Coras47596832019-03-12 18:58:54 -07001188 if (tc->snd_una == tc->snd_nxt)
Florin Coras93992a92017-05-24 18:03:56 -07001189 {
1190 tcp_retransmit_timer_reset (tc);
1191 if (tc->snd_wnd < tc->snd_mss)
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001192 tcp_persist_timer_update (tc);
Florin Coras93992a92017-05-24 18:03:56 -07001193 }
1194 else
1195 tcp_timer_update (tc, TCP_TIMER_RETRANSMIT,
1196 clib_max (tc->rto * TCP_TO_TIMER_TICK, 1));
1197}
1198
Dave Barach68b0fb02017-02-28 15:15:56 -05001199always_inline u8
1200tcp_timer_is_active (tcp_connection_t * tc, tcp_timers_e timer)
1201{
1202 return tc->timers[timer] != TCP_TIMER_HANDLE_INVALID;
1203}
1204
Florin Coras93992a92017-05-24 18:03:56 -07001205#define tcp_validate_txf_size(_tc, _a) \
1206 ASSERT(_tc->state != TCP_STATE_ESTABLISHED \
Florin Coras31c99552019-03-01 13:00:58 -08001207 || transport_max_tx_dequeue (&_tc->connection) >= _a)
Florin Coras93992a92017-05-24 18:03:56 -07001208
Florin Coras6792ec02017-03-13 03:49:51 -07001209void tcp_rcv_sacks (tcp_connection_t * tc, u32 ack);
Florin Coras0dbd5172018-06-25 16:19:34 -07001210u8 *tcp_scoreboard_replay (u8 * s, tcp_connection_t * tc, u8 verbose);
Florin Coras6792ec02017-03-13 03:49:51 -07001211
Florin Coras4e116fb2019-06-10 08:33:50 -07001212/**
1213 * Register exiting cc algo type
1214 */
Florin Coras0dbd5172018-06-25 16:19:34 -07001215void tcp_cc_algo_register (tcp_cc_algorithm_type_e type,
1216 const tcp_cc_algorithm_t * vft);
Dave Barach68b0fb02017-02-28 15:15:56 -05001217
Florin Coras4e116fb2019-06-10 08:33:50 -07001218/**
1219 * Register new cc algo type
1220 */
1221tcp_cc_algorithm_type_e tcp_cc_algo_new_type (const tcp_cc_algorithm_t * vft);
Florin Coras0dbd5172018-06-25 16:19:34 -07001222tcp_cc_algorithm_t *tcp_cc_algo_get (tcp_cc_algorithm_type_e type);
Dave Barach68b0fb02017-02-28 15:15:56 -05001223
Florin Coras2e31cc32018-09-25 14:00:34 -07001224static inline void *
1225tcp_cc_data (tcp_connection_t * tc)
1226{
1227 return (void *) tc->cc_data;
1228}
1229
Florin Coras52814732019-06-12 15:38:19 -07001230void newreno_rcv_cong_ack (tcp_connection_t * tc, tcp_cc_ack_t ack_type,
1231 tcp_rate_sample_t * rs);
Florin Coras2e31cc32018-09-25 14:00:34 -07001232
Dave Barach68b0fb02017-02-28 15:15:56 -05001233/**
1234 * Push TCP header to buffer
1235 *
1236 * @param vm - vlib_main
1237 * @param b - buffer to write the header to
1238 * @param sp_net - source port net order
1239 * @param dp_net - destination port net order
1240 * @param seq - sequence number net order
1241 * @param ack - ack number net order
1242 * @param tcp_hdr_opts_len - header and options length in bytes
1243 * @param flags - header flags
1244 * @param wnd - window size
1245 *
1246 * @return - pointer to start of TCP header
1247 */
1248always_inline void *
1249vlib_buffer_push_tcp_net_order (vlib_buffer_t * b, u16 sp, u16 dp, u32 seq,
1250 u32 ack, u8 tcp_hdr_opts_len, u8 flags,
1251 u16 wnd)
1252{
1253 tcp_header_t *th;
1254
1255 th = vlib_buffer_push_uninit (b, tcp_hdr_opts_len);
1256
1257 th->src_port = sp;
1258 th->dst_port = dp;
1259 th->seq_number = seq;
1260 th->ack_number = ack;
1261 th->data_offset_and_reserved = (tcp_hdr_opts_len >> 2) << 4;
1262 th->flags = flags;
1263 th->window = wnd;
1264 th->checksum = 0;
1265 th->urgent_pointer = 0;
Srikanth A02833ff2019-10-02 17:48:58 -07001266 vnet_buffer (b)->l4_hdr_offset = (u8 *) th - b->data;
1267 b->flags |= VNET_BUFFER_F_L4_HDR_OFFSET_VALID;
Dave Barach68b0fb02017-02-28 15:15:56 -05001268 return th;
1269}
1270
1271/**
1272 * Push TCP header to buffer
1273 *
Dave Barach68b0fb02017-02-28 15:15:56 -05001274 * @param b - buffer to write the header to
1275 * @param sp_net - source port net order
1276 * @param dp_net - destination port net order
1277 * @param seq - sequence number host order
1278 * @param ack - ack number host order
1279 * @param tcp_hdr_opts_len - header and options length in bytes
1280 * @param flags - header flags
1281 * @param wnd - window size
1282 *
1283 * @return - pointer to start of TCP header
1284 */
1285always_inline void *
1286vlib_buffer_push_tcp (vlib_buffer_t * b, u16 sp_net, u16 dp_net, u32 seq,
1287 u32 ack, u8 tcp_hdr_opts_len, u8 flags, u16 wnd)
1288{
1289 return vlib_buffer_push_tcp_net_order (b, sp_net, dp_net,
1290 clib_host_to_net_u32 (seq),
1291 clib_host_to_net_u32 (ack),
1292 tcp_hdr_opts_len, flags,
1293 clib_host_to_net_u16 (wnd));
1294}
1295
1296#endif /* _vnet_tcp_h_ */
1297
1298/*
1299 * fd.io coding-style-patch-verification: ON
1300 *
1301 * Local Variables:
1302 * eval: (c-set-style "gnu")
1303 * End:
1304 */