blob: 5235c20a565930f2059dfc33d14bfa6c86d63b7a [file] [log] [blame]
Dave Barach68b0fb02017-02-28 15:15:56 -05001/*
Florin Coras830fe732019-02-15 18:20:58 -08002 * Copyright (c) 2016-2019 Cisco and/or its affiliates.
Dave Barach68b0fb02017-02-28 15:15:56 -05003 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16#include <vppinfra/sparse_vec.h>
17#include <vnet/tcp/tcp_packet.h>
18#include <vnet/tcp/tcp.h>
19#include <vnet/session/session.h>
20#include <math.h>
21
22static char *tcp_error_strings[] = {
23#define tcp_error(n,s) s,
24#include <vnet/tcp/tcp_error.def>
25#undef tcp_error
26};
27
28/* All TCP nodes have the same outgoing arcs */
29#define foreach_tcp_state_next \
Vijayabhaskar Katamreddyce074122017-11-15 13:50:26 -080030 _ (DROP4, "ip4-drop") \
31 _ (DROP6, "ip6-drop") \
Dave Barach68b0fb02017-02-28 15:15:56 -050032 _ (TCP4_OUTPUT, "tcp4-output") \
33 _ (TCP6_OUTPUT, "tcp6-output")
34
35typedef enum _tcp_established_next
36{
37#define _(s,n) TCP_ESTABLISHED_NEXT_##s,
38 foreach_tcp_state_next
39#undef _
40 TCP_ESTABLISHED_N_NEXT,
41} tcp_established_next_t;
42
43typedef enum _tcp_rcv_process_next
44{
45#define _(s,n) TCP_RCV_PROCESS_NEXT_##s,
46 foreach_tcp_state_next
47#undef _
48 TCP_RCV_PROCESS_N_NEXT,
49} tcp_rcv_process_next_t;
50
51typedef enum _tcp_syn_sent_next
52{
53#define _(s,n) TCP_SYN_SENT_NEXT_##s,
54 foreach_tcp_state_next
55#undef _
56 TCP_SYN_SENT_N_NEXT,
57} tcp_syn_sent_next_t;
58
59typedef enum _tcp_listen_next
60{
61#define _(s,n) TCP_LISTEN_NEXT_##s,
62 foreach_tcp_state_next
63#undef _
64 TCP_LISTEN_N_NEXT,
65} tcp_listen_next_t;
66
67/* Generic, state independent indices */
68typedef enum _tcp_state_next
69{
70#define _(s,n) TCP_NEXT_##s,
71 foreach_tcp_state_next
72#undef _
73 TCP_STATE_N_NEXT,
74} tcp_state_next_t;
75
76#define tcp_next_output(is_ip4) (is_ip4 ? TCP_NEXT_TCP4_OUTPUT \
77 : TCP_NEXT_TCP6_OUTPUT)
78
Vijayabhaskar Katamreddyce074122017-11-15 13:50:26 -080079#define tcp_next_drop(is_ip4) (is_ip4 ? TCP_NEXT_DROP4 \
80 : TCP_NEXT_DROP6)
81
Dave Barach68b0fb02017-02-28 15:15:56 -050082/**
83 * Validate segment sequence number. As per RFC793:
84 *
85 * Segment Receive Test
86 * Length Window
87 * ------- ------- -------------------------------------------
88 * 0 0 SEG.SEQ = RCV.NXT
89 * 0 >0 RCV.NXT =< SEG.SEQ < RCV.NXT+RCV.WND
90 * >0 0 not acceptable
91 * >0 >0 RCV.NXT =< SEG.SEQ < RCV.NXT+RCV.WND
92 * or RCV.NXT =< SEG.SEQ+SEG.LEN-1 < RCV.NXT+RCV.WND
93 *
94 * This ultimately consists in checking if segment falls within the window.
95 * The one important difference compared to RFC793 is that we use rcv_las,
96 * or the rcv_nxt at last ack sent instead of rcv_nxt since that's the
97 * peer's reference when computing our receive window.
98 *
Florin Coras6792ec02017-03-13 03:49:51 -070099 * This:
100 * seq_leq (end_seq, tc->rcv_las + tc->rcv_wnd) && seq_geq (seq, tc->rcv_las)
101 * however, is too strict when we have retransmits. Instead we just check that
102 * the seq is not beyond the right edge and that the end of the segment is not
103 * less than the left edge.
104 *
105 * N.B. rcv_nxt and rcv_wnd are both updated in this node if acks are sent, so
106 * use rcv_nxt in the right edge window test instead of rcv_las.
107 *
Dave Barach68b0fb02017-02-28 15:15:56 -0500108 */
109always_inline u8
110tcp_segment_in_rcv_wnd (tcp_connection_t * tc, u32 seq, u32 end_seq)
111{
Florin Coras6792ec02017-03-13 03:49:51 -0700112 return (seq_geq (end_seq, tc->rcv_las)
113 && seq_leq (seq, tc->rcv_nxt + tc->rcv_wnd));
Dave Barach68b0fb02017-02-28 15:15:56 -0500114}
115
Florin Corasdb84e572017-05-09 18:54:52 -0700116/**
117 * Parse TCP header options.
118 *
119 * @param th TCP header
120 * @param to TCP options data structure to be populated
Florin Coras80231112018-12-05 15:59:31 -0800121 * @param is_syn set if packet is syn
Florin Corasdb84e572017-05-09 18:54:52 -0700122 * @return -1 if parsing failed
123 */
Florin Coras80231112018-12-05 15:59:31 -0800124static inline int
125tcp_options_parse (tcp_header_t * th, tcp_options_t * to, u8 is_syn)
Dave Barach68b0fb02017-02-28 15:15:56 -0500126{
127 const u8 *data;
128 u8 opt_len, opts_len, kind;
129 int j;
130 sack_block_t b;
131
132 opts_len = (tcp_doff (th) << 2) - sizeof (tcp_header_t);
133 data = (const u8 *) (th + 1);
134
135 /* Zero out all flags but those set in SYN */
Florin Corasd6fe5bd2018-10-16 19:52:10 -0700136 to->flags &= (TCP_OPTS_FLAG_SACK_PERMITTED | TCP_OPTS_FLAG_WSCALE
Florin Coras80231112018-12-05 15:59:31 -0800137 | TCP_OPTS_FLAG_TSTAMP | TCP_OPTION_MSS);
Dave Barach68b0fb02017-02-28 15:15:56 -0500138
139 for (; opts_len > 0; opts_len -= opt_len, data += opt_len)
140 {
141 kind = data[0];
142
143 /* Get options length */
144 if (kind == TCP_OPTION_EOL)
145 break;
146 else if (kind == TCP_OPTION_NOOP)
Florin Corasdb84e572017-05-09 18:54:52 -0700147 {
148 opt_len = 1;
149 continue;
150 }
Dave Barach68b0fb02017-02-28 15:15:56 -0500151 else
152 {
153 /* broken options */
154 if (opts_len < 2)
Florin Corasdb84e572017-05-09 18:54:52 -0700155 return -1;
Dave Barach68b0fb02017-02-28 15:15:56 -0500156 opt_len = data[1];
157
158 /* weird option length */
159 if (opt_len < 2 || opt_len > opts_len)
Florin Corasdb84e572017-05-09 18:54:52 -0700160 return -1;
Dave Barach68b0fb02017-02-28 15:15:56 -0500161 }
162
163 /* Parse options */
164 switch (kind)
165 {
166 case TCP_OPTION_MSS:
Florin Coras80231112018-12-05 15:59:31 -0800167 if (!is_syn)
168 break;
Dave Barach68b0fb02017-02-28 15:15:56 -0500169 if ((opt_len == TCP_OPTION_LEN_MSS) && tcp_syn (th))
170 {
171 to->flags |= TCP_OPTS_FLAG_MSS;
172 to->mss = clib_net_to_host_u16 (*(u16 *) (data + 2));
173 }
174 break;
175 case TCP_OPTION_WINDOW_SCALE:
Florin Coras80231112018-12-05 15:59:31 -0800176 if (!is_syn)
177 break;
Dave Barach68b0fb02017-02-28 15:15:56 -0500178 if ((opt_len == TCP_OPTION_LEN_WINDOW_SCALE) && tcp_syn (th))
179 {
180 to->flags |= TCP_OPTS_FLAG_WSCALE;
181 to->wscale = data[2];
182 if (to->wscale > TCP_MAX_WND_SCALE)
Florin Corasa9d5bea2018-12-17 08:24:19 -0800183 to->wscale = TCP_MAX_WND_SCALE;
Dave Barach68b0fb02017-02-28 15:15:56 -0500184 }
185 break;
186 case TCP_OPTION_TIMESTAMP:
Florin Coras80231112018-12-05 15:59:31 -0800187 if (is_syn)
188 to->flags |= TCP_OPTS_FLAG_TSTAMP;
189 if ((to->flags & TCP_OPTS_FLAG_TSTAMP)
190 && opt_len == TCP_OPTION_LEN_TIMESTAMP)
Dave Barach68b0fb02017-02-28 15:15:56 -0500191 {
Dave Barach68b0fb02017-02-28 15:15:56 -0500192 to->tsval = clib_net_to_host_u32 (*(u32 *) (data + 2));
193 to->tsecr = clib_net_to_host_u32 (*(u32 *) (data + 6));
194 }
195 break;
196 case TCP_OPTION_SACK_PERMITTED:
Florin Coras80231112018-12-05 15:59:31 -0800197 if (!is_syn)
198 break;
Dave Barach68b0fb02017-02-28 15:15:56 -0500199 if (opt_len == TCP_OPTION_LEN_SACK_PERMITTED && tcp_syn (th))
200 to->flags |= TCP_OPTS_FLAG_SACK_PERMITTED;
201 break;
202 case TCP_OPTION_SACK_BLOCK:
203 /* If SACK permitted was not advertised or a SYN, break */
204 if ((to->flags & TCP_OPTS_FLAG_SACK_PERMITTED) == 0 || tcp_syn (th))
205 break;
206
207 /* If too short or not correctly formatted, break */
208 if (opt_len < 10 || ((opt_len - 2) % TCP_OPTION_LEN_SACK_BLOCK))
209 break;
210
211 to->flags |= TCP_OPTS_FLAG_SACK;
212 to->n_sack_blocks = (opt_len - 2) / TCP_OPTION_LEN_SACK_BLOCK;
213 vec_reset_length (to->sacks);
214 for (j = 0; j < to->n_sack_blocks; j++)
215 {
Florin Coras3eb50622017-07-13 01:24:57 -0400216 b.start = clib_net_to_host_u32 (*(u32 *) (data + 2 + 8 * j));
217 b.end = clib_net_to_host_u32 (*(u32 *) (data + 6 + 8 * j));
Dave Barach68b0fb02017-02-28 15:15:56 -0500218 vec_add1 (to->sacks, b);
219 }
220 break;
221 default:
222 /* Nothing to see here */
223 continue;
224 }
225 }
Florin Corasdb84e572017-05-09 18:54:52 -0700226 return 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500227}
228
Florin Corasc28764f2017-04-26 00:08:42 -0700229/**
230 * RFC1323: Check against wrapped sequence numbers (PAWS). If we have
231 * timestamp to echo and it's less than tsval_recent, drop segment
232 * but still send an ACK in order to retain TCP's mechanism for detecting
233 * and recovering from half-open connections
234 *
235 * Or at least that's what the theory says. It seems that this might not work
236 * very well with packet reordering and fast retransmit. XXX
237 */
Dave Barach68b0fb02017-02-28 15:15:56 -0500238always_inline int
239tcp_segment_check_paws (tcp_connection_t * tc)
240{
Florin Corasea41aac2018-12-06 17:24:59 -0800241 return tcp_opts_tstamp (&tc->rcv_opts)
Florin Coras93992a92017-05-24 18:03:56 -0700242 && timestamp_lt (tc->rcv_opts.tsval, tc->tsval_recent);
Dave Barach68b0fb02017-02-28 15:15:56 -0500243}
244
245/**
Florin Corasc28764f2017-04-26 00:08:42 -0700246 * Update tsval recent
247 */
248always_inline void
249tcp_update_timestamp (tcp_connection_t * tc, u32 seq, u32 seq_end)
250{
251 /*
252 * RFC1323: If Last.ACK.sent falls within the range of sequence numbers
253 * of an incoming segment:
254 * SEG.SEQ <= Last.ACK.sent < SEG.SEQ + SEG.LEN
255 * then the TSval from the segment is copied to TS.Recent;
256 * otherwise, the TSval is ignored.
257 */
Florin Corasf1762d62017-09-24 19:43:08 -0400258 if (tcp_opts_tstamp (&tc->rcv_opts) && seq_leq (seq, tc->rcv_las)
259 && seq_leq (tc->rcv_las, seq_end))
Florin Corasc28764f2017-04-26 00:08:42 -0700260 {
Dave Barach2c25a622017-06-26 11:35:07 -0400261 ASSERT (timestamp_leq (tc->tsval_recent, tc->rcv_opts.tsval));
Florin Coras93992a92017-05-24 18:03:56 -0700262 tc->tsval_recent = tc->rcv_opts.tsval;
Florin Corasbe72ae62018-11-01 11:23:03 -0700263 tc->tsval_recent_age = tcp_time_now_w_thread (tc->c_thread_index);
Florin Corasc28764f2017-04-26 00:08:42 -0700264 }
265}
266
267/**
Dave Barach68b0fb02017-02-28 15:15:56 -0500268 * Validate incoming segment as per RFC793 p. 69 and RFC1323 p. 19
269 *
270 * It first verifies if segment has a wrapped sequence number (PAWS) and then
271 * does the processing associated to the first four steps (ignoring security
272 * and precedence): sequence number, rst bit and syn bit checks.
273 *
274 * @return 0 if segments passes validation.
275 */
276static int
Florin Coras7ac053b2018-11-05 15:57:21 -0800277tcp_segment_validate (tcp_worker_ctx_t * wrk, tcp_connection_t * tc0,
278 vlib_buffer_t * b0, tcp_header_t * th0, u32 * error0)
Dave Barach68b0fb02017-02-28 15:15:56 -0500279{
Florin Corasca1c8f32018-05-23 21:01:30 -0700280 /* We could get a burst of RSTs interleaved with acks */
281 if (PREDICT_FALSE (tc0->state == TCP_STATE_CLOSED))
282 {
283 tcp_send_reset (tc0);
284 *error0 = TCP_ERROR_CONNECTION_CLOSED;
Florin Coras7ac053b2018-11-05 15:57:21 -0800285 goto error;
Florin Corasca1c8f32018-05-23 21:01:30 -0700286 }
287
Dave Barach68b0fb02017-02-28 15:15:56 -0500288 if (PREDICT_FALSE (!tcp_ack (th0) && !tcp_rst (th0) && !tcp_syn (th0)))
Florin Coras00cd22d2018-04-18 13:20:18 -0700289 {
290 *error0 = TCP_ERROR_SEGMENT_INVALID;
Florin Coras7ac053b2018-11-05 15:57:21 -0800291 goto error;
Florin Coras00cd22d2018-04-18 13:20:18 -0700292 }
Dave Barach68b0fb02017-02-28 15:15:56 -0500293
Florin Coras80231112018-12-05 15:59:31 -0800294 if (PREDICT_FALSE (tcp_options_parse (th0, &tc0->rcv_opts, 0)))
Florin Corasdb84e572017-05-09 18:54:52 -0700295 {
Florin Coras00cd22d2018-04-18 13:20:18 -0700296 *error0 = TCP_ERROR_OPTIONS;
Florin Coras7ac053b2018-11-05 15:57:21 -0800297 goto error;
Florin Corasdb84e572017-05-09 18:54:52 -0700298 }
Dave Barach68b0fb02017-02-28 15:15:56 -0500299
Florin Coras00cd22d2018-04-18 13:20:18 -0700300 if (PREDICT_FALSE (tcp_segment_check_paws (tc0)))
Dave Barach68b0fb02017-02-28 15:15:56 -0500301 {
Florin Coras00cd22d2018-04-18 13:20:18 -0700302 *error0 = TCP_ERROR_PAWS;
Florin Corasc28764f2017-04-26 00:08:42 -0700303 TCP_EVT_DBG (TCP_EVT_PAWS_FAIL, tc0, vnet_buffer (b0)->tcp.seq_number,
304 vnet_buffer (b0)->tcp.seq_end);
Dave Barach68b0fb02017-02-28 15:15:56 -0500305
306 /* If it just so happens that a segment updates tsval_recent for a
307 * segment over 24 days old, invalidate tsval_recent. */
308 if (timestamp_lt (tc0->tsval_recent_age + TCP_PAWS_IDLE,
Florin Corasbe72ae62018-11-01 11:23:03 -0700309 tcp_time_now_w_thread (tc0->c_thread_index)))
Dave Barach68b0fb02017-02-28 15:15:56 -0500310 {
Florin Corasea41aac2018-12-06 17:24:59 -0800311 tc0->tsval_recent = tc0->rcv_opts.tsval;
Florin Coras91236ce2018-12-16 11:41:45 -0800312 clib_warning ("paws failed: 24-day old segment");
Dave Barach68b0fb02017-02-28 15:15:56 -0500313 }
Florin Coras91236ce2018-12-16 11:41:45 -0800314 /* Drop after ack if not rst. Resets can fail paws check as per
315 * RFC 7323 sec. 5.2: When an <RST> segment is received, it MUST NOT
316 * be subjected to the PAWS check by verifying an acceptable value in
317 * SEG.TSval */
318 else if (!tcp_rst (th0))
Dave Barach68b0fb02017-02-28 15:15:56 -0500319 {
Florin Coras91236ce2018-12-16 11:41:45 -0800320 tcp_program_ack (wrk, tc0);
321 TCP_EVT_DBG (TCP_EVT_DUPACK_SENT, tc0, vnet_buffer (b0)->tcp);
322 goto error;
Dave Barach68b0fb02017-02-28 15:15:56 -0500323 }
324 }
325
326 /* 1st: check sequence number */
327 if (!tcp_segment_in_rcv_wnd (tc0, vnet_buffer (b0)->tcp.seq_number,
328 vnet_buffer (b0)->tcp.seq_end))
329 {
Florin Coras830fe732019-02-15 18:20:58 -0800330 /* SYN/SYN-ACK retransmit */
331 if (tcp_syn (th0)
332 && vnet_buffer (b0)->tcp.seq_number == tc0->rcv_nxt - 1)
333 {
334 tcp_options_parse (th0, &tc0->rcv_opts, 1);
335 if (tc0->state == TCP_STATE_SYN_RCVD)
336 {
337 tcp_send_synack (tc0);
338 TCP_EVT_DBG (TCP_EVT_SYN_RCVD, tc0, 0);
339 *error0 = TCP_ERROR_SYNS_RCVD;
340 }
341 else
342 {
343 tcp_program_ack (wrk, tc0);
344 TCP_EVT_DBG (TCP_EVT_SYNACK_RCVD, tc0);
345 *error0 = TCP_ERROR_SYN_ACKS_RCVD;
346 }
347 goto error;
348 }
349
Florin Coras6792ec02017-03-13 03:49:51 -0700350 /* If our window is 0 and the packet is in sequence, let it pass
Florin Coras00cd22d2018-04-18 13:20:18 -0700351 * through for ack processing. It should be dropped later. */
Florin Coras7e74bf32019-03-06 16:51:58 -0800352 if (tc0->rcv_wnd < tc0->snd_mss
Florin Coras222e1f412019-02-16 20:47:32 -0800353 && tc0->rcv_nxt == vnet_buffer (b0)->tcp.seq_number)
354 goto check_reset;
355
356 /* If we entered recovery and peer did so as well, there's a chance that
357 * dup acks won't be acceptable on either end because seq_end may be less
358 * than rcv_las. This can happen if acks are lost in both directions. */
359 if (tcp_in_recovery (tc0)
360 && seq_geq (vnet_buffer (b0)->tcp.seq_number,
361 tc0->rcv_las - tc0->rcv_wnd)
362 && seq_leq (vnet_buffer (b0)->tcp.seq_end,
363 tc0->rcv_nxt + tc0->rcv_wnd))
364 goto check_reset;
365
366 *error0 = TCP_ERROR_RCV_WND;
367
368 /* If not RST, send dup ack */
369 if (!tcp_rst (th0))
Florin Coras6792ec02017-03-13 03:49:51 -0700370 {
Florin Coras222e1f412019-02-16 20:47:32 -0800371 tcp_program_dupack (wrk, tc0);
372 TCP_EVT_DBG (TCP_EVT_DUPACK_SENT, tc0, vnet_buffer (b0)->tcp);
Florin Coras6792ec02017-03-13 03:49:51 -0700373 }
Florin Coras222e1f412019-02-16 20:47:32 -0800374 goto error;
375
376 check_reset:
377 ;
Dave Barach68b0fb02017-02-28 15:15:56 -0500378 }
379
380 /* 2nd: check the RST bit */
Florin Coras00cd22d2018-04-18 13:20:18 -0700381 if (PREDICT_FALSE (tcp_rst (th0)))
Dave Barach68b0fb02017-02-28 15:15:56 -0500382 {
Florin Corasd79b41e2017-03-04 05:37:52 -0800383 tcp_connection_reset (tc0);
Florin Coras00cd22d2018-04-18 13:20:18 -0700384 *error0 = TCP_ERROR_RST_RCVD;
Florin Coras7ac053b2018-11-05 15:57:21 -0800385 goto error;
Dave Barach68b0fb02017-02-28 15:15:56 -0500386 }
387
388 /* 3rd: check security and precedence (skip) */
389
Florin Coras830fe732019-02-15 18:20:58 -0800390 /* 4th: check the SYN bit (in window) */
Florin Coras00cd22d2018-04-18 13:20:18 -0700391 if (PREDICT_FALSE (tcp_syn (th0)))
Dave Barach68b0fb02017-02-28 15:15:56 -0500392 {
Florin Coras830fe732019-02-15 18:20:58 -0800393 *error0 = TCP_ERROR_SPURIOUS_SYN;
394 tcp_send_reset (tc0);
Florin Coras00cd22d2018-04-18 13:20:18 -0700395 goto error;
Dave Barach68b0fb02017-02-28 15:15:56 -0500396 }
397
Florin Corasc28764f2017-04-26 00:08:42 -0700398 /* If segment in window, save timestamp */
399 tcp_update_timestamp (tc0, vnet_buffer (b0)->tcp.seq_number,
400 vnet_buffer (b0)->tcp.seq_end);
Dave Barach68b0fb02017-02-28 15:15:56 -0500401 return 0;
Florin Coras00cd22d2018-04-18 13:20:18 -0700402
Florin Coras00cd22d2018-04-18 13:20:18 -0700403error:
Florin Coras00cd22d2018-04-18 13:20:18 -0700404 return -1;
Dave Barach68b0fb02017-02-28 15:15:56 -0500405}
406
407always_inline int
408tcp_rcv_ack_is_acceptable (tcp_connection_t * tc0, vlib_buffer_t * tb0)
409{
410 /* SND.UNA =< SEG.ACK =< SND.NXT */
411 return (seq_leq (tc0->snd_una, vnet_buffer (tb0)->tcp.ack_number)
412 && seq_leq (vnet_buffer (tb0)->tcp.ack_number, tc0->snd_nxt));
413}
414
415/**
416 * Compute smoothed RTT as per VJ's '88 SIGCOMM and RFC6298
417 *
418 * Note that although the original article, srtt and rttvar are scaled
419 * to minimize round-off errors, here we don't. Instead, we rely on
420 * better precision time measurements.
421 *
422 * TODO support us rtt resolution
423 */
424static void
425tcp_estimate_rtt (tcp_connection_t * tc, u32 mrtt)
426{
Florin Corasf03a59a2017-06-09 21:07:32 -0700427 int err, diff;
Dave Barach68b0fb02017-02-28 15:15:56 -0500428
429 if (tc->srtt != 0)
430 {
431 err = mrtt - tc->srtt;
Dave Barach68b0fb02017-02-28 15:15:56 -0500432
433 /* XXX Drop in RTT results in RTTVAR increase and bigger RTO.
434 * The increase should be bound */
Florin Corasf03a59a2017-06-09 21:07:32 -0700435 tc->srtt = clib_max ((int) tc->srtt + (err >> 3), 1);
436 diff = (clib_abs (err) - (int) tc->rttvar) >> 2;
437 tc->rttvar = clib_max ((int) tc->rttvar + diff, 1);
Dave Barach68b0fb02017-02-28 15:15:56 -0500438 }
439 else
440 {
441 /* First measurement. */
442 tc->srtt = mrtt;
Florin Coras6792ec02017-03-13 03:49:51 -0700443 tc->rttvar = mrtt >> 1;
Dave Barach68b0fb02017-02-28 15:15:56 -0500444 }
445}
446
Filip Tehlare275bed2019-03-06 00:06:56 -0800447#ifndef CLIB_MARCH_VARIANT
Florin Coras93992a92017-05-24 18:03:56 -0700448void
449tcp_update_rto (tcp_connection_t * tc)
450{
451 tc->rto = clib_min (tc->srtt + (tc->rttvar << 2), TCP_RTO_MAX);
Florin Corasf03a59a2017-06-09 21:07:32 -0700452 tc->rto = clib_max (tc->rto, TCP_RTO_MIN);
Florin Coras93992a92017-05-24 18:03:56 -0700453}
Filip Tehlare275bed2019-03-06 00:06:56 -0800454#endif /* CLIB_MARCH_VARIANT */
Florin Coras93992a92017-05-24 18:03:56 -0700455
Florin Corasf1762d62017-09-24 19:43:08 -0400456/**
457 * Update RTT estimate and RTO timer
Dave Barach68b0fb02017-02-28 15:15:56 -0500458 *
459 * Measure RTT: We have two sources of RTT measurements: TSOPT and ACK
460 * timing. Middle boxes are known to fiddle with TCP options so we
461 * should give higher priority to ACK timing.
462 *
Florin Corasf1762d62017-09-24 19:43:08 -0400463 * This should be called only if previously sent bytes have been acked.
464 *
Dave Barach68b0fb02017-02-28 15:15:56 -0500465 * return 1 if valid rtt 0 otherwise
466 */
467static int
468tcp_update_rtt (tcp_connection_t * tc, u32 ack)
469{
470 u32 mrtt = 0;
471
472 /* Karn's rule, part 1. Don't use retransmitted segments to estimate
473 * RTT because they're ambiguous. */
Florin Corasf1762d62017-09-24 19:43:08 -0400474 if (tcp_in_cong_recovery (tc) || tc->sack_sb.sacked_bytes)
Florin Coras3ec66b02018-08-23 16:27:05 -0700475 {
476 if (tcp_in_recovery (tc))
477 return 0;
478 goto done;
479 }
Florin Corasf1762d62017-09-24 19:43:08 -0400480
481 if (tc->rtt_ts && seq_geq (ack, tc->rtt_seq))
Dave Barach68b0fb02017-02-28 15:15:56 -0500482 {
Florin Corasefefc6b2018-11-07 12:49:19 -0800483 f64 sample = tcp_time_now_us (tc->c_thread_index) - tc->rtt_ts;
484 tc->mrtt_us = tc->mrtt_us + (sample - tc->mrtt_us) * 0.125;
485 mrtt = clib_max ((u32) (sample * THZ), 1);
486 /* Allow measuring of a new RTT */
487 tc->rtt_ts = 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500488 }
Dave Barach68b0fb02017-02-28 15:15:56 -0500489 /* As per RFC7323 TSecr can be used for RTTM only if the segment advances
490 * snd_una, i.e., the left side of the send window:
Florin Corasf1762d62017-09-24 19:43:08 -0400491 * seq_lt (tc->snd_una, ack). This is a condition for calling update_rtt */
492 else if (tcp_opts_tstamp (&tc->rcv_opts) && tc->rcv_opts.tsecr)
Dave Barach68b0fb02017-02-28 15:15:56 -0500493 {
Florin Corasbe72ae62018-11-01 11:23:03 -0700494 u32 now = tcp_time_now_w_thread (tc->c_thread_index);
495 mrtt = clib_max (now - tc->rcv_opts.tsecr, 1);
Dave Barach68b0fb02017-02-28 15:15:56 -0500496 }
497
Florin Corasf1762d62017-09-24 19:43:08 -0400498 /* Ignore dubious measurements */
499 if (mrtt == 0 || mrtt > TCP_RTT_MAX)
500 goto done;
501
502 tcp_estimate_rtt (tc, mrtt);
503
504done:
505
Florin Corasf1762d62017-09-24 19:43:08 -0400506 /* If we got here something must've been ACKed so make sure boff is 0,
Florin Coras3ec66b02018-08-23 16:27:05 -0700507 * even if mrtt is not valid since we update the rto lower */
Florin Corasf1762d62017-09-24 19:43:08 -0400508 tc->rto_boff = 0;
Florin Coras93992a92017-05-24 18:03:56 -0700509 tcp_update_rto (tc);
Dave Barach68b0fb02017-02-28 15:15:56 -0500510
Florin Coras3af90fc2017-05-03 21:09:42 -0700511 return 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500512}
513
Florin Corasefefc6b2018-11-07 12:49:19 -0800514static void
515tcp_estimate_initial_rtt (tcp_connection_t * tc)
516{
517 u8 thread_index = vlib_num_workers ()? 1 : 0;
518 int mrtt;
519
520 if (tc->rtt_ts)
521 {
522 tc->mrtt_us = tcp_time_now_us (thread_index) - tc->rtt_ts;
Florin Coras222e1f412019-02-16 20:47:32 -0800523 tc->mrtt_us = clib_max (tc->mrtt_us, 0.0001);
Florin Corasefefc6b2018-11-07 12:49:19 -0800524 mrtt = clib_max ((u32) (tc->mrtt_us * THZ), 1);
525 tc->rtt_ts = 0;
526 }
527 else
528 {
529 mrtt = tcp_time_now_w_thread (thread_index) - tc->rcv_opts.tsecr;
Florin Coras4af830c2018-12-04 09:21:36 -0800530 mrtt = clib_max (mrtt, 1);
Florin Coras222e1f412019-02-16 20:47:32 -0800531 /* Due to retransmits we don't know the initial mrtt */
532 if (tc->rto_boff && mrtt > 1 * THZ)
533 mrtt = 1 * THZ;
Florin Corasefefc6b2018-11-07 12:49:19 -0800534 tc->mrtt_us = (f64) mrtt *TCP_TICK;
Florin Corasefefc6b2018-11-07 12:49:19 -0800535 }
536
537 if (mrtt > 0 && mrtt < TCP_RTT_MAX)
538 tcp_estimate_rtt (tc, mrtt);
Florin Coras54ddf432018-12-21 13:54:09 -0800539 tcp_update_rto (tc);
Florin Corasefefc6b2018-11-07 12:49:19 -0800540}
541
Dave Barach68b0fb02017-02-28 15:15:56 -0500542/**
Florin Coras9ece3c02018-11-05 11:06:53 -0800543 * Dequeue bytes for connections that have received acks in last burst
Dave Barach68b0fb02017-02-28 15:15:56 -0500544 */
545static void
Florin Coras9ece3c02018-11-05 11:06:53 -0800546tcp_handle_postponed_dequeues (tcp_worker_ctx_t * wrk)
Dave Barach68b0fb02017-02-28 15:15:56 -0500547{
Florin Coras9ece3c02018-11-05 11:06:53 -0800548 u32 thread_index = wrk->vm->thread_index;
549 u32 *pending_deq_acked;
550 tcp_connection_t *tc;
551 int i;
Florin Coras93992a92017-05-24 18:03:56 -0700552
Florin Coras9ece3c02018-11-05 11:06:53 -0800553 if (!vec_len (wrk->pending_deq_acked))
554 return;
Dave Barach68b0fb02017-02-28 15:15:56 -0500555
Florin Coras9ece3c02018-11-05 11:06:53 -0800556 pending_deq_acked = wrk->pending_deq_acked;
557 for (i = 0; i < vec_len (pending_deq_acked); i++)
558 {
559 tc = tcp_connection_get (pending_deq_acked[i], thread_index);
560 tc->flags &= ~TCP_CONN_DEQ_PENDING;
Florin Coras93992a92017-05-24 18:03:56 -0700561
Florin Corasefefc6b2018-11-07 12:49:19 -0800562 if (PREDICT_FALSE (!tc->burst_acked))
563 continue;
564
Florin Coras9ece3c02018-11-05 11:06:53 -0800565 /* Dequeue the newly ACKed bytes */
Florin Coras31c99552019-03-01 13:00:58 -0800566 session_tx_fifo_dequeue_drop (&tc->connection, tc->burst_acked);
Florin Coras9ece3c02018-11-05 11:06:53 -0800567 tc->burst_acked = 0;
568 tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una);
569
Florin Coras42ceddb2018-12-12 10:56:01 -0800570 if (PREDICT_FALSE (tc->flags & TCP_CONN_PSH_PENDING))
571 {
572 if (seq_leq (tc->psh_seq, tc->snd_una))
573 tc->flags &= ~TCP_CONN_PSH_PENDING;
574 }
575
Florin Coras9ece3c02018-11-05 11:06:53 -0800576 /* If everything has been acked, stop retransmit timer
577 * otherwise update. */
578 tcp_retransmit_timer_update (tc);
Florin Corasefefc6b2018-11-07 12:49:19 -0800579
580 /* If not congested, update pacer based on our new
581 * cwnd estimate */
582 if (!tcp_in_fastrecovery (tc))
583 tcp_connection_tx_pacer_update (tc);
Florin Coras9ece3c02018-11-05 11:06:53 -0800584 }
585 _vec_len (wrk->pending_deq_acked) = 0;
586}
587
588static void
589tcp_program_dequeue (tcp_worker_ctx_t * wrk, tcp_connection_t * tc)
590{
591 if (!(tc->flags & TCP_CONN_DEQ_PENDING))
592 {
593 vec_add1 (wrk->pending_deq_acked, tc->c_c_index);
594 tc->flags |= TCP_CONN_DEQ_PENDING;
595 }
596 tc->burst_acked += tc->bytes_acked + tc->sack_sb.snd_una_adv;
Dave Barach68b0fb02017-02-28 15:15:56 -0500597}
598
Florin Coras6792ec02017-03-13 03:49:51 -0700599/**
Florin Coras93992a92017-05-24 18:03:56 -0700600 * Check if duplicate ack as per RFC5681 Sec. 2
601 */
602static u8
603tcp_ack_is_dupack (tcp_connection_t * tc, vlib_buffer_t * b, u32 prev_snd_wnd,
604 u32 prev_snd_una)
Dave Barach68b0fb02017-02-28 15:15:56 -0500605{
Florin Coras93992a92017-05-24 18:03:56 -0700606 return ((vnet_buffer (b)->tcp.ack_number == prev_snd_una)
Dave Barach68b0fb02017-02-28 15:15:56 -0500607 && seq_gt (tc->snd_una_max, tc->snd_una)
608 && (vnet_buffer (b)->tcp.seq_end == vnet_buffer (b)->tcp.seq_number)
Florin Coras93992a92017-05-24 18:03:56 -0700609 && (prev_snd_wnd == tc->snd_wnd));
610}
611
612/**
613 * Checks if ack is a congestion control event.
614 */
615static u8
616tcp_ack_is_cc_event (tcp_connection_t * tc, vlib_buffer_t * b,
617 u32 prev_snd_wnd, u32 prev_snd_una, u8 * is_dack)
618{
619 /* Check if ack is duplicate. Per RFC 6675, ACKs that SACK new data are
620 * defined to be 'duplicate' */
621 *is_dack = tc->sack_sb.last_sacked_bytes
622 || tcp_ack_is_dupack (tc, b, prev_snd_wnd, prev_snd_una);
623
Dave Barach2c25a622017-06-26 11:35:07 -0400624 return ((*is_dack || tcp_in_cong_recovery (tc)) && !tcp_is_lost_fin (tc));
Dave Barach68b0fb02017-02-28 15:15:56 -0500625}
626
Filip Tehlare275bed2019-03-06 00:06:56 -0800627#ifndef CLIB_MARCH_VARIANT
Florin Coras0dbd5172018-06-25 16:19:34 -0700628static u32
629scoreboard_hole_index (sack_scoreboard_t * sb, sack_scoreboard_hole_t * hole)
630{
631 ASSERT (!pool_is_free_index (sb->holes, hole - sb->holes));
632 return hole - sb->holes;
633}
634
635static u32
636scoreboard_hole_bytes (sack_scoreboard_hole_t * hole)
637{
638 return hole->end - hole->start;
639}
640
641sack_scoreboard_hole_t *
642scoreboard_get_hole (sack_scoreboard_t * sb, u32 index)
643{
644 if (index != TCP_INVALID_SACK_HOLE_INDEX)
645 return pool_elt_at_index (sb->holes, index);
646 return 0;
647}
648
649sack_scoreboard_hole_t *
650scoreboard_next_hole (sack_scoreboard_t * sb, sack_scoreboard_hole_t * hole)
651{
652 if (hole->next != TCP_INVALID_SACK_HOLE_INDEX)
653 return pool_elt_at_index (sb->holes, hole->next);
654 return 0;
655}
656
657sack_scoreboard_hole_t *
658scoreboard_prev_hole (sack_scoreboard_t * sb, sack_scoreboard_hole_t * hole)
659{
660 if (hole->prev != TCP_INVALID_SACK_HOLE_INDEX)
661 return pool_elt_at_index (sb->holes, hole->prev);
662 return 0;
663}
664
665sack_scoreboard_hole_t *
666scoreboard_first_hole (sack_scoreboard_t * sb)
667{
668 if (sb->head != TCP_INVALID_SACK_HOLE_INDEX)
669 return pool_elt_at_index (sb->holes, sb->head);
670 return 0;
671}
672
673sack_scoreboard_hole_t *
674scoreboard_last_hole (sack_scoreboard_t * sb)
675{
676 if (sb->tail != TCP_INVALID_SACK_HOLE_INDEX)
677 return pool_elt_at_index (sb->holes, sb->tail);
678 return 0;
679}
680
681static void
Dave Barach68b0fb02017-02-28 15:15:56 -0500682scoreboard_remove_hole (sack_scoreboard_t * sb, sack_scoreboard_hole_t * hole)
683{
684 sack_scoreboard_hole_t *next, *prev;
685
686 if (hole->next != TCP_INVALID_SACK_HOLE_INDEX)
687 {
688 next = pool_elt_at_index (sb->holes, hole->next);
689 next->prev = hole->prev;
690 }
Florin Coras93992a92017-05-24 18:03:56 -0700691 else
692 {
693 sb->tail = hole->prev;
694 }
Dave Barach68b0fb02017-02-28 15:15:56 -0500695
696 if (hole->prev != TCP_INVALID_SACK_HOLE_INDEX)
697 {
698 prev = pool_elt_at_index (sb->holes, hole->prev);
699 prev->next = hole->next;
700 }
701 else
702 {
703 sb->head = hole->next;
704 }
705
Florin Coras93992a92017-05-24 18:03:56 -0700706 if (scoreboard_hole_index (sb, hole) == sb->cur_rxt_hole)
707 sb->cur_rxt_hole = TCP_INVALID_SACK_HOLE_INDEX;
708
Florin Coras3eb50622017-07-13 01:24:57 -0400709 /* Poison the entry */
710 if (CLIB_DEBUG > 0)
Dave Barachb7b92992018-10-17 10:38:51 -0400711 clib_memset (hole, 0xfe, sizeof (*hole));
Florin Coras3eb50622017-07-13 01:24:57 -0400712
Dave Barach68b0fb02017-02-28 15:15:56 -0500713 pool_put (sb->holes, hole);
714}
715
Florin Coras0dbd5172018-06-25 16:19:34 -0700716static sack_scoreboard_hole_t *
Florin Coras6792ec02017-03-13 03:49:51 -0700717scoreboard_insert_hole (sack_scoreboard_t * sb, u32 prev_index,
Dave Barach68b0fb02017-02-28 15:15:56 -0500718 u32 start, u32 end)
719{
Florin Coras6792ec02017-03-13 03:49:51 -0700720 sack_scoreboard_hole_t *hole, *next, *prev;
Dave Barach68b0fb02017-02-28 15:15:56 -0500721 u32 hole_index;
722
723 pool_get (sb->holes, hole);
Dave Barachb7b92992018-10-17 10:38:51 -0400724 clib_memset (hole, 0, sizeof (*hole));
Dave Barach68b0fb02017-02-28 15:15:56 -0500725
726 hole->start = start;
727 hole->end = end;
Florin Coras3eb50622017-07-13 01:24:57 -0400728 hole_index = scoreboard_hole_index (sb, hole);
Dave Barach68b0fb02017-02-28 15:15:56 -0500729
Florin Coras6792ec02017-03-13 03:49:51 -0700730 prev = scoreboard_get_hole (sb, prev_index);
Dave Barach68b0fb02017-02-28 15:15:56 -0500731 if (prev)
732 {
Florin Corasf03a59a2017-06-09 21:07:32 -0700733 hole->prev = prev_index;
Dave Barach68b0fb02017-02-28 15:15:56 -0500734 hole->next = prev->next;
735
736 if ((next = scoreboard_next_hole (sb, hole)))
737 next->prev = hole_index;
Florin Corasf03a59a2017-06-09 21:07:32 -0700738 else
739 sb->tail = hole_index;
Dave Barach68b0fb02017-02-28 15:15:56 -0500740
741 prev->next = hole_index;
742 }
743 else
744 {
745 sb->head = hole_index;
746 hole->prev = TCP_INVALID_SACK_HOLE_INDEX;
747 hole->next = TCP_INVALID_SACK_HOLE_INDEX;
748 }
749
750 return hole;
751}
Filip Tehlare275bed2019-03-06 00:06:56 -0800752#endif /* CLIB_MARCH_VARIANT */
Dave Barach68b0fb02017-02-28 15:15:56 -0500753
Filip Tehlare275bed2019-03-06 00:06:56 -0800754#ifndef CLIB_MARCH_VARIANT
Florin Coras0dbd5172018-06-25 16:19:34 -0700755static void
Florin Corasf03a59a2017-06-09 21:07:32 -0700756scoreboard_update_bytes (tcp_connection_t * tc, sack_scoreboard_t * sb)
Florin Coras93992a92017-05-24 18:03:56 -0700757{
Florin Corasecbd20b2018-10-17 23:34:54 -0700758 sack_scoreboard_hole_t *left, *right;
Florin Coras93992a92017-05-24 18:03:56 -0700759 u32 bytes = 0, blks = 0;
760
761 sb->lost_bytes = 0;
Florin Corasf03a59a2017-06-09 21:07:32 -0700762 sb->sacked_bytes = 0;
Florin Corasecbd20b2018-10-17 23:34:54 -0700763 left = scoreboard_last_hole (sb);
764 if (!left)
Florin Coras93992a92017-05-24 18:03:56 -0700765 return;
766
Florin Corasecbd20b2018-10-17 23:34:54 -0700767 if (seq_gt (sb->high_sacked, left->end))
Florin Coras93992a92017-05-24 18:03:56 -0700768 {
Florin Corasecbd20b2018-10-17 23:34:54 -0700769 bytes = sb->high_sacked - left->end;
Florin Coras93992a92017-05-24 18:03:56 -0700770 blks = 1;
771 }
772
Florin Coras9f9e9692018-10-19 17:49:00 -0700773 while ((right = left)
774 && bytes < (TCP_DUPACK_THRESHOLD - 1) * tc->snd_mss
775 && blks < TCP_DUPACK_THRESHOLD
776 /* left not updated if above conditions fail */
777 && (left = scoreboard_prev_hole (sb, right)))
Florin Coras93992a92017-05-24 18:03:56 -0700778 {
Florin Corasecbd20b2018-10-17 23:34:54 -0700779 bytes += right->start - left->end;
Florin Coras93992a92017-05-24 18:03:56 -0700780 blks++;
Florin Coras93992a92017-05-24 18:03:56 -0700781 }
782
Florin Coras9f9e9692018-10-19 17:49:00 -0700783 /* left is first lost */
784 if (left)
Florin Coras93992a92017-05-24 18:03:56 -0700785 {
Florin Coras9f9e9692018-10-19 17:49:00 -0700786 do
787 {
788 sb->lost_bytes += scoreboard_hole_bytes (right);
789 left->is_lost = 1;
790 left = scoreboard_prev_hole (sb, right);
791 if (left)
792 bytes += right->start - left->end;
793 }
794 while ((right = left));
Florin Coras93992a92017-05-24 18:03:56 -0700795 }
Florin Coras9f9e9692018-10-19 17:49:00 -0700796
Florin Corasf03a59a2017-06-09 21:07:32 -0700797 sb->sacked_bytes = bytes;
Florin Coras93992a92017-05-24 18:03:56 -0700798}
799
800/**
801 * Figure out the next hole to retransmit
802 *
803 * Follows logic proposed in RFC6675 Sec. 4, NextSeg()
804 */
805sack_scoreboard_hole_t *
806scoreboard_next_rxt_hole (sack_scoreboard_t * sb,
807 sack_scoreboard_hole_t * start,
Florin Coras36ee9f12018-11-02 12:52:10 -0700808 u8 have_unsent, u8 * can_rescue, u8 * snd_limited)
Florin Coras93992a92017-05-24 18:03:56 -0700809{
810 sack_scoreboard_hole_t *hole = 0;
811
812 hole = start ? start : scoreboard_first_hole (sb);
813 while (hole && seq_leq (hole->end, sb->high_rxt) && hole->is_lost)
814 hole = scoreboard_next_hole (sb, hole);
815
816 /* Nothing, return */
817 if (!hole)
818 {
819 sb->cur_rxt_hole = TCP_INVALID_SACK_HOLE_INDEX;
820 return 0;
821 }
822
823 /* Rule (1): if higher than rxt, less than high_sacked and lost */
824 if (hole->is_lost && seq_lt (hole->start, sb->high_sacked))
825 {
826 sb->cur_rxt_hole = scoreboard_hole_index (sb, hole);
827 }
828 else
829 {
Florin Coras36ee9f12018-11-02 12:52:10 -0700830 /* Rule (2): available unsent data */
831 if (have_unsent)
Florin Coras93992a92017-05-24 18:03:56 -0700832 {
Florin Coras93992a92017-05-24 18:03:56 -0700833 sb->cur_rxt_hole = TCP_INVALID_SACK_HOLE_INDEX;
Florin Coras36ee9f12018-11-02 12:52:10 -0700834 return 0;
Florin Coras93992a92017-05-24 18:03:56 -0700835 }
836 /* Rule (3): if hole not lost */
837 else if (seq_lt (hole->start, sb->high_sacked))
838 {
Florin Corasbf4d5ce2018-10-19 16:26:24 -0700839 *snd_limited = 0;
Florin Coras93992a92017-05-24 18:03:56 -0700840 sb->cur_rxt_hole = scoreboard_hole_index (sb, hole);
841 }
842 /* Rule (4): if hole beyond high_sacked */
843 else
844 {
845 ASSERT (seq_geq (hole->start, sb->high_sacked));
846 *snd_limited = 1;
847 *can_rescue = 1;
848 /* HighRxt MUST NOT be updated */
849 return 0;
850 }
851 }
852
853 if (hole && seq_lt (sb->high_rxt, hole->start))
854 sb->high_rxt = hole->start;
855
856 return hole;
857}
Filip Tehlare275bed2019-03-06 00:06:56 -0800858#endif /* CLIB_MARCH_VARIANT */
Florin Coras93992a92017-05-24 18:03:56 -0700859
Florin Coras0dbd5172018-06-25 16:19:34 -0700860static void
Florin Coras36ee9f12018-11-02 12:52:10 -0700861scoreboard_init_high_rxt (sack_scoreboard_t * sb, u32 snd_una)
Florin Coras93992a92017-05-24 18:03:56 -0700862{
863 sack_scoreboard_hole_t *hole;
864 hole = scoreboard_first_hole (sb);
Florin Coras3eb50622017-07-13 01:24:57 -0400865 if (hole)
866 {
Florin Coras36ee9f12018-11-02 12:52:10 -0700867 snd_una = seq_gt (snd_una, hole->start) ? snd_una : hole->start;
Florin Coras3eb50622017-07-13 01:24:57 -0400868 sb->cur_rxt_hole = sb->head;
869 }
Florin Coras36ee9f12018-11-02 12:52:10 -0700870 sb->high_rxt = snd_una;
871 sb->rescue_rxt = snd_una - 1;
Florin Coras3eb50622017-07-13 01:24:57 -0400872}
873
Filip Tehlare275bed2019-03-06 00:06:56 -0800874#ifndef CLIB_MARCH_VARIANT
Florin Coras0dbd5172018-06-25 16:19:34 -0700875void
876scoreboard_init (sack_scoreboard_t * sb)
877{
878 sb->head = TCP_INVALID_SACK_HOLE_INDEX;
879 sb->tail = TCP_INVALID_SACK_HOLE_INDEX;
880 sb->cur_rxt_hole = TCP_INVALID_SACK_HOLE_INDEX;
881}
882
883void
884scoreboard_clear (sack_scoreboard_t * sb)
885{
886 sack_scoreboard_hole_t *hole;
887 while ((hole = scoreboard_first_hole (sb)))
888 {
889 scoreboard_remove_hole (sb, hole);
890 }
891 ASSERT (sb->head == sb->tail && sb->head == TCP_INVALID_SACK_HOLE_INDEX);
892 ASSERT (pool_elts (sb->holes) == 0);
893 sb->sacked_bytes = 0;
894 sb->last_sacked_bytes = 0;
895 sb->last_bytes_delivered = 0;
896 sb->snd_una_adv = 0;
897 sb->high_sacked = 0;
898 sb->high_rxt = 0;
899 sb->lost_bytes = 0;
900 sb->cur_rxt_hole = TCP_INVALID_SACK_HOLE_INDEX;
901}
Filip Tehlare275bed2019-03-06 00:06:56 -0800902#endif /* CLIB_MARCH_VARIANT */
Florin Coras0dbd5172018-06-25 16:19:34 -0700903
Florin Coras3eb50622017-07-13 01:24:57 -0400904/**
905 * Test that scoreboard is sane after recovery
906 *
907 * Returns 1 if scoreboard is empty or if first hole beyond
908 * snd_una.
909 */
Florin Coras0dbd5172018-06-25 16:19:34 -0700910static u8
Florin Coras3eb50622017-07-13 01:24:57 -0400911tcp_scoreboard_is_sane_post_recovery (tcp_connection_t * tc)
912{
913 sack_scoreboard_hole_t *hole;
914 hole = scoreboard_first_hole (&tc->sack_sb);
Florin Corasecbd20b2018-10-17 23:34:54 -0700915 return (!hole || (seq_geq (hole->start, tc->snd_una)
916 && seq_lt (hole->end, tc->snd_una_max)));
Florin Coras93992a92017-05-24 18:03:56 -0700917}
918
Filip Tehlare275bed2019-03-06 00:06:56 -0800919#ifndef CLIB_MARCH_VARIANT
Florin Coras93992a92017-05-24 18:03:56 -0700920void
Dave Barach68b0fb02017-02-28 15:15:56 -0500921tcp_rcv_sacks (tcp_connection_t * tc, u32 ack)
922{
923 sack_scoreboard_t *sb = &tc->sack_sb;
924 sack_block_t *blk, tmp;
Florin Corasf03a59a2017-06-09 21:07:32 -0700925 sack_scoreboard_hole_t *hole, *next_hole, *last_hole;
Florin Coras93992a92017-05-24 18:03:56 -0700926 u32 blk_index = 0, old_sacked_bytes, hole_index;
Dave Barach68b0fb02017-02-28 15:15:56 -0500927 int i, j;
928
Florin Coras6792ec02017-03-13 03:49:51 -0700929 sb->last_sacked_bytes = 0;
Florin Coras93992a92017-05-24 18:03:56 -0700930 sb->last_bytes_delivered = 0;
Florin Coras352c2c42018-06-26 01:22:41 -0700931 sb->snd_una_adv = 0;
Florin Coras6792ec02017-03-13 03:49:51 -0700932
Florin Coras93992a92017-05-24 18:03:56 -0700933 if (!tcp_opts_sack (&tc->rcv_opts)
934 && sb->head == TCP_INVALID_SACK_HOLE_INDEX)
Dave Barach68b0fb02017-02-28 15:15:56 -0500935 return;
936
Florin Coras352c2c42018-06-26 01:22:41 -0700937 old_sacked_bytes = sb->sacked_bytes;
938
Dave Barach68b0fb02017-02-28 15:15:56 -0500939 /* Remove invalid blocks */
Florin Coras93992a92017-05-24 18:03:56 -0700940 blk = tc->rcv_opts.sacks;
941 while (blk < vec_end (tc->rcv_opts.sacks))
Florin Coras6792ec02017-03-13 03:49:51 -0700942 {
943 if (seq_lt (blk->start, blk->end)
944 && seq_gt (blk->start, tc->snd_una)
Florin Corasab86f862018-12-06 18:24:19 -0800945 && seq_gt (blk->start, ack)
946 && seq_lt (blk->start, tc->snd_una_max)
947 && seq_leq (blk->end, tc->snd_una_max))
Florin Coras6792ec02017-03-13 03:49:51 -0700948 {
949 blk++;
950 continue;
951 }
Florin Coras93992a92017-05-24 18:03:56 -0700952 vec_del1 (tc->rcv_opts.sacks, blk - tc->rcv_opts.sacks);
Florin Coras6792ec02017-03-13 03:49:51 -0700953 }
Dave Barach68b0fb02017-02-28 15:15:56 -0500954
955 /* Add block for cumulative ack */
956 if (seq_gt (ack, tc->snd_una))
957 {
958 tmp.start = tc->snd_una;
959 tmp.end = ack;
Florin Coras93992a92017-05-24 18:03:56 -0700960 vec_add1 (tc->rcv_opts.sacks, tmp);
Dave Barach68b0fb02017-02-28 15:15:56 -0500961 }
962
Florin Coras93992a92017-05-24 18:03:56 -0700963 if (vec_len (tc->rcv_opts.sacks) == 0)
Dave Barach68b0fb02017-02-28 15:15:56 -0500964 return;
965
Florin Coras3eb50622017-07-13 01:24:57 -0400966 tcp_scoreboard_trace_add (tc, ack);
967
Dave Barach68b0fb02017-02-28 15:15:56 -0500968 /* Make sure blocks are ordered */
Florin Coras93992a92017-05-24 18:03:56 -0700969 for (i = 0; i < vec_len (tc->rcv_opts.sacks); i++)
970 for (j = i + 1; j < vec_len (tc->rcv_opts.sacks); j++)
971 if (seq_lt (tc->rcv_opts.sacks[j].start, tc->rcv_opts.sacks[i].start))
Dave Barach68b0fb02017-02-28 15:15:56 -0500972 {
Florin Coras93992a92017-05-24 18:03:56 -0700973 tmp = tc->rcv_opts.sacks[i];
974 tc->rcv_opts.sacks[i] = tc->rcv_opts.sacks[j];
975 tc->rcv_opts.sacks[j] = tmp;
Dave Barach68b0fb02017-02-28 15:15:56 -0500976 }
977
Dave Barach68b0fb02017-02-28 15:15:56 -0500978 if (sb->head == TCP_INVALID_SACK_HOLE_INDEX)
979 {
Florin Coras6792ec02017-03-13 03:49:51 -0700980 /* If no holes, insert the first that covers all outstanding bytes */
981 last_hole = scoreboard_insert_hole (sb, TCP_INVALID_SACK_HOLE_INDEX,
982 tc->snd_una, tc->snd_una_max);
983 sb->tail = scoreboard_hole_index (sb, last_hole);
Florin Coras93992a92017-05-24 18:03:56 -0700984 tmp = tc->rcv_opts.sacks[vec_len (tc->rcv_opts.sacks) - 1];
985 sb->high_sacked = tmp.end;
Florin Coras6792ec02017-03-13 03:49:51 -0700986 }
987 else
988 {
989 /* If we have holes but snd_una_max is beyond the last hole, update
990 * last hole end */
Florin Coras93992a92017-05-24 18:03:56 -0700991 tmp = tc->rcv_opts.sacks[vec_len (tc->rcv_opts.sacks) - 1];
Florin Coras6792ec02017-03-13 03:49:51 -0700992 last_hole = scoreboard_last_hole (sb);
Dave Barach2c25a622017-06-26 11:35:07 -0400993 if (seq_gt (tc->snd_una_max, last_hole->end))
994 {
995 if (seq_geq (last_hole->start, sb->high_sacked))
996 {
997 last_hole->end = tc->snd_una_max;
998 }
999 /* New hole after high sacked block */
1000 else if (seq_lt (sb->high_sacked, tc->snd_una_max))
1001 {
1002 scoreboard_insert_hole (sb, sb->tail, sb->high_sacked,
1003 tc->snd_una_max);
1004 }
1005 }
1006 /* Keep track of max byte sacked for when the last hole
Florin Corasf03a59a2017-06-09 21:07:32 -07001007 * is acked */
1008 if (seq_gt (tmp.end, sb->high_sacked))
1009 sb->high_sacked = tmp.end;
Dave Barach68b0fb02017-02-28 15:15:56 -05001010 }
1011
1012 /* Walk the holes with the SACK blocks */
1013 hole = pool_elt_at_index (sb->holes, sb->head);
Florin Coras93992a92017-05-24 18:03:56 -07001014 while (hole && blk_index < vec_len (tc->rcv_opts.sacks))
Dave Barach68b0fb02017-02-28 15:15:56 -05001015 {
Florin Coras93992a92017-05-24 18:03:56 -07001016 blk = &tc->rcv_opts.sacks[blk_index];
Dave Barach68b0fb02017-02-28 15:15:56 -05001017 if (seq_leq (blk->start, hole->start))
1018 {
1019 /* Block covers hole. Remove hole */
1020 if (seq_geq (blk->end, hole->end))
1021 {
1022 next_hole = scoreboard_next_hole (sb, hole);
1023
Florin Corasf03a59a2017-06-09 21:07:32 -07001024 /* Byte accounting: snd_una needs to be advanced */
1025 if (blk->end == ack)
Dave Barach68b0fb02017-02-28 15:15:56 -05001026 {
Florin Corasf03a59a2017-06-09 21:07:32 -07001027 if (next_hole)
Florin Coras06d11012017-05-17 14:21:51 -07001028 {
Florin Corasf03a59a2017-06-09 21:07:32 -07001029 if (seq_lt (ack, next_hole->start))
1030 sb->snd_una_adv = next_hole->start - ack;
1031 sb->last_bytes_delivered +=
1032 next_hole->start - hole->end;
Florin Coras06d11012017-05-17 14:21:51 -07001033 }
Florin Coras3eb50622017-07-13 01:24:57 -04001034 else
Florin Coras06d11012017-05-17 14:21:51 -07001035 {
Dave Barach2c25a622017-06-26 11:35:07 -04001036 ASSERT (seq_geq (sb->high_sacked, ack));
Florin Coras93992a92017-05-24 18:03:56 -07001037 sb->snd_una_adv = sb->high_sacked - ack;
Florin Corasf03a59a2017-06-09 21:07:32 -07001038 sb->last_bytes_delivered += sb->high_sacked - hole->end;
Florin Coras06d11012017-05-17 14:21:51 -07001039 }
1040 }
1041
Dave Barach68b0fb02017-02-28 15:15:56 -05001042 scoreboard_remove_hole (sb, hole);
1043 hole = next_hole;
1044 }
Florin Coras6792ec02017-03-13 03:49:51 -07001045 /* Partial 'head' overlap */
Dave Barach68b0fb02017-02-28 15:15:56 -05001046 else
1047 {
Florin Coras6792ec02017-03-13 03:49:51 -07001048 if (seq_gt (blk->end, hole->start))
1049 {
Florin Coras6792ec02017-03-13 03:49:51 -07001050 hole->start = blk->end;
1051 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001052 blk_index++;
1053 }
1054 }
1055 else
1056 {
1057 /* Hole must be split */
Florin Coras6792ec02017-03-13 03:49:51 -07001058 if (seq_lt (blk->end, hole->end))
Dave Barach68b0fb02017-02-28 15:15:56 -05001059 {
Florin Coras6792ec02017-03-13 03:49:51 -07001060 hole_index = scoreboard_hole_index (sb, hole);
Florin Coras3eb50622017-07-13 01:24:57 -04001061 next_hole = scoreboard_insert_hole (sb, hole_index, blk->end,
1062 hole->end);
Florin Coras6792ec02017-03-13 03:49:51 -07001063
1064 /* Pool might've moved */
1065 hole = scoreboard_get_hole (sb, hole_index);
1066 hole->end = blk->start;
Dave Barach68b0fb02017-02-28 15:15:56 -05001067 blk_index++;
Florin Coras3eb50622017-07-13 01:24:57 -04001068 ASSERT (hole->next == scoreboard_hole_index (sb, next_hole));
Dave Barach68b0fb02017-02-28 15:15:56 -05001069 }
Florin Corasf03a59a2017-06-09 21:07:32 -07001070 else if (seq_lt (blk->start, hole->end))
Dave Barach68b0fb02017-02-28 15:15:56 -05001071 {
Florin Coras6792ec02017-03-13 03:49:51 -07001072 hole->end = blk->start;
Dave Barach68b0fb02017-02-28 15:15:56 -05001073 }
Florin Coras93992a92017-05-24 18:03:56 -07001074 hole = scoreboard_next_hole (sb, hole);
Dave Barach68b0fb02017-02-28 15:15:56 -05001075 }
1076 }
Florin Coras6792ec02017-03-13 03:49:51 -07001077
Florin Corasecbd20b2018-10-17 23:34:54 -07001078 if (pool_elts (sb->holes) == 1)
1079 {
1080 hole = scoreboard_first_hole (sb);
1081 if (hole->start == ack + sb->snd_una_adv
1082 && hole->end == tc->snd_una_max)
1083 scoreboard_remove_hole (sb, hole);
1084 }
1085
Florin Corasf03a59a2017-06-09 21:07:32 -07001086 scoreboard_update_bytes (tc, sb);
1087 sb->last_sacked_bytes = sb->sacked_bytes
1088 - (old_sacked_bytes - sb->last_bytes_delivered);
Florin Corasca1c8f32018-05-23 21:01:30 -07001089 ASSERT (sb->last_sacked_bytes <= sb->sacked_bytes || tcp_in_recovery (tc));
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001090 ASSERT (sb->sacked_bytes == 0 || tcp_in_recovery (tc)
Florin Corasf03a59a2017-06-09 21:07:32 -07001091 || sb->sacked_bytes < tc->snd_una_max - seq_max (tc->snd_una, ack));
1092 ASSERT (sb->last_sacked_bytes + sb->lost_bytes <= tc->snd_una_max
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001093 - seq_max (tc->snd_una, ack) || tcp_in_recovery (tc));
Dave Barach2c25a622017-06-26 11:35:07 -04001094 ASSERT (sb->head == TCP_INVALID_SACK_HOLE_INDEX || tcp_in_recovery (tc)
1095 || sb->holes[sb->head].start == ack + sb->snd_una_adv);
Florin Corasca1c8f32018-05-23 21:01:30 -07001096 TCP_EVT_DBG (TCP_EVT_CC_SCOREBOARD, tc);
Dave Barach68b0fb02017-02-28 15:15:56 -05001097}
Filip Tehlare275bed2019-03-06 00:06:56 -08001098#endif /* CLIB_MARCH_VARIANT */
Dave Barach68b0fb02017-02-28 15:15:56 -05001099
Florin Coras93992a92017-05-24 18:03:56 -07001100/**
1101 * Try to update snd_wnd based on feedback received from peer.
Dave Barach68b0fb02017-02-28 15:15:56 -05001102 *
Florin Coras93992a92017-05-24 18:03:56 -07001103 * If successful, and new window is 'effectively' 0, activate persist
1104 * timer.
1105 */
Dave Barach68b0fb02017-02-28 15:15:56 -05001106static void
1107tcp_update_snd_wnd (tcp_connection_t * tc, u32 seq, u32 ack, u32 snd_wnd)
1108{
Florin Coras93992a92017-05-24 18:03:56 -07001109 /* If (SND.WL1 < SEG.SEQ or (SND.WL1 = SEG.SEQ and SND.WL2 =< SEG.ACK)), set
1110 * SND.WND <- SEG.WND, set SND.WL1 <- SEG.SEQ, and set SND.WL2 <- SEG.ACK */
Florin Coras6792ec02017-03-13 03:49:51 -07001111 if (seq_lt (tc->snd_wl1, seq)
1112 || (tc->snd_wl1 == seq && seq_leq (tc->snd_wl2, ack)))
Dave Barach68b0fb02017-02-28 15:15:56 -05001113 {
1114 tc->snd_wnd = snd_wnd;
1115 tc->snd_wl1 = seq;
1116 tc->snd_wl2 = ack;
Florin Coras6792ec02017-03-13 03:49:51 -07001117 TCP_EVT_DBG (TCP_EVT_SND_WND, tc);
Florin Coras3e350af2017-03-30 02:54:28 -07001118
Florin Coras9ece3c02018-11-05 11:06:53 -08001119 if (PREDICT_FALSE (tc->snd_wnd < tc->snd_mss))
Florin Corasbb292f42017-05-19 09:49:19 -07001120 {
Florin Coras93992a92017-05-24 18:03:56 -07001121 /* Set persist timer if not set and we just got 0 wnd */
1122 if (!tcp_timer_is_active (tc, TCP_TIMER_PERSIST)
1123 && !tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT))
Florin Corasbb292f42017-05-19 09:49:19 -07001124 tcp_persist_timer_set (tc);
1125 }
Florin Coras3e350af2017-03-30 02:54:28 -07001126 else
Florin Coras93992a92017-05-24 18:03:56 -07001127 {
1128 tcp_persist_timer_reset (tc);
Florin Coras9ece3c02018-11-05 11:06:53 -08001129 if (PREDICT_FALSE (!tcp_in_recovery (tc) && tc->rto_boff > 0))
Florin Coras93992a92017-05-24 18:03:56 -07001130 {
1131 tc->rto_boff = 0;
1132 tcp_update_rto (tc);
1133 }
1134 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001135 }
1136}
1137
Filip Tehlare275bed2019-03-06 00:06:56 -08001138#ifndef CLIB_MARCH_VARIANT
Florin Corasd2aab832018-05-22 11:39:59 -07001139/**
1140 * Init loss recovery/fast recovery.
1141 *
1142 * Triggered by dup acks as opposed to timer timeout. Note that cwnd is
1143 * updated in @ref tcp_cc_handle_event after fast retransmit
1144 */
Florin Coras6792ec02017-03-13 03:49:51 -07001145void
Florin Coras93992a92017-05-24 18:03:56 -07001146tcp_cc_init_congestion (tcp_connection_t * tc)
Dave Barach68b0fb02017-02-28 15:15:56 -05001147{
Florin Coras93992a92017-05-24 18:03:56 -07001148 tcp_fastrecovery_on (tc);
1149 tc->snd_congestion = tc->snd_una_max;
Florin Coras62166002018-04-18 16:40:55 -07001150 tc->cwnd_acc_bytes = 0;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001151 tc->snd_rxt_bytes = 0;
1152 tc->prev_ssthresh = tc->ssthresh;
1153 tc->prev_cwnd = tc->cwnd;
Dave Barach68b0fb02017-02-28 15:15:56 -05001154 tc->cc_algo->congestion (tc);
Florin Coras6792ec02017-03-13 03:49:51 -07001155 TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 4);
Dave Barach68b0fb02017-02-28 15:15:56 -05001156}
Filip Tehlare275bed2019-03-06 00:06:56 -08001157#endif /* CLIB_MARCH_VARIANT */
Dave Barach68b0fb02017-02-28 15:15:56 -05001158
Florin Coras93992a92017-05-24 18:03:56 -07001159static void
1160tcp_cc_recovery_exit (tcp_connection_t * tc)
Dave Barach68b0fb02017-02-28 15:15:56 -05001161{
Florin Coras93992a92017-05-24 18:03:56 -07001162 tc->rto_boff = 0;
Florin Corasf1762d62017-09-24 19:43:08 -04001163 tcp_update_rto (tc);
Florin Coras93992a92017-05-24 18:03:56 -07001164 tc->snd_rxt_ts = 0;
Florin Coras84275e92017-09-26 12:30:40 -04001165 tc->snd_nxt = tc->snd_una_max;
Florin Corasefefc6b2018-11-07 12:49:19 -08001166 tc->rtt_ts = 0;
Florin Coras93992a92017-05-24 18:03:56 -07001167 tcp_recovery_off (tc);
Florin Corasf1762d62017-09-24 19:43:08 -04001168 TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 3);
Florin Coras93992a92017-05-24 18:03:56 -07001169}
Florin Coras3e350af2017-03-30 02:54:28 -07001170
Filip Tehlare275bed2019-03-06 00:06:56 -08001171#ifndef CLIB_MARCH_VARIANT
Florin Coras93992a92017-05-24 18:03:56 -07001172void
1173tcp_cc_fastrecovery_exit (tcp_connection_t * tc)
1174{
Florin Coras6792ec02017-03-13 03:49:51 -07001175 tc->cc_algo->recovered (tc);
Florin Coras93992a92017-05-24 18:03:56 -07001176 tc->snd_rxt_bytes = 0;
Florin Coras6792ec02017-03-13 03:49:51 -07001177 tc->rcv_dupacks = 0;
Florin Coras84275e92017-09-26 12:30:40 -04001178 tc->snd_nxt = tc->snd_una_max;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001179 tc->snd_rxt_bytes = 0;
Florin Corasefefc6b2018-11-07 12:49:19 -08001180 tc->rtt_ts = 0;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001181
Florin Coras93992a92017-05-24 18:03:56 -07001182 tcp_fastrecovery_off (tc);
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001183 tcp_fastrecovery_first_off (tc);
Florin Corasd67f1122018-05-21 17:47:40 -07001184
Florin Corasf1762d62017-09-24 19:43:08 -04001185 TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 3);
Dave Barach68b0fb02017-02-28 15:15:56 -05001186}
Filip Tehlare275bed2019-03-06 00:06:56 -08001187#endif /* CLIB_MARCH_VARIANT */
Dave Barach68b0fb02017-02-28 15:15:56 -05001188
1189static void
Florin Coras93992a92017-05-24 18:03:56 -07001190tcp_cc_congestion_undo (tcp_connection_t * tc)
Dave Barach68b0fb02017-02-28 15:15:56 -05001191{
Florin Coras93992a92017-05-24 18:03:56 -07001192 tc->cwnd = tc->prev_cwnd;
1193 tc->ssthresh = tc->prev_ssthresh;
1194 tc->snd_nxt = tc->snd_una_max;
1195 tc->rcv_dupacks = 0;
1196 if (tcp_in_recovery (tc))
1197 tcp_cc_recovery_exit (tc);
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001198 else if (tcp_in_fastrecovery (tc))
1199 tcp_cc_fastrecovery_exit (tc);
Florin Coras93992a92017-05-24 18:03:56 -07001200 ASSERT (tc->rto_boff == 0);
Florin Corasf1762d62017-09-24 19:43:08 -04001201 TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 5);
Florin Coras93992a92017-05-24 18:03:56 -07001202}
Dave Barach68b0fb02017-02-28 15:15:56 -05001203
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001204static inline u8
1205tcp_cc_is_spurious_timeout_rxt (tcp_connection_t * tc)
Florin Coras93992a92017-05-24 18:03:56 -07001206{
Florin Corasf1762d62017-09-24 19:43:08 -04001207 return (tcp_in_recovery (tc) && tc->rto_boff == 1
Dave Barach2c25a622017-06-26 11:35:07 -04001208 && tc->snd_rxt_ts
Florin Coras93992a92017-05-24 18:03:56 -07001209 && tcp_opts_tstamp (&tc->rcv_opts)
1210 && timestamp_lt (tc->rcv_opts.tsecr, tc->snd_rxt_ts));
1211}
1212
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001213static inline u8
1214tcp_cc_is_spurious_fast_rxt (tcp_connection_t * tc)
1215{
1216 return (tcp_in_fastrecovery (tc)
1217 && tc->cwnd > tc->ssthresh + 3 * tc->snd_mss);
1218}
1219
1220static u8
1221tcp_cc_is_spurious_retransmit (tcp_connection_t * tc)
1222{
1223 return (tcp_cc_is_spurious_timeout_rxt (tc)
1224 || tcp_cc_is_spurious_fast_rxt (tc));
1225}
1226
Florin Coras0dbd5172018-06-25 16:19:34 -07001227static int
Florin Coras93992a92017-05-24 18:03:56 -07001228tcp_cc_recover (tcp_connection_t * tc)
1229{
1230 ASSERT (tcp_in_cong_recovery (tc));
1231 if (tcp_cc_is_spurious_retransmit (tc))
Dave Barach68b0fb02017-02-28 15:15:56 -05001232 {
Florin Coras93992a92017-05-24 18:03:56 -07001233 tcp_cc_congestion_undo (tc);
1234 return 1;
1235 }
1236
1237 if (tcp_in_recovery (tc))
1238 tcp_cc_recovery_exit (tc);
1239 else if (tcp_in_fastrecovery (tc))
1240 tcp_cc_fastrecovery_exit (tc);
1241
1242 ASSERT (tc->rto_boff == 0);
1243 ASSERT (!tcp_in_cong_recovery (tc));
Florin Coras3eb50622017-07-13 01:24:57 -04001244 ASSERT (tcp_scoreboard_is_sane_post_recovery (tc));
Florin Coras93992a92017-05-24 18:03:56 -07001245 return 0;
1246}
1247
1248static void
1249tcp_cc_update (tcp_connection_t * tc, vlib_buffer_t * b)
1250{
Florin Coras3eb50622017-07-13 01:24:57 -04001251 ASSERT (!tcp_in_cong_recovery (tc) || tcp_is_lost_fin (tc));
Florin Coras93992a92017-05-24 18:03:56 -07001252
1253 /* Congestion avoidance */
Florin Corasd67f1122018-05-21 17:47:40 -07001254 tcp_cc_rcv_ack (tc);
Florin Coras93992a92017-05-24 18:03:56 -07001255
1256 /* If a cumulative ack, make sure dupacks is 0 */
1257 tc->rcv_dupacks = 0;
1258
1259 /* When dupacks hits the threshold we only enter fast retransmit if
1260 * cumulative ack covers more than snd_congestion. Should snd_una
1261 * wrap this test may fail under otherwise valid circumstances.
1262 * Therefore, proactively update snd_congestion when wrap detected. */
1263 if (PREDICT_FALSE
1264 (seq_leq (tc->snd_congestion, tc->snd_una - tc->bytes_acked)
1265 && seq_gt (tc->snd_congestion, tc->snd_una)))
1266 tc->snd_congestion = tc->snd_una - 1;
1267}
1268
1269static u8
1270tcp_should_fastrecover_sack (tcp_connection_t * tc)
1271{
1272 return (TCP_DUPACK_THRESHOLD - 1) * tc->snd_mss < tc->sack_sb.sacked_bytes;
1273}
1274
1275static u8
1276tcp_should_fastrecover (tcp_connection_t * tc)
1277{
1278 return (tc->rcv_dupacks == TCP_DUPACK_THRESHOLD
1279 || tcp_should_fastrecover_sack (tc));
1280}
1281
Filip Tehlare275bed2019-03-06 00:06:56 -08001282#ifndef CLIB_MARCH_VARIANT
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001283void
Florin Corasbe72ae62018-11-01 11:23:03 -07001284tcp_program_fastretransmit (tcp_worker_ctx_t * wrk, tcp_connection_t * tc)
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001285{
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001286 if (!(tc->flags & TCP_CONN_FRXT_PENDING))
1287 {
1288 vec_add1 (wrk->pending_fast_rxt, tc->c_c_index);
1289 tc->flags |= TCP_CONN_FRXT_PENDING;
1290 }
1291}
1292
1293void
Florin Corasbe72ae62018-11-01 11:23:03 -07001294tcp_do_fastretransmits (tcp_worker_ctx_t * wrk)
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001295{
Florin Corasbe72ae62018-11-01 11:23:03 -07001296 u32 *ongoing_fast_rxt, burst_bytes, sent_bytes, thread_index;
Florin Corase55a6d72018-10-31 23:09:22 -07001297 u32 max_burst_size, burst_size, n_segs = 0, n_segs_now;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001298 tcp_connection_t *tc;
Florin Corase55a6d72018-10-31 23:09:22 -07001299 u64 last_cpu_time;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001300 int i;
1301
Florin Corase55a6d72018-10-31 23:09:22 -07001302 if (vec_len (wrk->pending_fast_rxt) == 0
1303 && vec_len (wrk->postponed_fast_rxt) == 0)
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001304 return;
1305
Florin Corasbe72ae62018-11-01 11:23:03 -07001306 thread_index = wrk->vm->thread_index;
Florin Corase55a6d72018-10-31 23:09:22 -07001307 last_cpu_time = wrk->vm->clib_time.last_cpu_time;
1308 ongoing_fast_rxt = wrk->ongoing_fast_rxt;
1309 vec_append (ongoing_fast_rxt, wrk->postponed_fast_rxt);
1310 vec_append (ongoing_fast_rxt, wrk->pending_fast_rxt);
1311
1312 _vec_len (wrk->postponed_fast_rxt) = 0;
1313 _vec_len (wrk->pending_fast_rxt) = 0;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001314
Florin Coras776f3d82018-11-02 08:23:58 -07001315 max_burst_size = VLIB_FRAME_SIZE / vec_len (ongoing_fast_rxt);
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001316 max_burst_size = clib_max (max_burst_size, 1);
1317
Florin Corase55a6d72018-10-31 23:09:22 -07001318 for (i = 0; i < vec_len (ongoing_fast_rxt); i++)
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001319 {
Florin Coras47a9c652019-03-12 20:37:09 -07001320 tc = tcp_connection_get (ongoing_fast_rxt[i], thread_index);
1321 if (!tcp_in_fastrecovery (tc))
1322 {
1323 tc->flags &= ~TCP_CONN_FRXT_PENDING;
1324 continue;
1325 }
1326
Florin Corase55a6d72018-10-31 23:09:22 -07001327 if (n_segs >= VLIB_FRAME_SIZE)
1328 {
1329 vec_add1 (wrk->postponed_fast_rxt, ongoing_fast_rxt[i]);
1330 continue;
1331 }
1332
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001333 tc->flags &= ~TCP_CONN_FRXT_PENDING;
Florin Corase55a6d72018-10-31 23:09:22 -07001334 burst_size = clib_min (max_burst_size, VLIB_FRAME_SIZE - n_segs);
1335 burst_bytes = transport_connection_tx_pacer_burst (&tc->connection,
1336 last_cpu_time);
1337 burst_size = clib_min (burst_size, burst_bytes / tc->snd_mss);
1338 if (!burst_size)
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001339 {
Florin Corasbe72ae62018-11-01 11:23:03 -07001340 tcp_program_fastretransmit (wrk, tc);
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001341 continue;
1342 }
1343
Florin Corasbe72ae62018-11-01 11:23:03 -07001344 n_segs_now = tcp_fast_retransmit (wrk, tc, burst_size);
Florin Corase55a6d72018-10-31 23:09:22 -07001345 sent_bytes = clib_min (n_segs_now * tc->snd_mss, burst_bytes);
1346 transport_connection_tx_pacer_update_bytes (&tc->connection,
1347 sent_bytes);
Florin Corase55a6d72018-10-31 23:09:22 -07001348 n_segs += n_segs_now;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001349 }
Florin Corase55a6d72018-10-31 23:09:22 -07001350 _vec_len (ongoing_fast_rxt) = 0;
1351 wrk->ongoing_fast_rxt = ongoing_fast_rxt;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001352}
Filip Tehlare275bed2019-03-06 00:06:56 -08001353#endif /* CLIB_MARCH_VARIANT */
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001354
Florin Corasf03a59a2017-06-09 21:07:32 -07001355/**
1356 * One function to rule them all ... and in the darkness bind them
1357 */
Florin Coras93992a92017-05-24 18:03:56 -07001358static void
1359tcp_cc_handle_event (tcp_connection_t * tc, u32 is_dack)
1360{
Florin Corasf03a59a2017-06-09 21:07:32 -07001361 u32 rxt_delivered;
1362
Florin Corasca1c8f32018-05-23 21:01:30 -07001363 if (tcp_in_fastrecovery (tc) && tcp_opts_sack_permitted (&tc->rcv_opts))
1364 {
1365 if (tc->bytes_acked)
1366 goto partial_ack;
Florin Corasbe72ae62018-11-01 11:23:03 -07001367 tcp_program_fastretransmit (tcp_get_worker (tc->c_thread_index), tc);
Florin Corasca1c8f32018-05-23 21:01:30 -07001368 return;
1369 }
Florin Coras93992a92017-05-24 18:03:56 -07001370 /*
1371 * Duplicate ACK. Check if we should enter fast recovery, or if already in
1372 * it account for the bytes that left the network.
1373 */
Florin Corasca1c8f32018-05-23 21:01:30 -07001374 else if (is_dack && !tcp_in_recovery (tc))
Florin Coras93992a92017-05-24 18:03:56 -07001375 {
Florin Corasd2aab832018-05-22 11:39:59 -07001376 TCP_EVT_DBG (TCP_EVT_DUPACK_RCVD, tc, 1);
Florin Coras93992a92017-05-24 18:03:56 -07001377 ASSERT (tc->snd_una != tc->snd_una_max
1378 || tc->sack_sb.last_sacked_bytes);
Dave Barach2c25a622017-06-26 11:35:07 -04001379
Florin Coras93992a92017-05-24 18:03:56 -07001380 tc->rcv_dupacks++;
1381
Florin Corasd2aab832018-05-22 11:39:59 -07001382 /* Pure duplicate ack. If some data got acked, it's handled lower */
Florin Coras93992a92017-05-24 18:03:56 -07001383 if (tc->rcv_dupacks > TCP_DUPACK_THRESHOLD && !tc->bytes_acked)
Dave Barach68b0fb02017-02-28 15:15:56 -05001384 {
Florin Coras93992a92017-05-24 18:03:56 -07001385 ASSERT (tcp_in_fastrecovery (tc));
Florin Coras93992a92017-05-24 18:03:56 -07001386 tc->cc_algo->rcv_cong_ack (tc, TCP_CC_DUPACK);
1387 return;
Dave Barach68b0fb02017-02-28 15:15:56 -05001388 }
Florin Coras93992a92017-05-24 18:03:56 -07001389 else if (tcp_should_fastrecover (tc))
Dave Barach68b0fb02017-02-28 15:15:56 -05001390 {
Florin Corasc44a5582018-11-01 16:30:54 -07001391 u32 pacer_wnd;
1392
Florin Corasd2aab832018-05-22 11:39:59 -07001393 ASSERT (!tcp_in_fastrecovery (tc));
Florin Coras93992a92017-05-24 18:03:56 -07001394
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001395 /* Heuristic to catch potential late dupacks
1396 * after fast retransmit exits */
1397 if (is_dack && tc->snd_una == tc->snd_congestion
1398 && timestamp_leq (tc->rcv_opts.tsecr, tc->tsecr_last_ack))
Florin Coras93992a92017-05-24 18:03:56 -07001399 {
1400 tc->rcv_dupacks = 0;
1401 return;
1402 }
1403
1404 tcp_cc_init_congestion (tc);
1405 tc->cc_algo->rcv_cong_ack (tc, TCP_CC_DUPACK);
1406
Florin Coras3eb50622017-07-13 01:24:57 -04001407 if (tcp_opts_sack_permitted (&tc->rcv_opts))
Florin Corase55a6d72018-10-31 23:09:22 -07001408 {
1409 tc->cwnd = tc->ssthresh;
1410 scoreboard_init_high_rxt (&tc->sack_sb, tc->snd_una);
Florin Corase55a6d72018-10-31 23:09:22 -07001411 }
1412 else
1413 {
1414 /* Post retransmit update cwnd to ssthresh and account for the
1415 * three segments that have left the network and should've been
1416 * buffered at the receiver XXX */
1417 tc->cwnd = tc->ssthresh + 3 * tc->snd_mss;
1418 }
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001419
Florin Coras36ee9f12018-11-02 12:52:10 -07001420 /* Constrain rate until we get a partial ack */
Florin Corasc44a5582018-11-01 16:30:54 -07001421 pacer_wnd = clib_max (0.1 * tc->cwnd, 2 * tc->snd_mss);
1422 tcp_connection_tx_pacer_reset (tc, pacer_wnd,
1423 0 /* start bucket */ );
Florin Corasbe72ae62018-11-01 11:23:03 -07001424 tcp_program_fastretransmit (tcp_get_worker (tc->c_thread_index),
1425 tc);
Florin Coras93992a92017-05-24 18:03:56 -07001426 return;
Dave Barach68b0fb02017-02-28 15:15:56 -05001427 }
Florin Coras93992a92017-05-24 18:03:56 -07001428 else if (!tc->bytes_acked
1429 || (tc->bytes_acked && !tcp_in_cong_recovery (tc)))
1430 {
1431 tc->cc_algo->rcv_cong_ack (tc, TCP_CC_DUPACK);
1432 return;
1433 }
1434 else
1435 goto partial_ack;
1436 }
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001437 /* Don't allow entry in fast recovery if still in recovery, for now */
1438 else if (0 && is_dack && tcp_in_recovery (tc))
1439 {
1440 /* If of of the two conditions lower hold, reset dupacks because
1441 * we're probably after timeout (RFC6582 heuristics).
1442 * If Cumulative ack does not cover more than congestion threshold,
1443 * and:
1444 * 1) The following doesn't hold: The congestion window is greater
1445 * than SMSS bytes and the difference between highest_ack
1446 * and prev_highest_ack is at most 4*SMSS bytes
1447 * 2) Echoed timestamp in the last non-dup ack does not equal the
1448 * stored timestamp
1449 */
1450 if (seq_leq (tc->snd_una, tc->snd_congestion)
1451 && ((!(tc->cwnd > tc->snd_mss
1452 && tc->bytes_acked <= 4 * tc->snd_mss))
1453 || (tc->rcv_opts.tsecr != tc->tsecr_last_ack)))
1454 {
1455 tc->rcv_dupacks = 0;
1456 return;
1457 }
1458 }
Florin Coras93992a92017-05-24 18:03:56 -07001459
Florin Coras93992a92017-05-24 18:03:56 -07001460 if (!tc->bytes_acked)
1461 return;
1462
1463partial_ack:
Florin Corasd2aab832018-05-22 11:39:59 -07001464 TCP_EVT_DBG (TCP_EVT_CC_PACK, tc);
1465
Florin Coras93992a92017-05-24 18:03:56 -07001466 /*
1467 * Legitimate ACK. 1) See if we can exit recovery
1468 */
Florin Coras93992a92017-05-24 18:03:56 -07001469
Florin Corasefefc6b2018-11-07 12:49:19 -08001470 /* Update the pacing rate. For the first partial ack we move from
1471 * the artificially constrained rate to the one after congestion */
1472 tcp_connection_tx_pacer_update (tc);
1473
Florin Coras93992a92017-05-24 18:03:56 -07001474 if (seq_geq (tc->snd_una, tc->snd_congestion))
1475 {
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001476 tcp_retransmit_timer_update (tc);
1477
Florin Coras93992a92017-05-24 18:03:56 -07001478 /* If spurious return, we've already updated everything */
1479 if (tcp_cc_recover (tc))
Florin Corasf03a59a2017-06-09 21:07:32 -07001480 {
1481 tc->tsecr_last_ack = tc->rcv_opts.tsecr;
1482 return;
1483 }
Florin Coras93992a92017-05-24 18:03:56 -07001484
1485 tc->snd_nxt = tc->snd_una_max;
1486
1487 /* Treat as congestion avoidance ack */
Florin Corasd67f1122018-05-21 17:47:40 -07001488 tcp_cc_rcv_ack (tc);
Florin Coras93992a92017-05-24 18:03:56 -07001489 return;
1490 }
1491
1492 /*
1493 * Legitimate ACK. 2) If PARTIAL ACK try to retransmit
1494 */
Florin Coras93992a92017-05-24 18:03:56 -07001495
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001496 /* XXX limit this only to first partial ack? */
Florin Coras2e31cc32018-09-25 14:00:34 -07001497 tcp_retransmit_timer_update (tc);
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001498
Florin Coras93992a92017-05-24 18:03:56 -07001499 /* RFC6675: If the incoming ACK is a cumulative acknowledgment,
Florin Corasd2aab832018-05-22 11:39:59 -07001500 * reset dupacks to 0. Also needed if in congestion recovery */
Florin Coras93992a92017-05-24 18:03:56 -07001501 tc->rcv_dupacks = 0;
1502
Florin Coras93992a92017-05-24 18:03:56 -07001503 /* Post RTO timeout don't try anything fancy */
1504 if (tcp_in_recovery (tc))
Florin Corasd2aab832018-05-22 11:39:59 -07001505 {
Florin Corasd67f1122018-05-21 17:47:40 -07001506 tcp_cc_rcv_ack (tc);
Florin Coras3ec66b02018-08-23 16:27:05 -07001507 transport_add_tx_event (&tc->connection);
Florin Corasd2aab832018-05-22 11:39:59 -07001508 return;
1509 }
Florin Coras93992a92017-05-24 18:03:56 -07001510
1511 /* Remove retransmitted bytes that have been delivered */
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001512 if (tcp_opts_sack_permitted (&tc->rcv_opts))
Florin Coras93992a92017-05-24 18:03:56 -07001513 {
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001514 ASSERT (tc->bytes_acked + tc->sack_sb.snd_una_adv
1515 >= tc->sack_sb.last_bytes_delivered
1516 || (tc->flags & TCP_CONN_FINSNT));
1517
Florin Coras93992a92017-05-24 18:03:56 -07001518 /* If we have sacks and we haven't gotten an ack beyond high_rxt,
1519 * remove sacked bytes delivered */
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001520 if (seq_lt (tc->snd_una, tc->sack_sb.high_rxt))
1521 {
1522 rxt_delivered = tc->bytes_acked + tc->sack_sb.snd_una_adv
1523 - tc->sack_sb.last_bytes_delivered;
1524 ASSERT (tc->snd_rxt_bytes >= rxt_delivered);
1525 tc->snd_rxt_bytes -= rxt_delivered;
1526 }
1527 else
1528 {
1529 /* Apparently all retransmitted holes have been acked */
1530 tc->snd_rxt_bytes = 0;
Florin Coras36ee9f12018-11-02 12:52:10 -07001531 tc->sack_sb.high_rxt = tc->snd_una;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001532 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001533 }
1534 else
1535 {
Florin Coras36ee9f12018-11-02 12:52:10 -07001536 tcp_fastrecovery_first_on (tc);
1537 /* Reuse last bytes delivered to track total bytes acked */
1538 tc->sack_sb.last_bytes_delivered += tc->bytes_acked;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001539 if (tc->snd_rxt_bytes > tc->bytes_acked)
1540 tc->snd_rxt_bytes -= tc->bytes_acked;
1541 else
1542 tc->snd_rxt_bytes = 0;
Dave Barach68b0fb02017-02-28 15:15:56 -05001543 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001544
Florin Coras93992a92017-05-24 18:03:56 -07001545 tc->cc_algo->rcv_cong_ack (tc, TCP_CC_PARTIALACK);
Dave Barach68b0fb02017-02-28 15:15:56 -05001546
Florin Coras93992a92017-05-24 18:03:56 -07001547 /*
1548 * Since this was a partial ack, try to retransmit some more data
1549 */
Florin Corasbe72ae62018-11-01 11:23:03 -07001550 tcp_program_fastretransmit (tcp_get_worker (tc->c_thread_index), tc);
Dave Barach68b0fb02017-02-28 15:15:56 -05001551}
1552
Florin Coras93992a92017-05-24 18:03:56 -07001553/**
1554 * Process incoming ACK
1555 */
Dave Barach68b0fb02017-02-28 15:15:56 -05001556static int
Florin Coras9ece3c02018-11-05 11:06:53 -08001557tcp_rcv_ack (tcp_worker_ctx_t * wrk, tcp_connection_t * tc, vlib_buffer_t * b,
Florin Coras7ac053b2018-11-05 15:57:21 -08001558 tcp_header_t * th, u32 * error)
Dave Barach68b0fb02017-02-28 15:15:56 -05001559{
Florin Coras93992a92017-05-24 18:03:56 -07001560 u32 prev_snd_wnd, prev_snd_una;
1561 u8 is_dack;
Dave Barach68b0fb02017-02-28 15:15:56 -05001562
Florin Corasf03a59a2017-06-09 21:07:32 -07001563 TCP_EVT_DBG (TCP_EVT_CC_STAT, tc);
1564
Florin Coras6792ec02017-03-13 03:49:51 -07001565 /* If the ACK acks something not yet sent (SEG.ACK > SND.NXT) */
Florin Coras93992a92017-05-24 18:03:56 -07001566 if (PREDICT_FALSE (seq_gt (vnet_buffer (b)->tcp.ack_number, tc->snd_nxt)))
Dave Barach68b0fb02017-02-28 15:15:56 -05001567 {
Florin Coras5fd32102019-02-28 12:49:45 -08001568 /* When we entered cong recovery, we reset snd_nxt to snd_una. Seems
1569 * peer still has the data so accept the ack */
1570 if (tcp_in_cong_recovery (tc)
1571 && seq_leq (vnet_buffer (b)->tcp.ack_number,
1572 tc->snd_una + tc->snd_wnd))
Florin Corasca1c8f32018-05-23 21:01:30 -07001573 {
Florin Coras3ec66b02018-08-23 16:27:05 -07001574 tc->snd_nxt = vnet_buffer (b)->tcp.ack_number;
1575 if (seq_gt (tc->snd_nxt, tc->snd_una_max))
1576 tc->snd_una_max = tc->snd_nxt;
Florin Corasca1c8f32018-05-23 21:01:30 -07001577 goto process_ack;
1578 }
1579
Florin Coras6792ec02017-03-13 03:49:51 -07001580 /* If we have outstanding data and this is within the window, accept it,
1581 * probably retransmit has timed out. Otherwise ACK segment and then
1582 * drop it */
1583 if (seq_gt (vnet_buffer (b)->tcp.ack_number, tc->snd_una_max))
1584 {
Florin Coras7ac053b2018-11-05 15:57:21 -08001585 tcp_program_ack (wrk, tc);
Florin Coras36ee9f12018-11-02 12:52:10 -07001586 *error = TCP_ERROR_ACK_FUTURE;
Florin Coras6792ec02017-03-13 03:49:51 -07001587 TCP_EVT_DBG (TCP_EVT_ACK_RCV_ERR, tc, 0,
1588 vnet_buffer (b)->tcp.ack_number);
1589 return -1;
1590 }
1591
Florin Coras6792ec02017-03-13 03:49:51 -07001592 TCP_EVT_DBG (TCP_EVT_ACK_RCV_ERR, tc, 2,
1593 vnet_buffer (b)->tcp.ack_number);
Florin Coras3e350af2017-03-30 02:54:28 -07001594
1595 tc->snd_nxt = vnet_buffer (b)->tcp.ack_number;
Florin Coras5fd32102019-02-28 12:49:45 -08001596 if (seq_gt (tc->snd_nxt, tc->snd_una_max))
1597 tc->snd_una_max = tc->snd_nxt;
1598
1599 goto process_ack;
Dave Barach68b0fb02017-02-28 15:15:56 -05001600 }
1601
Florin Coras6792ec02017-03-13 03:49:51 -07001602 /* If old ACK, probably it's an old dupack */
Florin Coras93992a92017-05-24 18:03:56 -07001603 if (PREDICT_FALSE (seq_lt (vnet_buffer (b)->tcp.ack_number, tc->snd_una)))
Dave Barach68b0fb02017-02-28 15:15:56 -05001604 {
1605 *error = TCP_ERROR_ACK_OLD;
Florin Coras6792ec02017-03-13 03:49:51 -07001606 TCP_EVT_DBG (TCP_EVT_ACK_RCV_ERR, tc, 1,
1607 vnet_buffer (b)->tcp.ack_number);
1608 if (tcp_in_fastrecovery (tc) && tc->rcv_dupacks == TCP_DUPACK_THRESHOLD)
Florin Corasca1c8f32018-05-23 21:01:30 -07001609 tcp_cc_handle_event (tc, 1);
Florin Corasc28764f2017-04-26 00:08:42 -07001610 /* Don't drop yet */
1611 return 0;
Dave Barach68b0fb02017-02-28 15:15:56 -05001612 }
1613
Florin Coras93992a92017-05-24 18:03:56 -07001614 /*
1615 * Looks okay, process feedback
1616 */
Florin Corasca1c8f32018-05-23 21:01:30 -07001617process_ack:
Florin Coras93992a92017-05-24 18:03:56 -07001618 if (tcp_opts_sack_permitted (&tc->rcv_opts))
Dave Barach68b0fb02017-02-28 15:15:56 -05001619 tcp_rcv_sacks (tc, vnet_buffer (b)->tcp.ack_number);
1620
Florin Coras93992a92017-05-24 18:03:56 -07001621 prev_snd_wnd = tc->snd_wnd;
1622 prev_snd_una = tc->snd_una;
1623 tcp_update_snd_wnd (tc, vnet_buffer (b)->tcp.seq_number,
1624 vnet_buffer (b)->tcp.ack_number,
1625 clib_net_to_host_u16 (th->window) << tc->snd_wscale);
1626 tc->bytes_acked = vnet_buffer (b)->tcp.ack_number - tc->snd_una;
1627 tc->snd_una = vnet_buffer (b)->tcp.ack_number + tc->sack_sb.snd_una_adv;
1628 tcp_validate_txf_size (tc, tc->bytes_acked);
Dave Barach68b0fb02017-02-28 15:15:56 -05001629
Florin Coras93992a92017-05-24 18:03:56 -07001630 if (tc->bytes_acked)
Florin Coras9ece3c02018-11-05 11:06:53 -08001631 {
1632 tcp_program_dequeue (wrk, tc);
1633 tcp_update_rtt (tc, vnet_buffer (b)->tcp.ack_number);
1634 }
Florin Coras93992a92017-05-24 18:03:56 -07001635
Florin Coras6534b7a2017-07-18 05:38:03 -04001636 TCP_EVT_DBG (TCP_EVT_ACK_RCVD, tc);
1637
Florin Coras93992a92017-05-24 18:03:56 -07001638 /*
1639 * Check if we have congestion event
1640 */
1641
1642 if (tcp_ack_is_cc_event (tc, b, prev_snd_wnd, prev_snd_una, &is_dack))
Dave Barach68b0fb02017-02-28 15:15:56 -05001643 {
Florin Coras93992a92017-05-24 18:03:56 -07001644 tcp_cc_handle_event (tc, is_dack);
Florin Corasb2215d62017-08-01 16:56:58 -07001645 if (!tcp_in_cong_recovery (tc))
Florin Corasa9d5bea2018-12-17 08:24:19 -08001646 {
1647 *error = TCP_ERROR_ACK_OK;
1648 return 0;
1649 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001650 *error = TCP_ERROR_ACK_DUP;
Florin Corasca09d072018-10-01 18:31:02 -07001651 if (vnet_buffer (b)->tcp.data_len || tcp_is_fin (th))
1652 return 0;
1653 return -1;
Dave Barach68b0fb02017-02-28 15:15:56 -05001654 }
1655
Florin Coras6792ec02017-03-13 03:49:51 -07001656 /*
Florin Coras93992a92017-05-24 18:03:56 -07001657 * Update congestion control (slow start/congestion avoidance)
Florin Coras6792ec02017-03-13 03:49:51 -07001658 */
Florin Coras93992a92017-05-24 18:03:56 -07001659 tcp_cc_update (tc, b);
Florin Coras00cd22d2018-04-18 13:20:18 -07001660 *error = TCP_ERROR_ACK_OK;
Dave Barach68b0fb02017-02-28 15:15:56 -05001661 return 0;
1662}
1663
Florin Corasb11175d2018-11-09 14:34:08 -08001664static void
1665tcp_program_disconnect (tcp_worker_ctx_t * wrk, tcp_connection_t * tc)
1666{
1667 if (!tcp_disconnect_pending (tc))
1668 {
1669 vec_add1 (wrk->pending_disconnects, tc->c_c_index);
1670 tcp_disconnect_pending_on (tc);
1671 }
1672}
1673
1674static void
1675tcp_handle_disconnects (tcp_worker_ctx_t * wrk)
1676{
1677 u32 thread_index, *pending_disconnects;
1678 tcp_connection_t *tc;
1679 int i;
1680
1681 if (!vec_len (wrk->pending_disconnects))
1682 return;
1683
1684 thread_index = wrk->vm->thread_index;
1685 pending_disconnects = wrk->pending_disconnects;
1686 for (i = 0; i < vec_len (pending_disconnects); i++)
1687 {
1688 tc = tcp_connection_get (pending_disconnects[i], thread_index);
1689 tcp_disconnect_pending_off (tc);
Florin Coras5a2ec8f2018-12-27 11:53:11 -08001690 session_transport_closing_notify (&tc->connection);
Florin Corasb11175d2018-11-09 14:34:08 -08001691 }
1692 _vec_len (wrk->pending_disconnects) = 0;
1693}
1694
1695static void
1696tcp_rcv_fin (tcp_worker_ctx_t * wrk, tcp_connection_t * tc, vlib_buffer_t * b,
1697 u32 * error)
1698{
Florin Coras3c514d52018-12-22 11:39:33 -08001699 /* Account for the FIN and send ack */
1700 tc->rcv_nxt += 1;
1701 tcp_program_ack (wrk, tc);
Florin Corasb11175d2018-11-09 14:34:08 -08001702 /* Enter CLOSE-WAIT and notify session. To avoid lingering
1703 * in CLOSE-WAIT, set timer (reuse WAITCLOSE). */
Florin Coras3c514d52018-12-22 11:39:33 -08001704 tcp_connection_set_state (tc, TCP_STATE_CLOSE_WAIT);
Florin Corasb11175d2018-11-09 14:34:08 -08001705 tcp_program_disconnect (wrk, tc);
1706 tcp_timer_update (tc, TCP_TIMER_WAITCLOSE, TCP_CLOSEWAIT_TIME);
1707 TCP_EVT_DBG (TCP_EVT_FIN_RCVD, tc);
1708 *error = TCP_ERROR_FIN_RCVD;
1709}
1710
Filip Tehlare275bed2019-03-06 00:06:56 -08001711#ifndef CLIB_MARCH_VARIANT
Florin Coras3eb50622017-07-13 01:24:57 -04001712static u8
1713tcp_sack_vector_is_sane (sack_block_t * sacks)
1714{
1715 int i;
1716 for (i = 1; i < vec_len (sacks); i++)
1717 {
1718 if (sacks[i - 1].end == sacks[i].start)
1719 return 0;
1720 }
1721 return 1;
1722}
1723
Dave Barach68b0fb02017-02-28 15:15:56 -05001724/**
1725 * Build SACK list as per RFC2018.
1726 *
1727 * Makes sure the first block contains the segment that generated the current
1728 * ACK and the following ones are the ones most recently reported in SACK
1729 * blocks.
1730 *
1731 * @param tc TCP connection for which the SACK list is updated
1732 * @param start Start sequence number of the newest SACK block
1733 * @param end End sequence of the newest SACK block
1734 */
Florin Coras45d34962017-04-25 00:05:27 -07001735void
Dave Barach68b0fb02017-02-28 15:15:56 -05001736tcp_update_sack_list (tcp_connection_t * tc, u32 start, u32 end)
1737{
Florin Corasb691f762019-02-22 09:07:20 -08001738 sack_block_t *new_list = tc->snd_sacks_fl, *block = 0;
Dave Barach68b0fb02017-02-28 15:15:56 -05001739 int i;
Dave Barach68b0fb02017-02-28 15:15:56 -05001740
1741 /* If the first segment is ooo add it to the list. Last write might've moved
1742 * rcv_nxt over the first segment. */
1743 if (seq_lt (tc->rcv_nxt, start))
1744 {
Florin Coras45d34962017-04-25 00:05:27 -07001745 vec_add2 (new_list, block, 1);
1746 block->start = start;
1747 block->end = end;
Dave Barach68b0fb02017-02-28 15:15:56 -05001748 }
1749
1750 /* Find the blocks still worth keeping. */
1751 for (i = 0; i < vec_len (tc->snd_sacks); i++)
1752 {
Florin Coras45d34962017-04-25 00:05:27 -07001753 /* Discard if rcv_nxt advanced beyond current block */
1754 if (seq_leq (tc->snd_sacks[i].start, tc->rcv_nxt))
Dave Barach68b0fb02017-02-28 15:15:56 -05001755 continue;
1756
Florin Coras45d34962017-04-25 00:05:27 -07001757 /* Merge or drop if segment overlapped by the new segment */
1758 if (block && (seq_geq (tc->snd_sacks[i].end, new_list[0].start)
1759 && seq_leq (tc->snd_sacks[i].start, new_list[0].end)))
1760 {
1761 if (seq_lt (tc->snd_sacks[i].start, new_list[0].start))
1762 new_list[0].start = tc->snd_sacks[i].start;
1763 if (seq_lt (new_list[0].end, tc->snd_sacks[i].end))
1764 new_list[0].end = tc->snd_sacks[i].end;
1765 continue;
1766 }
1767
1768 /* Save to new SACK list if we have space. */
1769 if (vec_len (new_list) < TCP_MAX_SACK_BLOCKS)
1770 {
1771 vec_add1 (new_list, tc->snd_sacks[i]);
1772 }
Florin Corasf03a59a2017-06-09 21:07:32 -07001773 else
1774 {
1775 clib_warning ("sack discarded");
1776 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001777 }
1778
Florin Coras45d34962017-04-25 00:05:27 -07001779 ASSERT (vec_len (new_list) <= TCP_MAX_SACK_BLOCKS);
Florin Coras6792ec02017-03-13 03:49:51 -07001780
Dave Barach68b0fb02017-02-28 15:15:56 -05001781 /* Replace old vector with new one */
Florin Corasb691f762019-02-22 09:07:20 -08001782 vec_reset_length (tc->snd_sacks);
1783 tc->snd_sacks_fl = tc->snd_sacks;
Dave Barach68b0fb02017-02-28 15:15:56 -05001784 tc->snd_sacks = new_list;
Florin Coras3eb50622017-07-13 01:24:57 -04001785
1786 /* Segments should not 'touch' */
1787 ASSERT (tcp_sack_vector_is_sane (tc->snd_sacks));
Dave Barach68b0fb02017-02-28 15:15:56 -05001788}
1789
Florin Corasca1c8f32018-05-23 21:01:30 -07001790u32
1791tcp_sack_list_bytes (tcp_connection_t * tc)
1792{
1793 u32 bytes = 0, i;
1794 for (i = 0; i < vec_len (tc->snd_sacks); i++)
1795 bytes += tc->snd_sacks[i].end - tc->snd_sacks[i].start;
1796 return bytes;
1797}
Filip Tehlare275bed2019-03-06 00:06:56 -08001798#endif /* CLIB_MARCH_VARIANT */
Florin Corasca1c8f32018-05-23 21:01:30 -07001799
Dave Barach68b0fb02017-02-28 15:15:56 -05001800/** Enqueue data for delivery to application */
Florin Coras0dbd5172018-06-25 16:19:34 -07001801static int
Dave Barach68b0fb02017-02-28 15:15:56 -05001802tcp_session_enqueue_data (tcp_connection_t * tc, vlib_buffer_t * b,
1803 u16 data_len)
1804{
Florin Coras1f152cd2017-08-18 19:28:03 -07001805 int written, error = TCP_ERROR_ENQUEUED;
Dave Barach68b0fb02017-02-28 15:15:56 -05001806
Dave Barach2c25a622017-06-26 11:35:07 -04001807 ASSERT (seq_geq (vnet_buffer (b)->tcp.seq_number, tc->rcv_nxt));
Florin Coras00cd22d2018-04-18 13:20:18 -07001808 ASSERT (data_len);
Florin Coras3cbc04b2017-10-02 00:18:51 -07001809 written = session_enqueue_stream_connection (&tc->connection, b, 0,
1810 1 /* queue event */ , 1);
Dave Barach68b0fb02017-02-28 15:15:56 -05001811
Florin Coras6792ec02017-03-13 03:49:51 -07001812 TCP_EVT_DBG (TCP_EVT_INPUT, tc, 0, data_len, written);
1813
Dave Barach68b0fb02017-02-28 15:15:56 -05001814 /* Update rcv_nxt */
1815 if (PREDICT_TRUE (written == data_len))
1816 {
Florin Coras1f152cd2017-08-18 19:28:03 -07001817 tc->rcv_nxt += written;
Dave Barach68b0fb02017-02-28 15:15:56 -05001818 }
1819 /* If more data written than expected, account for out-of-order bytes. */
1820 else if (written > data_len)
1821 {
Florin Coras1f152cd2017-08-18 19:28:03 -07001822 tc->rcv_nxt += written;
Florin Corasca1c8f32018-05-23 21:01:30 -07001823 TCP_EVT_DBG (TCP_EVT_CC_INPUT, tc, data_len, written);
Florin Coras6792ec02017-03-13 03:49:51 -07001824 }
1825 else if (written > 0)
1826 {
1827 /* We've written something but FIFO is probably full now */
1828 tc->rcv_nxt += written;
Florin Coras1f152cd2017-08-18 19:28:03 -07001829 error = TCP_ERROR_PARTIALLY_ENQUEUED;
Dave Barach68b0fb02017-02-28 15:15:56 -05001830 }
1831 else
1832 {
Dave Barach68b0fb02017-02-28 15:15:56 -05001833 return TCP_ERROR_FIFO_FULL;
1834 }
1835
Florin Coras6792ec02017-03-13 03:49:51 -07001836 /* Update SACK list if need be */
Florin Coras93992a92017-05-24 18:03:56 -07001837 if (tcp_opts_sack_permitted (&tc->rcv_opts))
Florin Coras6792ec02017-03-13 03:49:51 -07001838 {
1839 /* Remove SACK blocks that have been delivered */
1840 tcp_update_sack_list (tc, tc->rcv_nxt, tc->rcv_nxt);
1841 }
1842
Florin Coras1f152cd2017-08-18 19:28:03 -07001843 return error;
Dave Barach68b0fb02017-02-28 15:15:56 -05001844}
1845
1846/** Enqueue out-of-order data */
Florin Coras0dbd5172018-06-25 16:19:34 -07001847static int
Dave Barach68b0fb02017-02-28 15:15:56 -05001848tcp_session_enqueue_ooo (tcp_connection_t * tc, vlib_buffer_t * b,
1849 u16 data_len)
1850{
Florin Coras288eaab2019-02-03 15:26:14 -08001851 session_t *s0;
Florin Coras3eb50622017-07-13 01:24:57 -04001852 int rv, offset;
Florin Coras6792ec02017-03-13 03:49:51 -07001853
Florin Corasf03a59a2017-06-09 21:07:32 -07001854 ASSERT (seq_gt (vnet_buffer (b)->tcp.seq_number, tc->rcv_nxt));
Florin Coras00cd22d2018-04-18 13:20:18 -07001855 ASSERT (data_len);
Dave Barach68b0fb02017-02-28 15:15:56 -05001856
Florin Corasf03a59a2017-06-09 21:07:32 -07001857 /* Enqueue out-of-order data with relative offset */
Florin Coras3cbc04b2017-10-02 00:18:51 -07001858 rv = session_enqueue_stream_connection (&tc->connection, b,
1859 vnet_buffer (b)->tcp.seq_number -
1860 tc->rcv_nxt, 0 /* queue event */ ,
1861 0);
Florin Coras6792ec02017-03-13 03:49:51 -07001862
1863 /* Nothing written */
1864 if (rv)
1865 {
1866 TCP_EVT_DBG (TCP_EVT_INPUT, tc, 1, data_len, 0);
1867 return TCP_ERROR_FIFO_FULL;
1868 }
1869
1870 TCP_EVT_DBG (TCP_EVT_INPUT, tc, 1, data_len, data_len);
Dave Barach68b0fb02017-02-28 15:15:56 -05001871
1872 /* Update SACK list if in use */
Florin Coras93992a92017-05-24 18:03:56 -07001873 if (tcp_opts_sack_permitted (&tc->rcv_opts))
Dave Barach68b0fb02017-02-28 15:15:56 -05001874 {
1875 ooo_segment_t *newest;
1876 u32 start, end;
1877
Florin Corascea194d2017-10-02 00:18:51 -07001878 s0 = session_get (tc->c_s_index, tc->c_thread_index);
Florin Corasf6d68ed2017-05-07 19:12:02 -07001879
Dave Barach68b0fb02017-02-28 15:15:56 -05001880 /* Get the newest segment from the fifo */
Florin Coras288eaab2019-02-03 15:26:14 -08001881 newest = svm_fifo_newest_ooo_segment (s0->rx_fifo);
Florin Corasf03a59a2017-06-09 21:07:32 -07001882 if (newest)
1883 {
Florin Coras288eaab2019-02-03 15:26:14 -08001884 offset = ooo_segment_offset (s0->rx_fifo, newest);
Florin Coras3eb50622017-07-13 01:24:57 -04001885 ASSERT (offset <= vnet_buffer (b)->tcp.seq_number - tc->rcv_nxt);
1886 start = tc->rcv_nxt + offset;
Florin Coras288eaab2019-02-03 15:26:14 -08001887 end = start + ooo_segment_length (s0->rx_fifo, newest);
Florin Corasf03a59a2017-06-09 21:07:32 -07001888 tcp_update_sack_list (tc, start, end);
Florin Coras288eaab2019-02-03 15:26:14 -08001889 svm_fifo_newest_ooo_segment_reset (s0->rx_fifo);
Florin Corasca1c8f32018-05-23 21:01:30 -07001890 TCP_EVT_DBG (TCP_EVT_CC_SACKS, tc);
Florin Corasf03a59a2017-06-09 21:07:32 -07001891 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001892 }
1893
Florin Coras00cd22d2018-04-18 13:20:18 -07001894 return TCP_ERROR_ENQUEUED_OOO;
Dave Barach68b0fb02017-02-28 15:15:56 -05001895}
1896
1897/**
Florin Coras6792ec02017-03-13 03:49:51 -07001898 * Check if ACK could be delayed. If ack can be delayed, it should return
1899 * true for a full frame. If we're always acking return 0.
Dave Barach68b0fb02017-02-28 15:15:56 -05001900 */
1901always_inline int
1902tcp_can_delack (tcp_connection_t * tc)
1903{
Florin Coras6792ec02017-03-13 03:49:51 -07001904 /* Send ack if ... */
1905 if (TCP_ALWAYS_ACK
1906 /* just sent a rcv wnd 0 */
1907 || (tc->flags & TCP_CONN_SENT_RCV_WND0) != 0
1908 /* constrained to send ack */
1909 || (tc->flags & TCP_CONN_SNDACK) != 0
1910 /* we're almost out of tx wnd */
Florin Corasca1c8f32018-05-23 21:01:30 -07001911 || tcp_available_cc_snd_space (tc) < 4 * tc->snd_mss)
Florin Coras6792ec02017-03-13 03:49:51 -07001912 return 0;
Dave Barach68b0fb02017-02-28 15:15:56 -05001913
Florin Coras6792ec02017-03-13 03:49:51 -07001914 return 1;
Dave Barach68b0fb02017-02-28 15:15:56 -05001915}
1916
1917static int
Florin Corasb2215d62017-08-01 16:56:58 -07001918tcp_buffer_discard_bytes (vlib_buffer_t * b, u32 n_bytes_to_drop)
1919{
Florin Coras1f152cd2017-08-18 19:28:03 -07001920 u32 discard, first = b->current_length;
Florin Corasb2215d62017-08-01 16:56:58 -07001921 vlib_main_t *vm = vlib_get_main ();
1922
Florin Coras1f152cd2017-08-18 19:28:03 -07001923 /* Handle multi-buffer segments */
Florin Corasb2215d62017-08-01 16:56:58 -07001924 if (n_bytes_to_drop > b->current_length)
1925 {
1926 if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))
1927 return -1;
1928 do
1929 {
1930 discard = clib_min (n_bytes_to_drop, b->current_length);
1931 vlib_buffer_advance (b, discard);
1932 b = vlib_get_buffer (vm, b->next_buffer);
1933 n_bytes_to_drop -= discard;
1934 }
1935 while (n_bytes_to_drop);
Florin Coras1f152cd2017-08-18 19:28:03 -07001936 if (n_bytes_to_drop > first)
1937 b->total_length_not_including_first_buffer -= n_bytes_to_drop - first;
Florin Corasb2215d62017-08-01 16:56:58 -07001938 }
Florin Coras1f152cd2017-08-18 19:28:03 -07001939 else
1940 vlib_buffer_advance (b, n_bytes_to_drop);
1941 vnet_buffer (b)->tcp.data_len -= n_bytes_to_drop;
Florin Corasb2215d62017-08-01 16:56:58 -07001942 return 0;
1943}
1944
Florin Coras00cd22d2018-04-18 13:20:18 -07001945/**
1946 * Receive buffer for connection and handle acks
1947 *
1948 * It handles both in order or out-of-order data.
1949 */
Florin Corasb2215d62017-08-01 16:56:58 -07001950static int
Florin Coras7ac053b2018-11-05 15:57:21 -08001951tcp_segment_rcv (tcp_worker_ctx_t * wrk, tcp_connection_t * tc,
1952 vlib_buffer_t * b)
Dave Barach68b0fb02017-02-28 15:15:56 -05001953{
Florin Coras00cd22d2018-04-18 13:20:18 -07001954 u32 error, n_bytes_to_drop, n_data_bytes;
Florin Coras6534b7a2017-07-18 05:38:03 -04001955
1956 vlib_buffer_advance (b, vnet_buffer (b)->tcp.data_offset);
1957 n_data_bytes = vnet_buffer (b)->tcp.data_len;
1958 ASSERT (n_data_bytes);
Dave Barach68b0fb02017-02-28 15:15:56 -05001959
1960 /* Handle out-of-order data */
1961 if (PREDICT_FALSE (vnet_buffer (b)->tcp.seq_number != tc->rcv_nxt))
1962 {
Florin Coras6792ec02017-03-13 03:49:51 -07001963 /* Old sequence numbers allowed through because they overlapped
1964 * the rx window */
1965 if (seq_lt (vnet_buffer (b)->tcp.seq_number, tc->rcv_nxt))
Dave Barach68b0fb02017-02-28 15:15:56 -05001966 {
Florin Coras00cd22d2018-04-18 13:20:18 -07001967 /* Completely in the past (possible retransmit). Ack
1968 * retransmissions since we may not have any data to send */
Florin Corasf03a59a2017-06-09 21:07:32 -07001969 if (seq_leq (vnet_buffer (b)->tcp.seq_end, tc->rcv_nxt))
Florin Coras6534b7a2017-07-18 05:38:03 -04001970 {
Florin Coras7ac053b2018-11-05 15:57:21 -08001971 tcp_program_ack (wrk, tc);
Florin Coras00cd22d2018-04-18 13:20:18 -07001972 error = TCP_ERROR_SEGMENT_OLD;
Florin Coras6534b7a2017-07-18 05:38:03 -04001973 goto done;
1974 }
Dave Barach259cdae2017-05-15 16:27:05 -04001975
Florin Coras00cd22d2018-04-18 13:20:18 -07001976 /* Chop off the bytes in the past and see if what is left
1977 * can be enqueued in order */
Florin Corasdb84e572017-05-09 18:54:52 -07001978 n_bytes_to_drop = tc->rcv_nxt - vnet_buffer (b)->tcp.seq_number;
1979 n_data_bytes -= n_bytes_to_drop;
Dave Barach2c25a622017-06-26 11:35:07 -04001980 vnet_buffer (b)->tcp.seq_number = tc->rcv_nxt;
Florin Corasb2215d62017-08-01 16:56:58 -07001981 if (tcp_buffer_discard_bytes (b, n_bytes_to_drop))
Florin Coras00cd22d2018-04-18 13:20:18 -07001982 {
1983 error = TCP_ERROR_SEGMENT_OLD;
Florin Coras00cd22d2018-04-18 13:20:18 -07001984 goto done;
1985 }
Florin Corasdb84e572017-05-09 18:54:52 -07001986 goto in_order;
Dave Barach68b0fb02017-02-28 15:15:56 -05001987 }
1988
Florin Coras00cd22d2018-04-18 13:20:18 -07001989 /* RFC2581: Enqueue and send DUPACK for fast retransmit */
Florin Coras6792ec02017-03-13 03:49:51 -07001990 error = tcp_session_enqueue_ooo (tc, b, n_data_bytes);
Florin Coras7ac053b2018-11-05 15:57:21 -08001991 tcp_program_dupack (wrk, tc);
Florin Corasca1c8f32018-05-23 21:01:30 -07001992 TCP_EVT_DBG (TCP_EVT_DUPACK_SENT, tc, vnet_buffer (b)->tcp);
Dave Barach68b0fb02017-02-28 15:15:56 -05001993 goto done;
1994 }
1995
Florin Corasdb84e572017-05-09 18:54:52 -07001996in_order:
1997
Dave Barach68b0fb02017-02-28 15:15:56 -05001998 /* In order data, enqueue. Fifo figures out by itself if any out-of-order
1999 * segments can be enqueued after fifo tail offset changes. */
2000 error = tcp_session_enqueue_data (tc, b, n_data_bytes);
Florin Coras3e350af2017-03-30 02:54:28 -07002001 if (tcp_can_delack (tc))
Dave Barach68b0fb02017-02-28 15:15:56 -05002002 {
Florin Coras6792ec02017-03-13 03:49:51 -07002003 if (!tcp_timer_is_active (tc, TCP_TIMER_DELACK))
2004 tcp_timer_set (tc, TCP_TIMER_DELACK, TCP_DELACK_TIME);
Florin Coras3e350af2017-03-30 02:54:28 -07002005 goto done;
Dave Barach68b0fb02017-02-28 15:15:56 -05002006 }
2007
Florin Coras7ac053b2018-11-05 15:57:21 -08002008 tcp_program_ack (wrk, tc);
Florin Coras3e350af2017-03-30 02:54:28 -07002009
Dave Barach68b0fb02017-02-28 15:15:56 -05002010done:
2011 return error;
2012}
2013
Clement Durand6cf260c2017-04-13 13:27:04 +02002014typedef struct
2015{
2016 tcp_header_t tcp_header;
2017 tcp_connection_t tcp_connection;
2018} tcp_rx_trace_t;
2019
Florin Coras0dbd5172018-06-25 16:19:34 -07002020static u8 *
Clement Durand6cf260c2017-04-13 13:27:04 +02002021format_tcp_rx_trace (u8 * s, va_list * args)
2022{
2023 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
2024 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
2025 tcp_rx_trace_t *t = va_arg (*args, tcp_rx_trace_t *);
Christophe Fontained3c008d2017-10-02 18:10:54 +02002026 u32 indent = format_get_indent (s);
Clement Durand6cf260c2017-04-13 13:27:04 +02002027
2028 s = format (s, "%U\n%U%U",
2029 format_tcp_header, &t->tcp_header, 128,
2030 format_white_space, indent,
Florin Corasbb292f42017-05-19 09:49:19 -07002031 format_tcp_connection, &t->tcp_connection, 1);
Clement Durand6cf260c2017-04-13 13:27:04 +02002032
2033 return s;
2034}
2035
Florin Coras0dbd5172018-06-25 16:19:34 -07002036static u8 *
Clement Durand6cf260c2017-04-13 13:27:04 +02002037format_tcp_rx_trace_short (u8 * s, va_list * args)
2038{
2039 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
2040 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
2041 tcp_rx_trace_t *t = va_arg (*args, tcp_rx_trace_t *);
2042
2043 s = format (s, "%d -> %d (%U)",
Florin Coras4df38712018-06-20 12:44:16 -07002044 clib_net_to_host_u16 (t->tcp_header.dst_port),
2045 clib_net_to_host_u16 (t->tcp_header.src_port), format_tcp_state,
Florin Corasbb292f42017-05-19 09:49:19 -07002046 t->tcp_connection.state);
Clement Durand6cf260c2017-04-13 13:27:04 +02002047
2048 return s;
2049}
2050
Florin Coras4df38712018-06-20 12:44:16 -07002051static void
Florin Coras82b13a82017-04-25 11:58:06 -07002052tcp_set_rx_trace_data (tcp_rx_trace_t * t0, tcp_connection_t * tc0,
2053 tcp_header_t * th0, vlib_buffer_t * b0, u8 is_ip4)
2054{
2055 if (tc0)
2056 {
Dave Barach178cf492018-11-13 16:34:13 -05002057 clib_memcpy_fast (&t0->tcp_connection, tc0,
2058 sizeof (t0->tcp_connection));
Florin Coras82b13a82017-04-25 11:58:06 -07002059 }
2060 else
2061 {
2062 th0 = tcp_buffer_hdr (b0);
2063 }
Dave Barach178cf492018-11-13 16:34:13 -05002064 clib_memcpy_fast (&t0->tcp_header, th0, sizeof (t0->tcp_header));
Florin Coras82b13a82017-04-25 11:58:06 -07002065}
2066
Florin Coras4df38712018-06-20 12:44:16 -07002067static void
2068tcp_established_trace_frame (vlib_main_t * vm, vlib_node_runtime_t * node,
2069 vlib_frame_t * frame, u8 is_ip4)
2070{
2071 u32 *from, n_left;
2072
2073 n_left = frame->n_vectors;
2074 from = vlib_frame_vector_args (frame);
2075
2076 while (n_left >= 1)
2077 {
2078 tcp_connection_t *tc0;
2079 tcp_rx_trace_t *t0;
2080 tcp_header_t *th0;
2081 vlib_buffer_t *b0;
2082 u32 bi0;
2083
2084 bi0 = from[0];
2085 b0 = vlib_get_buffer (vm, bi0);
2086
2087 if (b0->flags & VLIB_BUFFER_IS_TRACED)
2088 {
2089 t0 = vlib_add_trace (vm, node, b0, sizeof (*t0));
2090 tc0 = tcp_connection_get (vnet_buffer (b0)->tcp.connection_index,
2091 vm->thread_index);
2092 th0 = tcp_buffer_hdr (b0);
2093 tcp_set_rx_trace_data (t0, tc0, th0, b0, is_ip4);
2094 }
2095
2096 from += 1;
2097 n_left -= 1;
2098 }
2099}
2100
Florin Coras6792ec02017-03-13 03:49:51 -07002101always_inline void
Florin Coras00cd22d2018-04-18 13:20:18 -07002102tcp_node_inc_counter_i (vlib_main_t * vm, u32 tcp4_node, u32 tcp6_node,
2103 u8 is_ip4, u32 evt, u32 val)
Dave Barach68b0fb02017-02-28 15:15:56 -05002104{
Florin Coras6792ec02017-03-13 03:49:51 -07002105 if (is_ip4)
Florin Coras3cbc04b2017-10-02 00:18:51 -07002106 vlib_node_increment_counter (vm, tcp4_node, evt, val);
Florin Coras6792ec02017-03-13 03:49:51 -07002107 else
Florin Coras3cbc04b2017-10-02 00:18:51 -07002108 vlib_node_increment_counter (vm, tcp6_node, evt, val);
Dave Barach68b0fb02017-02-28 15:15:56 -05002109}
2110
Florin Coras00cd22d2018-04-18 13:20:18 -07002111#define tcp_maybe_inc_counter(node_id, err, count) \
2112{ \
2113 if (next0 != tcp_next_drop (is_ip4)) \
2114 tcp_node_inc_counter_i (vm, tcp4_##node_id##_node.index, \
2115 tcp6_##node_id##_node.index, is_ip4, err, \
2116 1); \
2117}
2118#define tcp_inc_counter(node_id, err, count) \
2119 tcp_node_inc_counter_i (vm, tcp4_##node_id##_node.index, \
2120 tcp6_##node_id##_node.index, is_ip4, \
2121 err, count)
2122#define tcp_maybe_inc_err_counter(cnts, err) \
2123{ \
2124 cnts[err] += (next0 != tcp_next_drop (is_ip4)); \
2125}
2126#define tcp_inc_err_counter(cnts, err, val) \
2127{ \
2128 cnts[err] += val; \
2129}
2130#define tcp_store_err_counters(node_id, cnts) \
2131{ \
2132 int i; \
2133 for (i = 0; i < TCP_N_ERROR; i++) \
2134 if (cnts[i]) \
2135 tcp_inc_counter(node_id, i, cnts[i]); \
2136}
2137
2138
Dave Barach68b0fb02017-02-28 15:15:56 -05002139always_inline uword
2140tcp46_established_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
Florin Coras4df38712018-06-20 12:44:16 -07002141 vlib_frame_t * frame, int is_ip4)
Dave Barach68b0fb02017-02-28 15:15:56 -05002142{
Florin Coras4df38712018-06-20 12:44:16 -07002143 u32 thread_index = vm->thread_index, errors = 0;
Florin Coras9ece3c02018-11-05 11:06:53 -08002144 tcp_worker_ctx_t *wrk = tcp_get_worker (thread_index);
Florin Coras7ac053b2018-11-05 15:57:21 -08002145 u32 n_left_from, *from, *first_buffer;
Florin Coras00cd22d2018-04-18 13:20:18 -07002146 u16 err_counters[TCP_N_ERROR] = { 0 };
Dave Barach68b0fb02017-02-28 15:15:56 -05002147
Florin Coras4df38712018-06-20 12:44:16 -07002148 if (node->flags & VLIB_NODE_FLAG_TRACE)
2149 tcp_established_trace_frame (vm, node, frame, is_ip4);
2150
Florin Coras7ac053b2018-11-05 15:57:21 -08002151 first_buffer = from = vlib_frame_vector_args (frame);
Florin Coras4df38712018-06-20 12:44:16 -07002152 n_left_from = frame->n_vectors;
Dave Barach68b0fb02017-02-28 15:15:56 -05002153
2154 while (n_left_from > 0)
2155 {
Florin Coras7ac053b2018-11-05 15:57:21 -08002156 u32 bi0, error0 = TCP_ERROR_ACK_OK;
2157 vlib_buffer_t *b0;
Florin Coras3c514d52018-12-22 11:39:33 -08002158 tcp_header_t *th0;
Florin Coras7ac053b2018-11-05 15:57:21 -08002159 tcp_connection_t *tc0;
Dave Barach68b0fb02017-02-28 15:15:56 -05002160
Florin Coras7ac053b2018-11-05 15:57:21 -08002161 if (n_left_from > 1)
Dave Barach68b0fb02017-02-28 15:15:56 -05002162 {
Florin Coras7ac053b2018-11-05 15:57:21 -08002163 vlib_buffer_t *pb;
2164 pb = vlib_get_buffer (vm, from[1]);
2165 vlib_prefetch_buffer_header (pb, LOAD);
2166 CLIB_PREFETCH (pb->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
Dave Barach68b0fb02017-02-28 15:15:56 -05002167 }
2168
Florin Coras7ac053b2018-11-05 15:57:21 -08002169 bi0 = from[0];
2170 from += 1;
2171 n_left_from -= 1;
2172
2173 b0 = vlib_get_buffer (vm, bi0);
2174 tc0 = tcp_connection_get (vnet_buffer (b0)->tcp.connection_index,
2175 thread_index);
2176
2177 if (PREDICT_FALSE (tc0 == 0))
2178 {
2179 error0 = TCP_ERROR_INVALID_CONNECTION;
2180 goto done;
2181 }
2182
2183 th0 = tcp_buffer_hdr (b0);
Florin Coras7ac053b2018-11-05 15:57:21 -08002184
2185 /* TODO header prediction fast path */
2186
2187 /* 1-4: check SEQ, RST, SYN */
2188 if (PREDICT_FALSE (tcp_segment_validate (wrk, tc0, b0, th0, &error0)))
2189 {
2190 TCP_EVT_DBG (TCP_EVT_SEG_INVALID, tc0, vnet_buffer (b0)->tcp);
2191 goto done;
2192 }
2193
2194 /* 5: check the ACK field */
2195 if (PREDICT_FALSE (tcp_rcv_ack (wrk, tc0, b0, th0, &error0)))
2196 goto done;
2197
2198 /* 6: check the URG bit TODO */
2199
2200 /* 7: process the segment text */
2201 if (vnet_buffer (b0)->tcp.data_len)
2202 error0 = tcp_segment_rcv (wrk, tc0, b0);
2203
2204 /* 8: check the FIN bit */
Florin Coras3c514d52018-12-22 11:39:33 -08002205 if (PREDICT_FALSE (tcp_is_fin (th0)))
Florin Corasb11175d2018-11-09 14:34:08 -08002206 tcp_rcv_fin (wrk, tc0, b0, &error0);
Florin Coras7ac053b2018-11-05 15:57:21 -08002207
2208 done:
2209 tcp_inc_err_counter (err_counters, error0, 1);
Dave Barach68b0fb02017-02-28 15:15:56 -05002210 }
2211
Florin Coras31c99552019-03-01 13:00:58 -08002212 errors = session_main_flush_enqueue_events (TRANSPORT_PROTO_TCP,
2213 thread_index);
Florin Corasa9d5bea2018-12-17 08:24:19 -08002214 err_counters[TCP_ERROR_MSG_QUEUE_FULL] = errors;
Florin Coras00cd22d2018-04-18 13:20:18 -07002215 tcp_store_err_counters (established, err_counters);
Florin Coras9ece3c02018-11-05 11:06:53 -08002216 tcp_handle_postponed_dequeues (wrk);
Florin Corasb11175d2018-11-09 14:34:08 -08002217 tcp_handle_disconnects (wrk);
Florin Coras7ac053b2018-11-05 15:57:21 -08002218 vlib_buffer_free (vm, first_buffer, frame->n_vectors);
Florin Coras4df38712018-06-20 12:44:16 -07002219
2220 return frame->n_vectors;
Dave Barach68b0fb02017-02-28 15:15:56 -05002221}
2222
Filip Tehlare275bed2019-03-06 00:06:56 -08002223VLIB_NODE_FN (tcp4_established_node) (vlib_main_t * vm,
2224 vlib_node_runtime_t * node,
2225 vlib_frame_t * from_frame)
Dave Barach68b0fb02017-02-28 15:15:56 -05002226{
2227 return tcp46_established_inline (vm, node, from_frame, 1 /* is_ip4 */ );
2228}
2229
Filip Tehlare275bed2019-03-06 00:06:56 -08002230VLIB_NODE_FN (tcp6_established_node) (vlib_main_t * vm,
2231 vlib_node_runtime_t * node,
2232 vlib_frame_t * from_frame)
Dave Barach68b0fb02017-02-28 15:15:56 -05002233{
2234 return tcp46_established_inline (vm, node, from_frame, 0 /* is_ip4 */ );
2235}
2236
2237/* *INDENT-OFF* */
2238VLIB_REGISTER_NODE (tcp4_established_node) =
2239{
Dave Barach68b0fb02017-02-28 15:15:56 -05002240 .name = "tcp4-established",
2241 /* Takes a vector of packets. */
2242 .vector_size = sizeof (u32),
Florin Corase69f4952017-03-07 10:06:24 -08002243 .n_errors = TCP_N_ERROR,
2244 .error_strings = tcp_error_strings,
Dave Barach68b0fb02017-02-28 15:15:56 -05002245 .n_next_nodes = TCP_ESTABLISHED_N_NEXT,
2246 .next_nodes =
2247 {
2248#define _(s,n) [TCP_ESTABLISHED_NEXT_##s] = n,
2249 foreach_tcp_state_next
2250#undef _
2251 },
Clement Durand6cf260c2017-04-13 13:27:04 +02002252 .format_trace = format_tcp_rx_trace_short,
Dave Barach68b0fb02017-02-28 15:15:56 -05002253};
2254/* *INDENT-ON* */
2255
Dave Barach68b0fb02017-02-28 15:15:56 -05002256/* *INDENT-OFF* */
2257VLIB_REGISTER_NODE (tcp6_established_node) =
2258{
Dave Barach68b0fb02017-02-28 15:15:56 -05002259 .name = "tcp6-established",
2260 /* Takes a vector of packets. */
2261 .vector_size = sizeof (u32),
2262 .n_errors = TCP_N_ERROR,
2263 .error_strings = tcp_error_strings,
2264 .n_next_nodes = TCP_ESTABLISHED_N_NEXT,
2265 .next_nodes =
2266 {
2267#define _(s,n) [TCP_ESTABLISHED_NEXT_##s] = n,
2268 foreach_tcp_state_next
2269#undef _
2270 },
Clement Durand6cf260c2017-04-13 13:27:04 +02002271 .format_trace = format_tcp_rx_trace_short,
Dave Barach68b0fb02017-02-28 15:15:56 -05002272};
2273/* *INDENT-ON* */
2274
2275
Florin Coras4eeeaaf2017-09-05 14:03:37 -04002276static u8
2277tcp_lookup_is_valid (tcp_connection_t * tc, tcp_header_t * hdr)
2278{
Florin Corascea194d2017-10-02 00:18:51 -07002279 transport_connection_t *tmp = 0;
2280 u64 handle;
2281
Florin Coras4eeeaaf2017-09-05 14:03:37 -04002282 if (!tc)
2283 return 1;
2284
Florin Coras70131132017-11-27 02:43:30 -08002285 /* Proxy case */
2286 if (tc->c_lcl_port == 0 && tc->state == TCP_STATE_LISTEN)
2287 return 1;
2288
Florin Coras4eeeaaf2017-09-05 14:03:37 -04002289 u8 is_valid = (tc->c_lcl_port == hdr->dst_port
2290 && (tc->state == TCP_STATE_LISTEN
2291 || tc->c_rmt_port == hdr->src_port));
2292
2293 if (!is_valid)
2294 {
Florin Corascea194d2017-10-02 00:18:51 -07002295 handle = session_lookup_half_open_handle (&tc->connection);
2296 tmp = session_lookup_half_open_connection (handle & 0xFFFFFFFF,
Florin Coras3cbc04b2017-10-02 00:18:51 -07002297 tc->c_proto, tc->c_is_ip4);
Florin Corascea194d2017-10-02 00:18:51 -07002298
2299 if (tmp)
Florin Coras4eeeaaf2017-09-05 14:03:37 -04002300 {
2301 if (tmp->lcl_port == hdr->dst_port
2302 && tmp->rmt_port == hdr->src_port)
2303 {
Florin Corascea194d2017-10-02 00:18:51 -07002304 TCP_DBG ("half-open is valid!");
Florin Coras4eeeaaf2017-09-05 14:03:37 -04002305 }
2306 }
2307 }
2308 return is_valid;
2309}
2310
2311/**
2312 * Lookup transport connection
2313 */
2314static tcp_connection_t *
Florin Corascea194d2017-10-02 00:18:51 -07002315tcp_lookup_connection (u32 fib_index, vlib_buffer_t * b, u8 thread_index,
2316 u8 is_ip4)
Florin Coras4eeeaaf2017-09-05 14:03:37 -04002317{
2318 tcp_header_t *tcp;
2319 transport_connection_t *tconn;
2320 tcp_connection_t *tc;
Florin Corasdff48db2017-11-19 18:06:58 -08002321 u8 is_filtered = 0;
Florin Coras4eeeaaf2017-09-05 14:03:37 -04002322 if (is_ip4)
2323 {
2324 ip4_header_t *ip4;
2325 ip4 = vlib_buffer_get_current (b);
2326 tcp = ip4_next_header (ip4);
Florin Corascea194d2017-10-02 00:18:51 -07002327 tconn = session_lookup_connection_wt4 (fib_index,
2328 &ip4->dst_address,
2329 &ip4->src_address,
2330 tcp->dst_port,
2331 tcp->src_port,
2332 TRANSPORT_PROTO_TCP,
Florin Corasdff48db2017-11-19 18:06:58 -08002333 thread_index, &is_filtered);
Florin Coras4eeeaaf2017-09-05 14:03:37 -04002334 tc = tcp_get_connection_from_transport (tconn);
2335 ASSERT (tcp_lookup_is_valid (tc, tcp));
2336 }
2337 else
2338 {
2339 ip6_header_t *ip6;
2340 ip6 = vlib_buffer_get_current (b);
2341 tcp = ip6_next_header (ip6);
Florin Corascea194d2017-10-02 00:18:51 -07002342 tconn = session_lookup_connection_wt6 (fib_index,
2343 &ip6->dst_address,
2344 &ip6->src_address,
2345 tcp->dst_port,
2346 tcp->src_port,
2347 TRANSPORT_PROTO_TCP,
Florin Corasdff48db2017-11-19 18:06:58 -08002348 thread_index, &is_filtered);
Florin Coras4eeeaaf2017-09-05 14:03:37 -04002349 tc = tcp_get_connection_from_transport (tconn);
2350 ASSERT (tcp_lookup_is_valid (tc, tcp));
2351 }
2352 return tc;
2353}
2354
Dave Barach68b0fb02017-02-28 15:15:56 -05002355always_inline uword
2356tcp46_syn_sent_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
2357 vlib_frame_t * from_frame, int is_ip4)
2358{
2359 tcp_main_t *tm = vnet_get_tcp_main ();
Florin Coras7ac053b2018-11-05 15:57:21 -08002360 u32 n_left_from, *from, *first_buffer, errors = 0;
2361 u32 my_thread_index = vm->thread_index;
2362 tcp_worker_ctx_t *wrk = tcp_get_worker (my_thread_index);
Dave Barach68b0fb02017-02-28 15:15:56 -05002363
Florin Coras7ac053b2018-11-05 15:57:21 -08002364 from = first_buffer = vlib_frame_vector_args (from_frame);
Dave Barach68b0fb02017-02-28 15:15:56 -05002365 n_left_from = from_frame->n_vectors;
2366
Dave Barach68b0fb02017-02-28 15:15:56 -05002367 while (n_left_from > 0)
2368 {
Florin Coras7ac053b2018-11-05 15:57:21 -08002369 u32 bi0, ack0, seq0, error0 = TCP_ERROR_NONE;
2370 tcp_connection_t *tc0, *new_tc0;
2371 tcp_header_t *tcp0 = 0;
2372 tcp_rx_trace_t *t0;
2373 vlib_buffer_t *b0;
Dave Barach68b0fb02017-02-28 15:15:56 -05002374
Florin Coras7ac053b2018-11-05 15:57:21 -08002375 bi0 = from[0];
2376 from += 1;
2377 n_left_from -= 1;
Dave Barach68b0fb02017-02-28 15:15:56 -05002378
Florin Coras7ac053b2018-11-05 15:57:21 -08002379 b0 = vlib_get_buffer (vm, bi0);
2380 tc0 =
2381 tcp_half_open_connection_get (vnet_buffer (b0)->tcp.connection_index);
2382 if (PREDICT_FALSE (tc0 == 0))
Dave Barach68b0fb02017-02-28 15:15:56 -05002383 {
Florin Coras7ac053b2018-11-05 15:57:21 -08002384 error0 = TCP_ERROR_INVALID_CONNECTION;
2385 goto drop;
Dave Barach68b0fb02017-02-28 15:15:56 -05002386 }
2387
Florin Coras7ac053b2018-11-05 15:57:21 -08002388 /* Half-open completed recently but the connection was't removed
2389 * yet by the owning thread */
2390 if (PREDICT_FALSE (tc0->flags & TCP_CONN_HALF_OPEN_DONE))
2391 {
2392 /* Make sure the connection actually exists */
2393 ASSERT (tcp_lookup_connection (tc0->c_fib_index, b0,
2394 my_thread_index, is_ip4));
Florin Coras222e1f412019-02-16 20:47:32 -08002395 error0 = TCP_ERROR_SPURIOUS_SYN_ACK;
Florin Coras7ac053b2018-11-05 15:57:21 -08002396 goto drop;
2397 }
2398
2399 ack0 = vnet_buffer (b0)->tcp.ack_number;
2400 seq0 = vnet_buffer (b0)->tcp.seq_number;
2401 tcp0 = tcp_buffer_hdr (b0);
2402
2403 /* Crude check to see if the connection handle does not match
2404 * the packet. Probably connection just switched to established */
2405 if (PREDICT_FALSE (tcp0->dst_port != tc0->c_lcl_port
2406 || tcp0->src_port != tc0->c_rmt_port))
2407 {
2408 error0 = TCP_ERROR_INVALID_CONNECTION;
2409 goto drop;
2410 }
2411
2412 if (PREDICT_FALSE (!tcp_ack (tcp0) && !tcp_rst (tcp0)
2413 && !tcp_syn (tcp0)))
2414 {
2415 error0 = TCP_ERROR_SEGMENT_INVALID;
2416 goto drop;
2417 }
2418
Florin Coras3c514d52018-12-22 11:39:33 -08002419 /* SYNs consume sequence numbers */
2420 vnet_buffer (b0)->tcp.seq_end += tcp_is_syn (tcp0);
Florin Coras7ac053b2018-11-05 15:57:21 -08002421
2422 /*
2423 * 1. check the ACK bit
2424 */
2425
2426 /*
2427 * If the ACK bit is set
2428 * If SEG.ACK =< ISS, or SEG.ACK > SND.NXT, send a reset (unless
2429 * the RST bit is set, if so drop the segment and return)
2430 * <SEQ=SEG.ACK><CTL=RST>
2431 * and discard the segment. Return.
2432 * If SND.UNA =< SEG.ACK =< SND.NXT then the ACK is acceptable.
2433 */
2434 if (tcp_ack (tcp0))
2435 {
2436 if (seq_leq (ack0, tc0->iss) || seq_gt (ack0, tc0->snd_nxt))
2437 {
2438 if (!tcp_rst (tcp0))
Florin Corasd4c49be2019-02-07 00:15:53 -08002439 tcp_send_reset_w_pkt (tc0, b0, my_thread_index, is_ip4);
Florin Coras7ac053b2018-11-05 15:57:21 -08002440 error0 = TCP_ERROR_RCV_WND;
2441 goto drop;
2442 }
2443
2444 /* Make sure ACK is valid */
2445 if (seq_gt (tc0->snd_una, ack0))
2446 {
2447 error0 = TCP_ERROR_ACK_INVALID;
2448 goto drop;
2449 }
2450 }
2451
2452 /*
2453 * 2. check the RST bit
2454 */
2455
2456 if (tcp_rst (tcp0))
2457 {
2458 /* If ACK is acceptable, signal client that peer is not
2459 * willing to accept connection and drop connection*/
2460 if (tcp_ack (tcp0))
2461 tcp_connection_reset (tc0);
2462 error0 = TCP_ERROR_RST_RCVD;
2463 goto drop;
2464 }
2465
2466 /*
2467 * 3. check the security and precedence (skipped)
2468 */
2469
2470 /*
2471 * 4. check the SYN bit
2472 */
2473
2474 /* No SYN flag. Drop. */
2475 if (!tcp_syn (tcp0))
2476 {
Florin Coras7ac053b2018-11-05 15:57:21 -08002477 error0 = TCP_ERROR_SEGMENT_INVALID;
2478 goto drop;
2479 }
2480
2481 /* Parse options */
Florin Coras80231112018-12-05 15:59:31 -08002482 if (tcp_options_parse (tcp0, &tc0->rcv_opts, 1))
Florin Coras7ac053b2018-11-05 15:57:21 -08002483 {
Florin Coras7ac053b2018-11-05 15:57:21 -08002484 error0 = TCP_ERROR_OPTIONS;
2485 goto drop;
2486 }
2487
2488 /* Valid SYN or SYN-ACK. Move connection from half-open pool to
2489 * current thread pool. */
2490 pool_get (tm->connections[my_thread_index], new_tc0);
Dave Barach178cf492018-11-13 16:34:13 -05002491 clib_memcpy_fast (new_tc0, tc0, sizeof (*new_tc0));
Florin Coras7ac053b2018-11-05 15:57:21 -08002492 new_tc0->c_c_index = new_tc0 - tm->connections[my_thread_index];
2493 new_tc0->c_thread_index = my_thread_index;
2494 new_tc0->rcv_nxt = vnet_buffer (b0)->tcp.seq_end;
2495 new_tc0->irs = seq0;
Florin Coras85a3ddd2018-12-24 16:54:34 -08002496 new_tc0->timers[TCP_TIMER_ESTABLISH_AO] = TCP_TIMER_HANDLE_INVALID;
Florin Coras7ac053b2018-11-05 15:57:21 -08002497 new_tc0->timers[TCP_TIMER_RETRANSMIT_SYN] = TCP_TIMER_HANDLE_INVALID;
2498 new_tc0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
2499
2500 /* If this is not the owning thread, wait for syn retransmit to
2501 * expire and cleanup then */
2502 if (tcp_half_open_connection_cleanup (tc0))
2503 tc0->flags |= TCP_CONN_HALF_OPEN_DONE;
2504
2505 if (tcp_opts_tstamp (&new_tc0->rcv_opts))
2506 {
2507 new_tc0->tsval_recent = new_tc0->rcv_opts.tsval;
2508 new_tc0->tsval_recent_age = tcp_time_now ();
2509 }
2510
2511 if (tcp_opts_wscale (&new_tc0->rcv_opts))
2512 new_tc0->snd_wscale = new_tc0->rcv_opts.wscale;
Florin Corase80b5912018-12-12 19:25:43 -08002513 else
2514 new_tc0->rcv_wscale = 0;
Florin Coras7ac053b2018-11-05 15:57:21 -08002515
2516 new_tc0->snd_wnd = clib_net_to_host_u16 (tcp0->window)
2517 << new_tc0->snd_wscale;
2518 new_tc0->snd_wl1 = seq0;
2519 new_tc0->snd_wl2 = ack0;
2520
2521 tcp_connection_init_vars (new_tc0);
2522
2523 /* SYN-ACK: See if we can switch to ESTABLISHED state */
2524 if (PREDICT_TRUE (tcp_ack (tcp0)))
2525 {
2526 /* Our SYN is ACKed: we have iss < ack = snd_una */
2527
2528 /* TODO Dequeue acknowledged segments if we support Fast Open */
2529 new_tc0->snd_una = ack0;
2530 new_tc0->state = TCP_STATE_ESTABLISHED;
2531
2532 /* Make sure las is initialized for the wnd computation */
2533 new_tc0->rcv_las = new_tc0->rcv_nxt;
2534
2535 /* Notify app that we have connection. If session layer can't
2536 * allocate session send reset */
2537 if (session_stream_connect_notify (&new_tc0->connection, 0))
2538 {
2539 clib_warning ("connect notify fail");
Florin Corasd4c49be2019-02-07 00:15:53 -08002540 tcp_send_reset_w_pkt (new_tc0, b0, my_thread_index, is_ip4);
Florin Coras7ac053b2018-11-05 15:57:21 -08002541 tcp_connection_cleanup (new_tc0);
2542 goto drop;
2543 }
2544
Florin Coras2e31cc32018-09-25 14:00:34 -07002545 new_tc0->tx_fifo_size =
2546 transport_tx_fifo_size (&new_tc0->connection);
Florin Coras7ac053b2018-11-05 15:57:21 -08002547 /* Update rtt with the syn-ack sample */
Florin Corasefefc6b2018-11-07 12:49:19 -08002548 tcp_estimate_initial_rtt (new_tc0);
Florin Coras7ac053b2018-11-05 15:57:21 -08002549 TCP_EVT_DBG (TCP_EVT_SYNACK_RCVD, new_tc0);
2550 error0 = TCP_ERROR_SYN_ACKS_RCVD;
2551 }
2552 /* SYN: Simultaneous open. Change state to SYN-RCVD and send SYN-ACK */
2553 else
2554 {
2555 new_tc0->state = TCP_STATE_SYN_RCVD;
2556
2557 /* Notify app that we have connection */
2558 if (session_stream_connect_notify (&new_tc0->connection, 0))
2559 {
2560 tcp_connection_cleanup (new_tc0);
Florin Corasd4c49be2019-02-07 00:15:53 -08002561 tcp_send_reset_w_pkt (tc0, b0, my_thread_index, is_ip4);
Florin Coras7ac053b2018-11-05 15:57:21 -08002562 TCP_EVT_DBG (TCP_EVT_RST_SENT, tc0);
2563 goto drop;
2564 }
2565
Florin Coras2e31cc32018-09-25 14:00:34 -07002566 new_tc0->tx_fifo_size =
2567 transport_tx_fifo_size (&new_tc0->connection);
2568 new_tc0->rtt_ts = 0;
2569 tcp_init_snd_vars (new_tc0);
Florin Coras7ac053b2018-11-05 15:57:21 -08002570 tcp_send_synack (new_tc0);
2571 error0 = TCP_ERROR_SYNS_RCVD;
2572 goto drop;
2573 }
2574
2575 /* Read data, if any */
2576 if (PREDICT_FALSE (vnet_buffer (b0)->tcp.data_len))
2577 {
2578 clib_warning ("rcvd data in syn-sent");
2579 error0 = tcp_segment_rcv (wrk, new_tc0, b0);
2580 if (error0 == TCP_ERROR_ACK_OK)
2581 error0 = TCP_ERROR_SYN_ACKS_RCVD;
2582 }
2583 else
2584 {
2585 tcp_program_ack (wrk, new_tc0);
2586 }
2587
2588 drop:
2589
2590 tcp_inc_counter (syn_sent, error0, 1);
2591 if (PREDICT_FALSE ((b0->flags & VLIB_BUFFER_IS_TRACED) && tcp0 != 0))
2592 {
2593 t0 = vlib_add_trace (vm, node, b0, sizeof (*t0));
Dave Barach178cf492018-11-13 16:34:13 -05002594 clib_memcpy_fast (&t0->tcp_header, tcp0, sizeof (t0->tcp_header));
2595 clib_memcpy_fast (&t0->tcp_connection, tc0,
2596 sizeof (t0->tcp_connection));
Florin Coras7ac053b2018-11-05 15:57:21 -08002597 }
Dave Barach68b0fb02017-02-28 15:15:56 -05002598 }
2599
Florin Coras31c99552019-03-01 13:00:58 -08002600 errors = session_main_flush_enqueue_events (TRANSPORT_PROTO_TCP,
2601 my_thread_index);
Florin Corasa9d5bea2018-12-17 08:24:19 -08002602 tcp_inc_counter (syn_sent, TCP_ERROR_MSG_QUEUE_FULL, errors);
Florin Coras7ac053b2018-11-05 15:57:21 -08002603 vlib_buffer_free (vm, first_buffer, from_frame->n_vectors);
2604
Dave Barach68b0fb02017-02-28 15:15:56 -05002605 return from_frame->n_vectors;
2606}
2607
Filip Tehlare275bed2019-03-06 00:06:56 -08002608VLIB_NODE_FN (tcp4_syn_sent_node) (vlib_main_t * vm,
2609 vlib_node_runtime_t * node,
2610 vlib_frame_t * from_frame)
Dave Barach68b0fb02017-02-28 15:15:56 -05002611{
2612 return tcp46_syn_sent_inline (vm, node, from_frame, 1 /* is_ip4 */ );
2613}
2614
Filip Tehlare275bed2019-03-06 00:06:56 -08002615VLIB_NODE_FN (tcp6_syn_sent_node) (vlib_main_t * vm,
2616 vlib_node_runtime_t * node,
2617 vlib_frame_t * from_frame)
Dave Barach68b0fb02017-02-28 15:15:56 -05002618{
2619 return tcp46_syn_sent_inline (vm, node, from_frame, 0 /* is_ip4 */ );
2620}
2621
2622/* *INDENT-OFF* */
2623VLIB_REGISTER_NODE (tcp4_syn_sent_node) =
2624{
Dave Barach68b0fb02017-02-28 15:15:56 -05002625 .name = "tcp4-syn-sent",
2626 /* Takes a vector of packets. */
2627 .vector_size = sizeof (u32),
2628 .n_errors = TCP_N_ERROR,
2629 .error_strings = tcp_error_strings,
2630 .n_next_nodes = TCP_SYN_SENT_N_NEXT,
2631 .next_nodes =
2632 {
2633#define _(s,n) [TCP_SYN_SENT_NEXT_##s] = n,
2634 foreach_tcp_state_next
2635#undef _
2636 },
Clement Durand6cf260c2017-04-13 13:27:04 +02002637 .format_trace = format_tcp_rx_trace_short,
Dave Barach68b0fb02017-02-28 15:15:56 -05002638};
2639/* *INDENT-ON* */
2640
Dave Barach68b0fb02017-02-28 15:15:56 -05002641/* *INDENT-OFF* */
2642VLIB_REGISTER_NODE (tcp6_syn_sent_node) =
2643{
Dave Barach68b0fb02017-02-28 15:15:56 -05002644 .name = "tcp6-syn-sent",
2645 /* Takes a vector of packets. */
2646 .vector_size = sizeof (u32),
2647 .n_errors = TCP_N_ERROR,
2648 .error_strings = tcp_error_strings,
2649 .n_next_nodes = TCP_SYN_SENT_N_NEXT,
2650 .next_nodes =
2651 {
2652#define _(s,n) [TCP_SYN_SENT_NEXT_##s] = n,
2653 foreach_tcp_state_next
2654#undef _
Clement Durand6cf260c2017-04-13 13:27:04 +02002655 },
2656 .format_trace = format_tcp_rx_trace_short,
2657};
Dave Barach68b0fb02017-02-28 15:15:56 -05002658/* *INDENT-ON* */
2659
Dave Barach68b0fb02017-02-28 15:15:56 -05002660/**
Florin Corasd79b41e2017-03-04 05:37:52 -08002661 * Handles reception for all states except LISTEN, SYN-SENT and ESTABLISHED
Dave Barach68b0fb02017-02-28 15:15:56 -05002662 * as per RFC793 p. 64
2663 */
2664always_inline uword
2665tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
2666 vlib_frame_t * from_frame, int is_ip4)
2667{
Florin Coras7ac053b2018-11-05 15:57:21 -08002668 u32 thread_index = vm->thread_index, errors = 0, *first_buffer;
2669 tcp_worker_ctx_t *wrk = tcp_get_worker (thread_index);
Florin Coras78cc4b02018-12-20 18:24:49 -08002670 u32 n_left_from, *from, max_dequeue;
Dave Barach68b0fb02017-02-28 15:15:56 -05002671
Florin Coras7ac053b2018-11-05 15:57:21 -08002672 from = first_buffer = vlib_frame_vector_args (from_frame);
Dave Barach68b0fb02017-02-28 15:15:56 -05002673 n_left_from = from_frame->n_vectors;
Dave Barach68b0fb02017-02-28 15:15:56 -05002674
2675 while (n_left_from > 0)
2676 {
Florin Coras7ac053b2018-11-05 15:57:21 -08002677 u32 bi0, error0 = TCP_ERROR_NONE;
2678 tcp_header_t *tcp0 = 0;
2679 tcp_connection_t *tc0;
2680 vlib_buffer_t *b0;
2681 u8 is_fin0;
Dave Barach68b0fb02017-02-28 15:15:56 -05002682
Florin Coras7ac053b2018-11-05 15:57:21 -08002683 bi0 = from[0];
2684 from += 1;
2685 n_left_from -= 1;
Dave Barach68b0fb02017-02-28 15:15:56 -05002686
Florin Coras7ac053b2018-11-05 15:57:21 -08002687 b0 = vlib_get_buffer (vm, bi0);
2688 tc0 = tcp_connection_get (vnet_buffer (b0)->tcp.connection_index,
2689 thread_index);
2690 if (PREDICT_FALSE (tc0 == 0))
Dave Barach68b0fb02017-02-28 15:15:56 -05002691 {
Florin Coras7ac053b2018-11-05 15:57:21 -08002692 error0 = TCP_ERROR_INVALID_CONNECTION;
2693 goto drop;
Dave Barach68b0fb02017-02-28 15:15:56 -05002694 }
2695
Florin Coras7ac053b2018-11-05 15:57:21 -08002696 tcp0 = tcp_buffer_hdr (b0);
2697 is_fin0 = tcp_is_fin (tcp0);
2698
Florin Coras7ac053b2018-11-05 15:57:21 -08002699 if (CLIB_DEBUG)
2700 {
2701 tcp_connection_t *tmp;
2702 tmp = tcp_lookup_connection (tc0->c_fib_index, b0, thread_index,
2703 is_ip4);
2704 if (tmp->state != tc0->state)
2705 {
Florin Corasb0f662f2018-12-27 14:51:46 -08002706 if (tc0->state != TCP_STATE_CLOSED)
2707 clib_warning ("state changed");
Florin Coras7ac053b2018-11-05 15:57:21 -08002708 goto drop;
2709 }
2710 }
2711
2712 /*
2713 * Special treatment for CLOSED
2714 */
2715 if (PREDICT_FALSE (tc0->state == TCP_STATE_CLOSED))
2716 {
2717 error0 = TCP_ERROR_CONNECTION_CLOSED;
2718 goto drop;
2719 }
2720
2721 /*
2722 * For all other states (except LISTEN)
2723 */
2724
2725 /* 1-4: check SEQ, RST, SYN */
2726 if (PREDICT_FALSE (tcp_segment_validate (wrk, tc0, b0, tcp0, &error0)))
2727 goto drop;
2728
2729 /* 5: check the ACK field */
2730 switch (tc0->state)
2731 {
2732 case TCP_STATE_SYN_RCVD:
2733 /*
2734 * If the segment acknowledgment is not acceptable, form a
2735 * reset segment,
2736 * <SEQ=SEG.ACK><CTL=RST>
2737 * and send it.
2738 */
2739 if (!tcp_rcv_ack_is_acceptable (tc0, b0))
2740 {
Florin Corase78ac9d2018-12-16 21:33:00 -08002741 tcp_connection_reset (tc0);
Florin Coras7ac053b2018-11-05 15:57:21 -08002742 error0 = TCP_ERROR_ACK_INVALID;
2743 goto drop;
2744 }
2745
Florin Coras4850e3e2018-12-12 14:34:38 -08002746 /* Make sure the ack is exactly right */
Florin Coras865872e2019-01-18 12:12:29 -08002747 if (tc0->rcv_nxt != vnet_buffer (b0)->tcp.seq_number || is_fin0)
Florin Coras4850e3e2018-12-12 14:34:38 -08002748 {
Florin Corase78ac9d2018-12-16 21:33:00 -08002749 tcp_connection_reset (tc0);
Florin Coras4850e3e2018-12-12 14:34:38 -08002750 error0 = TCP_ERROR_SEGMENT_INVALID;
Florin Coras4850e3e2018-12-12 14:34:38 -08002751 goto drop;
2752 }
2753
Florin Coras7ac053b2018-11-05 15:57:21 -08002754 /* Update rtt and rto */
Florin Corasefefc6b2018-11-07 12:49:19 -08002755 tcp_estimate_initial_rtt (tc0);
Florin Coras7ac053b2018-11-05 15:57:21 -08002756
2757 /* Switch state to ESTABLISHED */
2758 tc0->state = TCP_STATE_ESTABLISHED;
2759 TCP_EVT_DBG (TCP_EVT_STATE_CHANGE, tc0);
2760
2761 /* Initialize session variables */
2762 tc0->snd_una = vnet_buffer (b0)->tcp.ack_number;
2763 tc0->snd_wnd = clib_net_to_host_u16 (tcp0->window)
2764 << tc0->rcv_opts.wscale;
2765 tc0->snd_wl1 = vnet_buffer (b0)->tcp.seq_number;
2766 tc0->snd_wl2 = vnet_buffer (b0)->tcp.ack_number;
2767
2768 /* Reset SYN-ACK retransmit and SYN_RCV establish timers */
2769 tcp_retransmit_timer_reset (tc0);
2770 tcp_timer_reset (tc0, TCP_TIMER_ESTABLISH);
Florin Corasa27a46e2019-02-18 13:02:28 -08002771 if (session_stream_accept_notify (&tc0->connection))
Florin Corasa9d5bea2018-12-17 08:24:19 -08002772 {
2773 error0 = TCP_ERROR_MSG_QUEUE_FULL;
2774 tcp_connection_reset (tc0);
2775 goto drop;
2776 }
Florin Coras7ac053b2018-11-05 15:57:21 -08002777 error0 = TCP_ERROR_ACK_OK;
2778 break;
2779 case TCP_STATE_ESTABLISHED:
2780 /* We can get packets in established state here because they
2781 * were enqueued before state change */
2782 if (tcp_rcv_ack (wrk, tc0, b0, tcp0, &error0))
2783 goto drop;
2784
2785 break;
2786 case TCP_STATE_FIN_WAIT_1:
2787 /* In addition to the processing for the ESTABLISHED state, if
2788 * our FIN is now acknowledged then enter FIN-WAIT-2 and
2789 * continue processing in that state. */
2790 if (tcp_rcv_ack (wrk, tc0, b0, tcp0, &error0))
2791 goto drop;
2792
2793 /* Still have to send the FIN */
2794 if (tc0->flags & TCP_CONN_FINPNDG)
2795 {
2796 /* TX fifo finally drained */
Florin Coras31c99552019-03-01 13:00:58 -08002797 max_dequeue = transport_max_tx_dequeue (&tc0->connection);
Florin Coras78cc4b02018-12-20 18:24:49 -08002798 if (max_dequeue <= tc0->burst_acked)
Florin Coras7ac053b2018-11-05 15:57:21 -08002799 tcp_send_fin (tc0);
2800 }
2801 /* If FIN is ACKed */
2802 else if (tc0->snd_una == tc0->snd_una_max)
2803 {
Florin Coras3c514d52018-12-22 11:39:33 -08002804 tcp_connection_set_state (tc0, TCP_STATE_FIN_WAIT_2);
Florin Coras7ac053b2018-11-05 15:57:21 -08002805
2806 /* Stop all retransmit timers because we have nothing more
2807 * to send. Enable waitclose though because we're willing to
2808 * wait for peer's FIN but not indefinitely. */
2809 tcp_connection_timers_reset (tc0);
Florin Coras85a3ddd2018-12-24 16:54:34 -08002810 tcp_timer_set (tc0, TCP_TIMER_WAITCLOSE, TCP_2MSL_TIME);
Florin Corasefefc6b2018-11-07 12:49:19 -08002811
2812 /* Don't try to deq the FIN acked */
2813 if (tc0->burst_acked > 1)
Florin Coras31c99552019-03-01 13:00:58 -08002814 session_tx_fifo_dequeue_drop (&tc0->connection,
2815 tc0->burst_acked - 1);
Florin Corasefefc6b2018-11-07 12:49:19 -08002816 tc0->burst_acked = 0;
Florin Coras7ac053b2018-11-05 15:57:21 -08002817 }
2818 break;
2819 case TCP_STATE_FIN_WAIT_2:
2820 /* In addition to the processing for the ESTABLISHED state, if
2821 * the retransmission queue is empty, the user's CLOSE can be
2822 * acknowledged ("ok") but do not delete the TCB. */
2823 if (tcp_rcv_ack (wrk, tc0, b0, tcp0, &error0))
2824 goto drop;
Florin Corasefefc6b2018-11-07 12:49:19 -08002825 tc0->burst_acked = 0;
Florin Coras7ac053b2018-11-05 15:57:21 -08002826 break;
2827 case TCP_STATE_CLOSE_WAIT:
2828 /* Do the same processing as for the ESTABLISHED state. */
2829 if (tcp_rcv_ack (wrk, tc0, b0, tcp0, &error0))
2830 goto drop;
2831
2832 if (tc0->flags & TCP_CONN_FINPNDG)
2833 {
2834 /* TX fifo finally drained */
Florin Coras31c99552019-03-01 13:00:58 -08002835 if (!transport_max_tx_dequeue (&tc0->connection))
Florin Coras7ac053b2018-11-05 15:57:21 -08002836 {
2837 tcp_send_fin (tc0);
2838 tcp_connection_timers_reset (tc0);
Florin Coras3c514d52018-12-22 11:39:33 -08002839 tcp_connection_set_state (tc0, TCP_STATE_LAST_ACK);
Florin Coras85a3ddd2018-12-24 16:54:34 -08002840 tcp_timer_set (tc0, TCP_TIMER_WAITCLOSE, TCP_2MSL_TIME);
Florin Coras7ac053b2018-11-05 15:57:21 -08002841 }
2842 }
2843 break;
2844 case TCP_STATE_CLOSING:
2845 /* In addition to the processing for the ESTABLISHED state, if
2846 * the ACK acknowledges our FIN then enter the TIME-WAIT state,
2847 * otherwise ignore the segment. */
2848 if (tcp_rcv_ack (wrk, tc0, b0, tcp0, &error0))
2849 goto drop;
2850
Florin Coras85a3ddd2018-12-24 16:54:34 -08002851 tcp_connection_timers_reset (tc0);
Florin Coras3c514d52018-12-22 11:39:33 -08002852 tcp_connection_set_state (tc0, TCP_STATE_TIME_WAIT);
Florin Coras85a3ddd2018-12-24 16:54:34 -08002853 tcp_timer_set (tc0, TCP_TIMER_WAITCLOSE, TCP_TIMEWAIT_TIME);
Florin Coras7ac053b2018-11-05 15:57:21 -08002854 goto drop;
2855
2856 break;
2857 case TCP_STATE_LAST_ACK:
2858 /* The only thing that [should] arrive in this state is an
2859 * acknowledgment of our FIN. If our FIN is now acknowledged,
2860 * delete the TCB, enter the CLOSED state, and return. */
2861
2862 if (!tcp_rcv_ack_is_acceptable (tc0, b0))
2863 {
2864 error0 = TCP_ERROR_ACK_INVALID;
2865 goto drop;
2866 }
2867 error0 = TCP_ERROR_ACK_OK;
2868 tc0->snd_una = vnet_buffer (b0)->tcp.ack_number;
2869 /* Apparently our ACK for the peer's FIN was lost */
2870 if (is_fin0 && tc0->snd_una != tc0->snd_una_max)
2871 {
2872 tcp_send_fin (tc0);
2873 goto drop;
2874 }
2875
Florin Coras3c514d52018-12-22 11:39:33 -08002876 tcp_connection_set_state (tc0, TCP_STATE_CLOSED);
Florin Coras7ac053b2018-11-05 15:57:21 -08002877
2878 /* Don't free the connection from the data path since
2879 * we can't ensure that we have no packets already enqueued
2880 * to output. Rely instead on the waitclose timer */
2881 tcp_connection_timers_reset (tc0);
Florin Coras85a3ddd2018-12-24 16:54:34 -08002882 tcp_timer_set (tc0, TCP_TIMER_WAITCLOSE, TCP_CLEANUP_TIME);
Florin Coras7ac053b2018-11-05 15:57:21 -08002883
2884 goto drop;
2885
2886 break;
2887 case TCP_STATE_TIME_WAIT:
2888 /* The only thing that can arrive in this state is a
2889 * retransmission of the remote FIN. Acknowledge it, and restart
2890 * the 2 MSL timeout. */
2891
2892 if (tcp_rcv_ack (wrk, tc0, b0, tcp0, &error0))
2893 goto drop;
2894
Florin Coras37db4302019-03-13 13:25:57 -07002895 if (!is_fin0)
2896 goto drop;
2897
Florin Coras7ac053b2018-11-05 15:57:21 -08002898 tcp_program_ack (wrk, tc0);
2899 tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE, TCP_TIMEWAIT_TIME);
2900 goto drop;
2901
2902 break;
2903 default:
2904 ASSERT (0);
2905 }
2906
2907 /* 6: check the URG bit TODO */
2908
2909 /* 7: process the segment text */
2910 switch (tc0->state)
2911 {
2912 case TCP_STATE_ESTABLISHED:
2913 case TCP_STATE_FIN_WAIT_1:
2914 case TCP_STATE_FIN_WAIT_2:
2915 if (vnet_buffer (b0)->tcp.data_len)
2916 error0 = tcp_segment_rcv (wrk, tc0, b0);
Florin Coras7ac053b2018-11-05 15:57:21 -08002917 break;
2918 case TCP_STATE_CLOSE_WAIT:
2919 case TCP_STATE_CLOSING:
2920 case TCP_STATE_LAST_ACK:
2921 case TCP_STATE_TIME_WAIT:
2922 /* This should not occur, since a FIN has been received from the
2923 * remote side. Ignore the segment text. */
2924 break;
2925 }
2926
2927 /* 8: check the FIN bit */
2928 if (!is_fin0)
2929 goto drop;
2930
Florin Coras3c514d52018-12-22 11:39:33 -08002931 TCP_EVT_DBG (TCP_EVT_FIN_RCVD, tc0);
2932
Florin Coras7ac053b2018-11-05 15:57:21 -08002933 switch (tc0->state)
2934 {
2935 case TCP_STATE_ESTABLISHED:
Florin Coras3c514d52018-12-22 11:39:33 -08002936 /* Account for the FIN and send ack */
2937 tc0->rcv_nxt += 1;
2938 tcp_program_ack (wrk, tc0);
2939 tcp_connection_set_state (tc0, TCP_STATE_CLOSE_WAIT);
2940 tcp_program_disconnect (wrk, tc0);
2941 tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE, TCP_CLOSEWAIT_TIME);
Florin Coras5c0f1662018-12-19 01:38:57 -08002942 break;
Florin Coras7ac053b2018-11-05 15:57:21 -08002943 case TCP_STATE_SYN_RCVD:
Florin Coras5c0f1662018-12-19 01:38:57 -08002944 /* Send FIN-ACK, enter LAST-ACK and because the app was not
2945 * notified yet, set a cleanup timer instead of relying on
2946 * disconnect notify and the implicit close call. */
Florin Coras7ac053b2018-11-05 15:57:21 -08002947 tcp_connection_timers_reset (tc0);
Florin Coras5c0f1662018-12-19 01:38:57 -08002948 tc0->rcv_nxt += 1;
Florin Coras7ac053b2018-11-05 15:57:21 -08002949 tcp_send_fin (tc0);
Florin Coras3c514d52018-12-22 11:39:33 -08002950 tcp_connection_set_state (tc0, TCP_STATE_LAST_ACK);
Florin Coras5c0f1662018-12-19 01:38:57 -08002951 tcp_timer_set (tc0, TCP_TIMER_WAITCLOSE, TCP_2MSL_TIME);
Florin Coras7ac053b2018-11-05 15:57:21 -08002952 break;
2953 case TCP_STATE_CLOSE_WAIT:
2954 case TCP_STATE_CLOSING:
2955 case TCP_STATE_LAST_ACK:
2956 /* move along .. */
2957 break;
2958 case TCP_STATE_FIN_WAIT_1:
Florin Coras3c514d52018-12-22 11:39:33 -08002959 tc0->rcv_nxt += 1;
2960 tcp_connection_set_state (tc0, TCP_STATE_CLOSING);
Florin Coras565115e2019-02-20 19:48:31 -08002961 if (tc0->flags & TCP_CONN_FINPNDG)
2962 {
2963 /* Drop all outstanding tx data. */
Florin Coras31c99552019-03-01 13:00:58 -08002964 session_tx_fifo_dequeue_drop (&tc0->connection,
2965 transport_max_tx_dequeue
2966 (&tc0->connection));
Florin Coras47a9c652019-03-12 20:37:09 -07002967 /* Make it look as if we've recovered, if needed */
2968 if (tcp_in_cong_recovery (tc0))
2969 {
2970 scoreboard_clear (&tc0->sack_sb);
2971 tcp_fastrecovery_off (tc0);
2972 tcp_recovery_off (tc0);
Florin Coras37db4302019-03-13 13:25:57 -07002973 tcp_connection_timers_reset (tc0);
Florin Coras47a9c652019-03-12 20:37:09 -07002974 tc0->snd_nxt = tc0->snd_una_max = tc0->snd_una;
2975 }
Florin Coras565115e2019-02-20 19:48:31 -08002976 tcp_send_fin (tc0);
2977 }
2978 else
2979 tcp_program_ack (wrk, tc0);
2980 /* Wait for ACK for our FIN but not forever */
Florin Coras7ac053b2018-11-05 15:57:21 -08002981 tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE, TCP_2MSL_TIME);
2982 break;
2983 case TCP_STATE_FIN_WAIT_2:
2984 /* Got FIN, send ACK! Be more aggressive with resource cleanup */
Florin Coras3c514d52018-12-22 11:39:33 -08002985 tc0->rcv_nxt += 1;
2986 tcp_connection_set_state (tc0, TCP_STATE_TIME_WAIT);
Florin Coras7ac053b2018-11-05 15:57:21 -08002987 tcp_connection_timers_reset (tc0);
Florin Coras85a3ddd2018-12-24 16:54:34 -08002988 tcp_timer_set (tc0, TCP_TIMER_WAITCLOSE, TCP_TIMEWAIT_TIME);
Florin Coras7ac053b2018-11-05 15:57:21 -08002989 tcp_program_ack (wrk, tc0);
Florin Coras7ac053b2018-11-05 15:57:21 -08002990 break;
2991 case TCP_STATE_TIME_WAIT:
2992 /* Remain in the TIME-WAIT state. Restart the time-wait
2993 * timeout.
2994 */
2995 tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE, TCP_TIMEWAIT_TIME);
2996 break;
2997 }
2998 error0 = TCP_ERROR_FIN_RCVD;
Florin Coras7ac053b2018-11-05 15:57:21 -08002999
3000 drop:
3001
3002 tcp_inc_counter (rcv_process, error0, 1);
3003 if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
3004 {
3005 tcp_rx_trace_t *t0 = vlib_add_trace (vm, node, b0, sizeof (*t0));
3006 tcp_set_rx_trace_data (t0, tc0, tcp0, b0, is_ip4);
3007 }
Dave Barach68b0fb02017-02-28 15:15:56 -05003008 }
3009
Florin Coras31c99552019-03-01 13:00:58 -08003010 errors = session_main_flush_enqueue_events (TRANSPORT_PROTO_TCP,
3011 thread_index);
Florin Corasa9d5bea2018-12-17 08:24:19 -08003012 tcp_inc_counter (rcv_process, TCP_ERROR_MSG_QUEUE_FULL, errors);
Florin Coras9ece3c02018-11-05 11:06:53 -08003013 tcp_handle_postponed_dequeues (wrk);
Florin Coras7ac053b2018-11-05 15:57:21 -08003014 vlib_buffer_free (vm, first_buffer, from_frame->n_vectors);
3015
Dave Barach68b0fb02017-02-28 15:15:56 -05003016 return from_frame->n_vectors;
3017}
3018
Filip Tehlare275bed2019-03-06 00:06:56 -08003019VLIB_NODE_FN (tcp4_rcv_process_node) (vlib_main_t * vm,
3020 vlib_node_runtime_t * node,
3021 vlib_frame_t * from_frame)
Dave Barach68b0fb02017-02-28 15:15:56 -05003022{
3023 return tcp46_rcv_process_inline (vm, node, from_frame, 1 /* is_ip4 */ );
3024}
3025
Filip Tehlare275bed2019-03-06 00:06:56 -08003026VLIB_NODE_FN (tcp6_rcv_process_node) (vlib_main_t * vm,
3027 vlib_node_runtime_t * node,
3028 vlib_frame_t * from_frame)
Dave Barach68b0fb02017-02-28 15:15:56 -05003029{
3030 return tcp46_rcv_process_inline (vm, node, from_frame, 0 /* is_ip4 */ );
3031}
3032
3033/* *INDENT-OFF* */
3034VLIB_REGISTER_NODE (tcp4_rcv_process_node) =
3035{
Dave Barach68b0fb02017-02-28 15:15:56 -05003036 .name = "tcp4-rcv-process",
3037 /* Takes a vector of packets. */
3038 .vector_size = sizeof (u32),
3039 .n_errors = TCP_N_ERROR,
3040 .error_strings = tcp_error_strings,
3041 .n_next_nodes = TCP_RCV_PROCESS_N_NEXT,
3042 .next_nodes =
3043 {
3044#define _(s,n) [TCP_RCV_PROCESS_NEXT_##s] = n,
3045 foreach_tcp_state_next
3046#undef _
3047 },
Clement Durand6cf260c2017-04-13 13:27:04 +02003048 .format_trace = format_tcp_rx_trace_short,
Dave Barach68b0fb02017-02-28 15:15:56 -05003049};
3050/* *INDENT-ON* */
3051
Dave Barach68b0fb02017-02-28 15:15:56 -05003052/* *INDENT-OFF* */
3053VLIB_REGISTER_NODE (tcp6_rcv_process_node) =
3054{
Dave Barach68b0fb02017-02-28 15:15:56 -05003055 .name = "tcp6-rcv-process",
3056 /* Takes a vector of packets. */
3057 .vector_size = sizeof (u32),
3058 .n_errors = TCP_N_ERROR,
3059 .error_strings = tcp_error_strings,
3060 .n_next_nodes = TCP_RCV_PROCESS_N_NEXT,
3061 .next_nodes =
3062 {
3063#define _(s,n) [TCP_RCV_PROCESS_NEXT_##s] = n,
3064 foreach_tcp_state_next
3065#undef _
3066 },
Clement Durand6cf260c2017-04-13 13:27:04 +02003067 .format_trace = format_tcp_rx_trace_short,
Dave Barach68b0fb02017-02-28 15:15:56 -05003068};
3069/* *INDENT-ON* */
3070
Dave Barach68b0fb02017-02-28 15:15:56 -05003071/**
3072 * LISTEN state processing as per RFC 793 p. 65
3073 */
3074always_inline uword
3075tcp46_listen_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
3076 vlib_frame_t * from_frame, int is_ip4)
3077{
Florin Coras7ac053b2018-11-05 15:57:21 -08003078 u32 n_left_from, *from, n_syns = 0, *first_buffer;
Damjan Marion586afd72017-04-05 19:18:20 +02003079 u32 my_thread_index = vm->thread_index;
Dave Barach68b0fb02017-02-28 15:15:56 -05003080
Florin Coras7ac053b2018-11-05 15:57:21 -08003081 from = first_buffer = vlib_frame_vector_args (from_frame);
Dave Barach68b0fb02017-02-28 15:15:56 -05003082 n_left_from = from_frame->n_vectors;
3083
Dave Barach68b0fb02017-02-28 15:15:56 -05003084 while (n_left_from > 0)
3085 {
Florin Coras7ac053b2018-11-05 15:57:21 -08003086 u32 bi0;
3087 vlib_buffer_t *b0;
3088 tcp_rx_trace_t *t0;
3089 tcp_header_t *th0 = 0;
3090 tcp_connection_t *lc0;
3091 ip4_header_t *ip40;
3092 ip6_header_t *ip60;
3093 tcp_connection_t *child0;
3094 u32 error0 = TCP_ERROR_NONE;
Dave Barach68b0fb02017-02-28 15:15:56 -05003095
Florin Coras7ac053b2018-11-05 15:57:21 -08003096 bi0 = from[0];
3097 from += 1;
3098 n_left_from -= 1;
Dave Barach68b0fb02017-02-28 15:15:56 -05003099
Florin Coras7ac053b2018-11-05 15:57:21 -08003100 b0 = vlib_get_buffer (vm, bi0);
3101 lc0 = tcp_listener_get (vnet_buffer (b0)->tcp.connection_index);
3102
3103 if (is_ip4)
Dave Barach68b0fb02017-02-28 15:15:56 -05003104 {
Florin Coras7ac053b2018-11-05 15:57:21 -08003105 ip40 = vlib_buffer_get_current (b0);
3106 th0 = ip4_next_header (ip40);
3107 }
3108 else
3109 {
3110 ip60 = vlib_buffer_get_current (b0);
3111 th0 = ip6_next_header (ip60);
Dave Barach68b0fb02017-02-28 15:15:56 -05003112 }
3113
Florin Coras7ac053b2018-11-05 15:57:21 -08003114 /* Create child session. For syn-flood protection use filter */
3115
3116 /* 1. first check for an RST: handled in dispatch */
3117 /* if (tcp_rst (th0))
3118 goto drop;
3119 */
3120
3121 /* 2. second check for an ACK: handled in dispatch */
3122 /* if (tcp_ack (th0))
3123 {
3124 tcp_send_reset (b0, is_ip4);
3125 goto drop;
3126 }
3127 */
3128
3129 /* 3. check for a SYN (did that already) */
3130
3131 /* Make sure connection wasn't just created */
3132 child0 = tcp_lookup_connection (lc0->c_fib_index, b0, my_thread_index,
3133 is_ip4);
3134 if (PREDICT_FALSE (child0->state != TCP_STATE_LISTEN))
3135 {
3136 error0 = TCP_ERROR_CREATE_EXISTS;
3137 goto drop;
3138 }
3139
3140 /* Create child session and send SYN-ACK */
Florin Coras8124cb72018-12-16 20:57:29 -08003141 child0 = tcp_connection_alloc (my_thread_index);
Florin Coras7ac053b2018-11-05 15:57:21 -08003142 child0->c_lcl_port = th0->dst_port;
3143 child0->c_rmt_port = th0->src_port;
3144 child0->c_is_ip4 = is_ip4;
3145 child0->state = TCP_STATE_SYN_RCVD;
3146 child0->c_fib_index = lc0->c_fib_index;
3147
3148 if (is_ip4)
3149 {
3150 child0->c_lcl_ip4.as_u32 = ip40->dst_address.as_u32;
3151 child0->c_rmt_ip4.as_u32 = ip40->src_address.as_u32;
3152 }
3153 else
3154 {
Dave Barach178cf492018-11-13 16:34:13 -05003155 clib_memcpy_fast (&child0->c_lcl_ip6, &ip60->dst_address,
3156 sizeof (ip6_address_t));
3157 clib_memcpy_fast (&child0->c_rmt_ip6, &ip60->src_address,
3158 sizeof (ip6_address_t));
Florin Coras7ac053b2018-11-05 15:57:21 -08003159 }
3160
Florin Coras80231112018-12-05 15:59:31 -08003161 if (tcp_options_parse (th0, &child0->rcv_opts, 1))
Florin Coras7ac053b2018-11-05 15:57:21 -08003162 {
Florin Coras8124cb72018-12-16 20:57:29 -08003163 error0 = TCP_ERROR_OPTIONS;
3164 tcp_connection_free (child0);
Florin Coras7ac053b2018-11-05 15:57:21 -08003165 goto drop;
3166 }
3167
3168 child0->irs = vnet_buffer (b0)->tcp.seq_number;
3169 child0->rcv_nxt = vnet_buffer (b0)->tcp.seq_number + 1;
3170 child0->rcv_las = child0->rcv_nxt;
3171 child0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
3172
3173 /* RFC1323: TSval timestamps sent on {SYN} and {SYN,ACK}
3174 * segments are used to initialize PAWS. */
3175 if (tcp_opts_tstamp (&child0->rcv_opts))
3176 {
3177 child0->tsval_recent = child0->rcv_opts.tsval;
3178 child0->tsval_recent_age = tcp_time_now ();
3179 }
3180
3181 if (tcp_opts_wscale (&child0->rcv_opts))
3182 child0->snd_wscale = child0->rcv_opts.wscale;
3183
3184 child0->snd_wnd = clib_net_to_host_u16 (th0->window)
3185 << child0->snd_wscale;
3186 child0->snd_wl1 = vnet_buffer (b0)->tcp.seq_number;
3187 child0->snd_wl2 = vnet_buffer (b0)->tcp.ack_number;
3188
3189 tcp_connection_init_vars (child0);
Florin Coras865872e2019-01-18 12:12:29 -08003190 child0->rto = TCP_RTO_MIN;
Florin Coras7ac053b2018-11-05 15:57:21 -08003191 TCP_EVT_DBG (TCP_EVT_SYN_RCVD, child0, 1);
3192
Florin Corasc9940fc2019-02-05 20:55:11 -08003193 if (session_stream_accept (&child0->connection, lc0->c_s_index,
Florin Coras7ac053b2018-11-05 15:57:21 -08003194 0 /* notify */ ))
3195 {
Florin Coras7ac053b2018-11-05 15:57:21 -08003196 tcp_connection_cleanup (child0);
3197 error0 = TCP_ERROR_CREATE_SESSION_FAIL;
3198 goto drop;
3199 }
3200
Florin Coras2e31cc32018-09-25 14:00:34 -07003201 child0->tx_fifo_size = transport_tx_fifo_size (&child0->connection);
Florin Coras7ac053b2018-11-05 15:57:21 -08003202 tcp_send_synack (child0);
3203 tcp_timer_set (child0, TCP_TIMER_ESTABLISH, TCP_SYN_RCVD_TIME);
3204
3205 drop:
3206
3207 if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
3208 {
3209 t0 = vlib_add_trace (vm, node, b0, sizeof (*t0));
Dave Barach178cf492018-11-13 16:34:13 -05003210 clib_memcpy_fast (&t0->tcp_header, th0, sizeof (t0->tcp_header));
3211 clib_memcpy_fast (&t0->tcp_connection, lc0,
3212 sizeof (t0->tcp_connection));
Florin Coras7ac053b2018-11-05 15:57:21 -08003213 }
3214
3215 n_syns += (error0 == TCP_ERROR_NONE);
Dave Barach68b0fb02017-02-28 15:15:56 -05003216 }
Florin Coras00cd22d2018-04-18 13:20:18 -07003217
3218 tcp_inc_counter (listen, TCP_ERROR_SYNS_RCVD, n_syns);
Florin Coras7ac053b2018-11-05 15:57:21 -08003219 vlib_buffer_free (vm, first_buffer, from_frame->n_vectors);
3220
Dave Barach68b0fb02017-02-28 15:15:56 -05003221 return from_frame->n_vectors;
3222}
3223
Filip Tehlare275bed2019-03-06 00:06:56 -08003224VLIB_NODE_FN (tcp4_listen_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
3225 vlib_frame_t * from_frame)
Dave Barach68b0fb02017-02-28 15:15:56 -05003226{
3227 return tcp46_listen_inline (vm, node, from_frame, 1 /* is_ip4 */ );
3228}
3229
Filip Tehlare275bed2019-03-06 00:06:56 -08003230VLIB_NODE_FN (tcp6_listen_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
3231 vlib_frame_t * from_frame)
Dave Barach68b0fb02017-02-28 15:15:56 -05003232{
3233 return tcp46_listen_inline (vm, node, from_frame, 0 /* is_ip4 */ );
3234}
3235
3236/* *INDENT-OFF* */
3237VLIB_REGISTER_NODE (tcp4_listen_node) =
3238{
Dave Barach68b0fb02017-02-28 15:15:56 -05003239 .name = "tcp4-listen",
3240 /* Takes a vector of packets. */
3241 .vector_size = sizeof (u32),
3242 .n_errors = TCP_N_ERROR,
3243 .error_strings = tcp_error_strings,
3244 .n_next_nodes = TCP_LISTEN_N_NEXT,
3245 .next_nodes =
3246 {
3247#define _(s,n) [TCP_LISTEN_NEXT_##s] = n,
3248 foreach_tcp_state_next
3249#undef _
3250 },
Clement Durand6cf260c2017-04-13 13:27:04 +02003251 .format_trace = format_tcp_rx_trace_short,
Dave Barach68b0fb02017-02-28 15:15:56 -05003252};
3253/* *INDENT-ON* */
3254
Dave Barach68b0fb02017-02-28 15:15:56 -05003255/* *INDENT-OFF* */
3256VLIB_REGISTER_NODE (tcp6_listen_node) =
3257{
Dave Barach68b0fb02017-02-28 15:15:56 -05003258 .name = "tcp6-listen",
3259 /* Takes a vector of packets. */
3260 .vector_size = sizeof (u32),
3261 .n_errors = TCP_N_ERROR,
3262 .error_strings = tcp_error_strings,
3263 .n_next_nodes = TCP_LISTEN_N_NEXT,
3264 .next_nodes =
3265 {
3266#define _(s,n) [TCP_LISTEN_NEXT_##s] = n,
3267 foreach_tcp_state_next
3268#undef _
3269 },
Clement Durand6cf260c2017-04-13 13:27:04 +02003270 .format_trace = format_tcp_rx_trace_short,
Dave Barach68b0fb02017-02-28 15:15:56 -05003271};
3272/* *INDENT-ON* */
3273
Dave Barach68b0fb02017-02-28 15:15:56 -05003274typedef enum _tcp_input_next
3275{
3276 TCP_INPUT_NEXT_DROP,
3277 TCP_INPUT_NEXT_LISTEN,
3278 TCP_INPUT_NEXT_RCV_PROCESS,
3279 TCP_INPUT_NEXT_SYN_SENT,
3280 TCP_INPUT_NEXT_ESTABLISHED,
3281 TCP_INPUT_NEXT_RESET,
Pierre Pfister7fe51f32017-09-20 08:48:36 +02003282 TCP_INPUT_NEXT_PUNT,
Dave Barach68b0fb02017-02-28 15:15:56 -05003283 TCP_INPUT_N_NEXT
3284} tcp_input_next_t;
3285
3286#define foreach_tcp4_input_next \
Vijayabhaskar Katamreddyce074122017-11-15 13:50:26 -08003287 _ (DROP, "ip4-drop") \
Dave Barach68b0fb02017-02-28 15:15:56 -05003288 _ (LISTEN, "tcp4-listen") \
3289 _ (RCV_PROCESS, "tcp4-rcv-process") \
3290 _ (SYN_SENT, "tcp4-syn-sent") \
3291 _ (ESTABLISHED, "tcp4-established") \
Pierre Pfister7fe51f32017-09-20 08:48:36 +02003292 _ (RESET, "tcp4-reset") \
Vijayabhaskar Katamreddyce074122017-11-15 13:50:26 -08003293 _ (PUNT, "ip4-punt")
Dave Barach68b0fb02017-02-28 15:15:56 -05003294
3295#define foreach_tcp6_input_next \
Vijayabhaskar Katamreddyce074122017-11-15 13:50:26 -08003296 _ (DROP, "ip6-drop") \
Dave Barach68b0fb02017-02-28 15:15:56 -05003297 _ (LISTEN, "tcp6-listen") \
3298 _ (RCV_PROCESS, "tcp6-rcv-process") \
3299 _ (SYN_SENT, "tcp6-syn-sent") \
3300 _ (ESTABLISHED, "tcp6-established") \
Pierre Pfister7fe51f32017-09-20 08:48:36 +02003301 _ (RESET, "tcp6-reset") \
Vijayabhaskar Katamreddyce074122017-11-15 13:50:26 -08003302 _ (PUNT, "ip6-punt")
Dave Barach68b0fb02017-02-28 15:15:56 -05003303
Dave Barach68b0fb02017-02-28 15:15:56 -05003304#define filter_flags (TCP_FLAG_SYN|TCP_FLAG_ACK|TCP_FLAG_RST|TCP_FLAG_FIN)
3305
Florin Coras0c8a3bc2018-06-14 17:11:56 -07003306static void
3307tcp_input_trace_frame (vlib_main_t * vm, vlib_node_runtime_t * node,
Florin Coras4df38712018-06-20 12:44:16 -07003308 vlib_buffer_t ** bs, u32 n_bufs, u8 is_ip4)
Dave Barach68b0fb02017-02-28 15:15:56 -05003309{
Florin Coras0c8a3bc2018-06-14 17:11:56 -07003310 tcp_connection_t *tc;
3311 tcp_header_t *tcp;
3312 tcp_rx_trace_t *t;
Florin Coras0c8a3bc2018-06-14 17:11:56 -07003313 int i;
Dave Barach68b0fb02017-02-28 15:15:56 -05003314
Florin Coras4df38712018-06-20 12:44:16 -07003315 for (i = 0; i < n_bufs; i++)
Dave Barach68b0fb02017-02-28 15:15:56 -05003316 {
Florin Coras4df38712018-06-20 12:44:16 -07003317 if (bs[i]->flags & VLIB_BUFFER_IS_TRACED)
3318 {
3319 t = vlib_add_trace (vm, node, bs[i], sizeof (*t));
3320 tc = tcp_connection_get (vnet_buffer (bs[i])->tcp.connection_index,
3321 vm->thread_index);
3322 tcp = vlib_buffer_get_current (bs[i]);
3323 tcp_set_rx_trace_data (t, tc, tcp, bs[i], is_ip4);
3324 }
Florin Coras0c8a3bc2018-06-14 17:11:56 -07003325 }
3326}
Dave Barach68b0fb02017-02-28 15:15:56 -05003327
Florin Coras0c8a3bc2018-06-14 17:11:56 -07003328static void
3329tcp_input_set_error_next (tcp_main_t * tm, u16 * next, u32 * error, u8 is_ip4)
3330{
Florin Corasb5e55a22019-01-10 12:42:47 -08003331 if (*error == TCP_ERROR_FILTERED || *error == TCP_ERROR_WRONG_THREAD)
Florin Coras0c8a3bc2018-06-14 17:11:56 -07003332 {
3333 *next = TCP_INPUT_NEXT_DROP;
3334 }
3335 else if ((is_ip4 && tm->punt_unknown4) || (!is_ip4 && tm->punt_unknown6))
3336 {
3337 *next = TCP_INPUT_NEXT_PUNT;
3338 *error = TCP_ERROR_PUNT;
3339 }
3340 else
3341 {
3342 *next = TCP_INPUT_NEXT_RESET;
3343 *error = TCP_ERROR_NO_LISTENER;
3344 }
3345}
Dave Barach68b0fb02017-02-28 15:15:56 -05003346
Florin Coras0c8a3bc2018-06-14 17:11:56 -07003347static inline tcp_connection_t *
3348tcp_input_lookup_buffer (vlib_buffer_t * b, u8 thread_index, u32 * error,
3349 u8 is_ip4)
3350{
3351 u32 fib_index = vnet_buffer (b)->ip.fib_index;
3352 int n_advance_bytes, n_data_bytes;
3353 transport_connection_t *tc;
3354 tcp_header_t *tcp;
Florin Corasb5e55a22019-01-10 12:42:47 -08003355 u8 result = 0;
Florin Coras0c8a3bc2018-06-14 17:11:56 -07003356
3357 if (is_ip4)
3358 {
3359 ip4_header_t *ip4 = vlib_buffer_get_current (b);
Florin Corasb8dda5f2018-12-05 10:03:34 -08003360 int ip_hdr_bytes = ip4_header_bytes (ip4);
3361 if (PREDICT_FALSE (b->current_length < ip_hdr_bytes + sizeof (*tcp)))
3362 {
3363 *error = TCP_ERROR_LENGTH;
3364 return 0;
3365 }
Florin Coras0c8a3bc2018-06-14 17:11:56 -07003366 tcp = ip4_next_header (ip4);
3367 vnet_buffer (b)->tcp.hdr_offset = (u8 *) tcp - (u8 *) ip4;
Florin Corasb8dda5f2018-12-05 10:03:34 -08003368 n_advance_bytes = (ip_hdr_bytes + tcp_header_bytes (tcp));
Florin Coras0c8a3bc2018-06-14 17:11:56 -07003369 n_data_bytes = clib_net_to_host_u16 (ip4->length) - n_advance_bytes;
3370
3371 /* Length check. Checksum computed by ipx_local no need to compute again */
Florin Corasb8dda5f2018-12-05 10:03:34 -08003372 if (PREDICT_FALSE (n_data_bytes < 0))
Dave Barach68b0fb02017-02-28 15:15:56 -05003373 {
Florin Coras0c8a3bc2018-06-14 17:11:56 -07003374 *error = TCP_ERROR_LENGTH;
3375 return 0;
Dave Barach68b0fb02017-02-28 15:15:56 -05003376 }
3377
Florin Coras0c8a3bc2018-06-14 17:11:56 -07003378 tc = session_lookup_connection_wt4 (fib_index, &ip4->dst_address,
3379 &ip4->src_address, tcp->dst_port,
3380 tcp->src_port, TRANSPORT_PROTO_TCP,
Florin Corasb5e55a22019-01-10 12:42:47 -08003381 thread_index, &result);
Florin Coras0c8a3bc2018-06-14 17:11:56 -07003382 }
3383 else
3384 {
3385 ip6_header_t *ip6 = vlib_buffer_get_current (b);
Florin Corasb8dda5f2018-12-05 10:03:34 -08003386 if (PREDICT_FALSE (b->current_length < sizeof (*ip6) + sizeof (*tcp)))
3387 {
3388 *error = TCP_ERROR_LENGTH;
3389 return 0;
3390 }
Florin Coras0c8a3bc2018-06-14 17:11:56 -07003391 tcp = ip6_next_header (ip6);
3392 vnet_buffer (b)->tcp.hdr_offset = (u8 *) tcp - (u8 *) ip6;
3393 n_advance_bytes = tcp_header_bytes (tcp);
3394 n_data_bytes = clib_net_to_host_u16 (ip6->payload_length)
3395 - n_advance_bytes;
3396 n_advance_bytes += sizeof (ip6[0]);
3397
Florin Corasb8dda5f2018-12-05 10:03:34 -08003398 if (PREDICT_FALSE (n_data_bytes < 0))
Florin Coras0c8a3bc2018-06-14 17:11:56 -07003399 {
3400 *error = TCP_ERROR_LENGTH;
3401 return 0;
3402 }
Florin Coras6164e972019-02-07 10:31:08 -08003403 if (PREDICT_FALSE
3404 (ip6_address_is_link_local_unicast (&ip6->dst_address)))
3405 {
3406 ip4_main_t *im = &ip4_main;
3407 fib_index = vec_elt (im->fib_index_by_sw_if_index,
3408 vnet_buffer (b)->sw_if_index[VLIB_RX]);
3409 }
Florin Coras0c8a3bc2018-06-14 17:11:56 -07003410
3411 tc = session_lookup_connection_wt6 (fib_index, &ip6->dst_address,
3412 &ip6->src_address, tcp->dst_port,
3413 tcp->src_port, TRANSPORT_PROTO_TCP,
Florin Corasb5e55a22019-01-10 12:42:47 -08003414 thread_index, &result);
Dave Barach68b0fb02017-02-28 15:15:56 -05003415 }
3416
Florin Coras0c8a3bc2018-06-14 17:11:56 -07003417 vnet_buffer (b)->tcp.seq_number = clib_net_to_host_u32 (tcp->seq_number);
3418 vnet_buffer (b)->tcp.ack_number = clib_net_to_host_u32 (tcp->ack_number);
3419 vnet_buffer (b)->tcp.data_offset = n_advance_bytes;
3420 vnet_buffer (b)->tcp.data_len = n_data_bytes;
Florin Coras3c514d52018-12-22 11:39:33 -08003421 vnet_buffer (b)->tcp.seq_end = vnet_buffer (b)->tcp.seq_number
3422 + n_data_bytes;
Florin Coras0c8a3bc2018-06-14 17:11:56 -07003423 vnet_buffer (b)->tcp.flags = 0;
3424
Florin Corasb5e55a22019-01-10 12:42:47 -08003425 *error = result ? TCP_ERROR_NONE + result : *error;
Florin Coras0c8a3bc2018-06-14 17:11:56 -07003426
3427 return tcp_get_connection_from_transport (tc);
3428}
3429
3430static inline void
3431tcp_input_dispatch_buffer (tcp_main_t * tm, tcp_connection_t * tc,
3432 vlib_buffer_t * b, u16 * next, u32 * error)
3433{
3434 tcp_header_t *tcp;
3435 u8 flags;
3436
3437 tcp = tcp_buffer_hdr (b);
3438 flags = tcp->flags & filter_flags;
3439 *next = tm->dispatch_table[tc->state][flags].next;
3440 *error = tm->dispatch_table[tc->state][flags].error;
3441
3442 if (PREDICT_FALSE (*error == TCP_ERROR_DISPATCH
3443 || *next == TCP_INPUT_NEXT_RESET))
3444 {
3445 /* Overload tcp flags to store state */
3446 tcp_state_t state = tc->state;
3447 vnet_buffer (b)->tcp.flags = tc->state;
3448
3449 if (*error == TCP_ERROR_DISPATCH)
Florin Corasa9d5bea2018-12-17 08:24:19 -08003450 clib_warning ("tcp conn %u disp error state %U flags %U",
3451 tc->c_c_index, format_tcp_state, state,
3452 format_tcp_flags, (int) flags);
Florin Coras0c8a3bc2018-06-14 17:11:56 -07003453 }
3454}
3455
3456always_inline uword
3457tcp46_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
3458 vlib_frame_t * frame, int is_ip4)
3459{
3460 u32 n_left_from, *from, thread_index = vm->thread_index;
3461 tcp_main_t *tm = vnet_get_tcp_main ();
3462 vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
3463 u16 nexts[VLIB_FRAME_SIZE], *next;
3464
Florin Corasbe72ae62018-11-01 11:23:03 -07003465 tcp_set_time_now (tcp_get_worker (thread_index));
Florin Coras0c8a3bc2018-06-14 17:11:56 -07003466
3467 from = vlib_frame_vector_args (frame);
3468 n_left_from = frame->n_vectors;
3469 vlib_get_buffers (vm, from, bufs, n_left_from);
3470
3471 b = bufs;
3472 next = nexts;
3473
3474 while (n_left_from >= 4)
3475 {
3476 u32 error0 = TCP_ERROR_NO_LISTENER, error1 = TCP_ERROR_NO_LISTENER;
3477 tcp_connection_t *tc0, *tc1;
3478
3479 {
3480 vlib_prefetch_buffer_header (b[2], STORE);
3481 CLIB_PREFETCH (b[2]->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
3482
3483 vlib_prefetch_buffer_header (b[3], STORE);
3484 CLIB_PREFETCH (b[3]->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
3485 }
3486
3487 next[0] = next[1] = TCP_INPUT_NEXT_DROP;
3488
3489 tc0 = tcp_input_lookup_buffer (b[0], thread_index, &error0, is_ip4);
3490 tc1 = tcp_input_lookup_buffer (b[1], thread_index, &error1, is_ip4);
3491
3492 if (PREDICT_TRUE (!tc0 + !tc1 == 0))
3493 {
3494 ASSERT (tcp_lookup_is_valid (tc0, tcp_buffer_hdr (b[0])));
3495 ASSERT (tcp_lookup_is_valid (tc1, tcp_buffer_hdr (b[1])));
3496
3497 vnet_buffer (b[0])->tcp.connection_index = tc0->c_c_index;
3498 vnet_buffer (b[1])->tcp.connection_index = tc1->c_c_index;
3499
3500 tcp_input_dispatch_buffer (tm, tc0, b[0], &next[0], &error0);
3501 tcp_input_dispatch_buffer (tm, tc1, b[1], &next[1], &error1);
3502 }
3503 else
3504 {
3505 if (PREDICT_TRUE (tc0 != 0))
3506 {
3507 ASSERT (tcp_lookup_is_valid (tc0, tcp_buffer_hdr (b[0])));
3508 vnet_buffer (b[0])->tcp.connection_index = tc0->c_c_index;
3509 tcp_input_dispatch_buffer (tm, tc0, b[0], &next[0], &error0);
3510 }
3511 else
3512 tcp_input_set_error_next (tm, &next[0], &error0, is_ip4);
3513
3514 if (PREDICT_TRUE (tc1 != 0))
3515 {
3516 ASSERT (tcp_lookup_is_valid (tc1, tcp_buffer_hdr (b[1])));
3517 vnet_buffer (b[1])->tcp.connection_index = tc1->c_c_index;
3518 tcp_input_dispatch_buffer (tm, tc1, b[1], &next[1], &error1);
3519 }
3520 else
3521 tcp_input_set_error_next (tm, &next[1], &error1, is_ip4);
3522 }
3523
3524 b += 2;
3525 next += 2;
3526 n_left_from -= 2;
3527 }
3528 while (n_left_from > 0)
3529 {
3530 tcp_connection_t *tc0;
3531 u32 error0 = TCP_ERROR_NO_LISTENER;
3532
3533 if (n_left_from > 1)
3534 {
3535 vlib_prefetch_buffer_header (b[1], STORE);
3536 CLIB_PREFETCH (b[1]->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
3537 }
3538
3539 next[0] = TCP_INPUT_NEXT_DROP;
3540 tc0 = tcp_input_lookup_buffer (b[0], thread_index, &error0, is_ip4);
3541 if (PREDICT_TRUE (tc0 != 0))
3542 {
3543 ASSERT (tcp_lookup_is_valid (tc0, tcp_buffer_hdr (b[0])));
3544 vnet_buffer (b[0])->tcp.connection_index = tc0->c_c_index;
3545 tcp_input_dispatch_buffer (tm, tc0, b[0], &next[0], &error0);
3546 }
3547 else
3548 tcp_input_set_error_next (tm, &next[0], &error0, is_ip4);
3549
3550 b += 1;
3551 next += 1;
3552 n_left_from -= 1;
3553 }
3554
3555 if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
3556 tcp_input_trace_frame (vm, node, bufs, frame->n_vectors, is_ip4);
3557
3558 vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
3559 return frame->n_vectors;
Dave Barach68b0fb02017-02-28 15:15:56 -05003560}
3561
Filip Tehlare275bed2019-03-06 00:06:56 -08003562VLIB_NODE_FN (tcp4_input_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
3563 vlib_frame_t * from_frame)
Dave Barach68b0fb02017-02-28 15:15:56 -05003564{
3565 return tcp46_input_inline (vm, node, from_frame, 1 /* is_ip4 */ );
3566}
3567
Filip Tehlare275bed2019-03-06 00:06:56 -08003568VLIB_NODE_FN (tcp6_input_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
3569 vlib_frame_t * from_frame)
Dave Barach68b0fb02017-02-28 15:15:56 -05003570{
3571 return tcp46_input_inline (vm, node, from_frame, 0 /* is_ip4 */ );
3572}
3573
3574/* *INDENT-OFF* */
3575VLIB_REGISTER_NODE (tcp4_input_node) =
3576{
Dave Barach68b0fb02017-02-28 15:15:56 -05003577 .name = "tcp4-input",
3578 /* Takes a vector of packets. */
3579 .vector_size = sizeof (u32),
3580 .n_errors = TCP_N_ERROR,
3581 .error_strings = tcp_error_strings,
3582 .n_next_nodes = TCP_INPUT_N_NEXT,
3583 .next_nodes =
3584 {
3585#define _(s,n) [TCP_INPUT_NEXT_##s] = n,
3586 foreach_tcp4_input_next
3587#undef _
3588 },
3589 .format_buffer = format_tcp_header,
3590 .format_trace = format_tcp_rx_trace,
3591};
3592/* *INDENT-ON* */
3593
Dave Barach68b0fb02017-02-28 15:15:56 -05003594/* *INDENT-OFF* */
3595VLIB_REGISTER_NODE (tcp6_input_node) =
3596{
Dave Barach68b0fb02017-02-28 15:15:56 -05003597 .name = "tcp6-input",
3598 /* Takes a vector of packets. */
3599 .vector_size = sizeof (u32),
3600 .n_errors = TCP_N_ERROR,
3601 .error_strings = tcp_error_strings,
3602 .n_next_nodes = TCP_INPUT_N_NEXT,
3603 .next_nodes =
3604 {
3605#define _(s,n) [TCP_INPUT_NEXT_##s] = n,
3606 foreach_tcp6_input_next
3607#undef _
3608 },
3609 .format_buffer = format_tcp_header,
3610 .format_trace = format_tcp_rx_trace,
3611};
3612/* *INDENT-ON* */
3613
Filip Tehlare275bed2019-03-06 00:06:56 -08003614#ifndef CLIB_MARCH_VARIANT
Dave Barach68b0fb02017-02-28 15:15:56 -05003615static void
3616tcp_dispatch_table_init (tcp_main_t * tm)
3617{
3618 int i, j;
3619 for (i = 0; i < ARRAY_LEN (tm->dispatch_table); i++)
3620 for (j = 0; j < ARRAY_LEN (tm->dispatch_table[i]); j++)
3621 {
3622 tm->dispatch_table[i][j].next = TCP_INPUT_NEXT_DROP;
3623 tm->dispatch_table[i][j].error = TCP_ERROR_DISPATCH;
3624 }
3625
3626#define _(t,f,n,e) \
3627do { \
3628 tm->dispatch_table[TCP_STATE_##t][f].next = (n); \
3629 tm->dispatch_table[TCP_STATE_##t][f].error = (e); \
3630} while (0)
3631
Florin Coras678a6572018-12-17 21:31:25 -08003632 /* RFC 793: In LISTEN if RST drop and if ACK return RST */
Florin Coras5c0f1662018-12-19 01:38:57 -08003633 _(LISTEN, 0, TCP_INPUT_NEXT_DROP, TCP_ERROR_SEGMENT_INVALID);
Florin Coras678a6572018-12-17 21:31:25 -08003634 _(LISTEN, TCP_FLAG_ACK, TCP_INPUT_NEXT_RESET, TCP_ERROR_ACK_INVALID);
3635 _(LISTEN, TCP_FLAG_RST, TCP_INPUT_NEXT_DROP, TCP_ERROR_INVALID_CONNECTION);
Dave Barach68b0fb02017-02-28 15:15:56 -05003636 _(LISTEN, TCP_FLAG_SYN, TCP_INPUT_NEXT_LISTEN, TCP_ERROR_NONE);
Florin Coras678a6572018-12-17 21:31:25 -08003637 _(LISTEN, TCP_FLAG_SYN | TCP_FLAG_ACK, TCP_INPUT_NEXT_RESET,
3638 TCP_ERROR_ACK_INVALID);
3639 _(LISTEN, TCP_FLAG_SYN | TCP_FLAG_RST, TCP_INPUT_NEXT_DROP,
3640 TCP_ERROR_SEGMENT_INVALID);
3641 _(LISTEN, TCP_FLAG_SYN | TCP_FLAG_RST | TCP_FLAG_ACK, TCP_INPUT_NEXT_DROP,
3642 TCP_ERROR_SEGMENT_INVALID);
3643 _(LISTEN, TCP_FLAG_RST | TCP_FLAG_ACK, TCP_INPUT_NEXT_DROP,
3644 TCP_ERROR_INVALID_CONNECTION);
3645 _(LISTEN, TCP_FLAG_FIN, TCP_INPUT_NEXT_RESET, TCP_ERROR_SEGMENT_INVALID);
Dave Barach2c25a622017-06-26 11:35:07 -04003646 _(LISTEN, TCP_FLAG_FIN | TCP_FLAG_ACK, TCP_INPUT_NEXT_RESET,
Florin Coras678a6572018-12-17 21:31:25 -08003647 TCP_ERROR_SEGMENT_INVALID);
3648 _(LISTEN, TCP_FLAG_FIN | TCP_FLAG_RST, TCP_INPUT_NEXT_DROP,
3649 TCP_ERROR_SEGMENT_INVALID);
3650 _(LISTEN, TCP_FLAG_FIN | TCP_FLAG_RST | TCP_FLAG_ACK, TCP_INPUT_NEXT_DROP,
Dave Barach2c25a622017-06-26 11:35:07 -04003651 TCP_ERROR_NONE);
Florin Coras678a6572018-12-17 21:31:25 -08003652 _(LISTEN, TCP_FLAG_FIN | TCP_FLAG_SYN, TCP_INPUT_NEXT_DROP,
3653 TCP_ERROR_SEGMENT_INVALID);
3654 _(LISTEN, TCP_FLAG_FIN | TCP_FLAG_SYN | TCP_FLAG_ACK, TCP_INPUT_NEXT_DROP,
3655 TCP_ERROR_SEGMENT_INVALID);
3656 _(LISTEN, TCP_FLAG_FIN | TCP_FLAG_SYN | TCP_FLAG_RST, TCP_INPUT_NEXT_DROP,
3657 TCP_ERROR_SEGMENT_INVALID);
3658 _(LISTEN, TCP_FLAG_FIN | TCP_FLAG_SYN | TCP_FLAG_RST | TCP_FLAG_ACK,
3659 TCP_INPUT_NEXT_DROP, TCP_ERROR_SEGMENT_INVALID);
Dave Barach68b0fb02017-02-28 15:15:56 -05003660 /* ACK for for a SYN-ACK -> tcp-rcv-process. */
3661 _(SYN_RCVD, TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
Florin Corasc28764f2017-04-26 00:08:42 -07003662 _(SYN_RCVD, TCP_FLAG_RST, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
Florin Corase8460c72018-09-24 14:40:40 -07003663 _(SYN_RCVD, TCP_FLAG_RST | TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS,
3664 TCP_ERROR_NONE);
Dave Barach2c25a622017-06-26 11:35:07 -04003665 _(SYN_RCVD, TCP_FLAG_SYN, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
Florin Coras678a6572018-12-17 21:31:25 -08003666 _(SYN_RCVD, TCP_FLAG_SYN | TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS,
3667 TCP_ERROR_NONE);
3668 _(SYN_RCVD, TCP_FLAG_SYN | TCP_FLAG_RST, TCP_INPUT_NEXT_RCV_PROCESS,
3669 TCP_ERROR_NONE);
3670 _(SYN_RCVD, TCP_FLAG_SYN | TCP_FLAG_RST | TCP_FLAG_ACK,
3671 TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3672 _(SYN_RCVD, TCP_FLAG_FIN, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
Florin Corasc01c4452018-09-20 18:36:54 -07003673 _(SYN_RCVD, TCP_FLAG_FIN | TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS,
3674 TCP_ERROR_NONE);
Florin Coras678a6572018-12-17 21:31:25 -08003675 _(SYN_RCVD, TCP_FLAG_FIN | TCP_FLAG_RST, TCP_INPUT_NEXT_RCV_PROCESS,
3676 TCP_ERROR_NONE);
3677 _(SYN_RCVD, TCP_FLAG_FIN | TCP_FLAG_RST | TCP_FLAG_ACK,
3678 TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3679 _(SYN_RCVD, TCP_FLAG_FIN | TCP_FLAG_SYN, TCP_INPUT_NEXT_RCV_PROCESS,
3680 TCP_ERROR_NONE);
3681 _(SYN_RCVD, TCP_FLAG_FIN | TCP_FLAG_SYN | TCP_FLAG_RST,
3682 TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3683 _(SYN_RCVD, TCP_FLAG_FIN | TCP_FLAG_SYN | TCP_FLAG_ACK,
3684 TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3685 _(SYN_RCVD, TCP_FLAG_FIN | TCP_FLAG_SYN | TCP_FLAG_RST | TCP_FLAG_ACK,
3686 TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3687 _(SYN_RCVD, 0, TCP_INPUT_NEXT_DROP, TCP_ERROR_SEGMENT_INVALID);
Dave Barach68b0fb02017-02-28 15:15:56 -05003688 /* SYN-ACK for a SYN */
3689 _(SYN_SENT, TCP_FLAG_SYN | TCP_FLAG_ACK, TCP_INPUT_NEXT_SYN_SENT,
3690 TCP_ERROR_NONE);
3691 _(SYN_SENT, TCP_FLAG_ACK, TCP_INPUT_NEXT_SYN_SENT, TCP_ERROR_NONE);
3692 _(SYN_SENT, TCP_FLAG_RST, TCP_INPUT_NEXT_SYN_SENT, TCP_ERROR_NONE);
3693 _(SYN_SENT, TCP_FLAG_RST | TCP_FLAG_ACK, TCP_INPUT_NEXT_SYN_SENT,
3694 TCP_ERROR_NONE);
Florin Coras222e1f412019-02-16 20:47:32 -08003695 _(SYN_SENT, TCP_FLAG_FIN, TCP_INPUT_NEXT_SYN_SENT, TCP_ERROR_NONE);
3696 _(SYN_SENT, TCP_FLAG_FIN | TCP_FLAG_ACK, TCP_INPUT_NEXT_SYN_SENT,
3697 TCP_ERROR_NONE);
Dave Barach68b0fb02017-02-28 15:15:56 -05003698 /* ACK for for established connection -> tcp-established. */
3699 _(ESTABLISHED, TCP_FLAG_ACK, TCP_INPUT_NEXT_ESTABLISHED, TCP_ERROR_NONE);
3700 /* FIN for for established connection -> tcp-established. */
3701 _(ESTABLISHED, TCP_FLAG_FIN, TCP_INPUT_NEXT_ESTABLISHED, TCP_ERROR_NONE);
3702 _(ESTABLISHED, TCP_FLAG_FIN | TCP_FLAG_ACK, TCP_INPUT_NEXT_ESTABLISHED,
3703 TCP_ERROR_NONE);
Florin Coras678a6572018-12-17 21:31:25 -08003704 _(ESTABLISHED, TCP_FLAG_FIN | TCP_FLAG_RST, TCP_INPUT_NEXT_ESTABLISHED,
3705 TCP_ERROR_NONE);
3706 _(ESTABLISHED, TCP_FLAG_FIN | TCP_FLAG_RST | TCP_FLAG_ACK,
3707 TCP_INPUT_NEXT_ESTABLISHED, TCP_ERROR_NONE);
3708 _(ESTABLISHED, TCP_FLAG_FIN | TCP_FLAG_SYN, TCP_INPUT_NEXT_ESTABLISHED,
3709 TCP_ERROR_NONE);
3710 _(ESTABLISHED, TCP_FLAG_FIN | TCP_FLAG_SYN | TCP_FLAG_ACK,
3711 TCP_INPUT_NEXT_ESTABLISHED, TCP_ERROR_NONE);
3712 _(ESTABLISHED, TCP_FLAG_FIN | TCP_FLAG_SYN | TCP_FLAG_RST,
3713 TCP_INPUT_NEXT_ESTABLISHED, TCP_ERROR_NONE);
3714 _(ESTABLISHED, TCP_FLAG_FIN | TCP_FLAG_SYN | TCP_FLAG_RST | TCP_FLAG_ACK,
3715 TCP_INPUT_NEXT_ESTABLISHED, TCP_ERROR_NONE);
Florin Corasd79b41e2017-03-04 05:37:52 -08003716 _(ESTABLISHED, TCP_FLAG_RST, TCP_INPUT_NEXT_ESTABLISHED, TCP_ERROR_NONE);
Florin Corasdc629cd2017-05-09 00:52:37 -07003717 _(ESTABLISHED, TCP_FLAG_RST | TCP_FLAG_ACK, TCP_INPUT_NEXT_ESTABLISHED,
3718 TCP_ERROR_NONE);
Dave Barach2c25a622017-06-26 11:35:07 -04003719 _(ESTABLISHED, TCP_FLAG_SYN, TCP_INPUT_NEXT_ESTABLISHED, TCP_ERROR_NONE);
Florin Coras3eb50622017-07-13 01:24:57 -04003720 _(ESTABLISHED, TCP_FLAG_SYN | TCP_FLAG_ACK, TCP_INPUT_NEXT_ESTABLISHED,
3721 TCP_ERROR_NONE);
Florin Coras678a6572018-12-17 21:31:25 -08003722 _(ESTABLISHED, TCP_FLAG_SYN | TCP_FLAG_RST, TCP_INPUT_NEXT_ESTABLISHED,
3723 TCP_ERROR_NONE);
3724 _(ESTABLISHED, TCP_FLAG_SYN | TCP_FLAG_RST | TCP_FLAG_ACK,
3725 TCP_INPUT_NEXT_ESTABLISHED, TCP_ERROR_NONE);
3726 _(ESTABLISHED, 0, TCP_INPUT_NEXT_DROP, TCP_ERROR_SEGMENT_INVALID);
Dave Barach68b0fb02017-02-28 15:15:56 -05003727 /* ACK or FIN-ACK to our FIN */
3728 _(FIN_WAIT_1, TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3729 _(FIN_WAIT_1, TCP_FLAG_ACK | TCP_FLAG_FIN, TCP_INPUT_NEXT_RCV_PROCESS,
3730 TCP_ERROR_NONE);
3731 /* FIN in reply to our FIN from the other side */
Florin Coras76bc1302018-12-23 23:36:36 -08003732 _(FIN_WAIT_1, 0, TCP_INPUT_NEXT_DROP, TCP_ERROR_SEGMENT_INVALID);
Dave Barach68b0fb02017-02-28 15:15:56 -05003733 _(FIN_WAIT_1, TCP_FLAG_FIN, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
Florin Coras76bc1302018-12-23 23:36:36 -08003734 _(FIN_WAIT_1, TCP_FLAG_FIN | TCP_FLAG_SYN, TCP_INPUT_NEXT_RCV_PROCESS,
3735 TCP_ERROR_NONE);
3736 _(FIN_WAIT_1, TCP_FLAG_FIN | TCP_FLAG_SYN | TCP_FLAG_ACK,
3737 TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3738 _(FIN_WAIT_1, TCP_FLAG_FIN | TCP_FLAG_SYN | TCP_FLAG_RST,
3739 TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3740 _(FIN_WAIT_1, TCP_FLAG_FIN | TCP_FLAG_SYN | TCP_FLAG_RST | TCP_FLAG_ACK,
3741 TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3742 _(FIN_WAIT_1, TCP_FLAG_FIN | TCP_FLAG_RST, TCP_INPUT_NEXT_RCV_PROCESS,
3743 TCP_ERROR_NONE);
3744 _(FIN_WAIT_1, TCP_FLAG_FIN | TCP_FLAG_RST | TCP_FLAG_ACK,
3745 TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
Florin Coras5c0f1662018-12-19 01:38:57 -08003746 _(FIN_WAIT_1, TCP_FLAG_SYN, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
Florin Coras76bc1302018-12-23 23:36:36 -08003747 _(FIN_WAIT_1, TCP_FLAG_SYN | TCP_FLAG_RST, TCP_INPUT_NEXT_RCV_PROCESS,
3748 TCP_ERROR_NONE);
3749 _(FIN_WAIT_1, TCP_FLAG_SYN | TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS,
3750 TCP_ERROR_NONE);
3751 _(FIN_WAIT_1, TCP_FLAG_SYN | TCP_FLAG_RST | TCP_FLAG_ACK,
3752 TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
Dave Barach2c25a622017-06-26 11:35:07 -04003753 _(FIN_WAIT_1, TCP_FLAG_RST, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
Florin Corase96bf632018-12-18 22:44:27 -08003754 _(FIN_WAIT_1, TCP_FLAG_RST | TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS,
3755 TCP_ERROR_NONE);
Florin Corasb56fcf12019-01-02 10:10:08 -08003756 _(CLOSING, 0, TCP_INPUT_NEXT_DROP, TCP_ERROR_SEGMENT_INVALID);
Florin Coras56318932018-05-23 20:44:12 -07003757 _(CLOSING, TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
Florin Coras54ddf432018-12-21 13:54:09 -08003758 _(CLOSING, TCP_FLAG_SYN, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
Florin Corasb56fcf12019-01-02 10:10:08 -08003759 _(CLOSING, TCP_FLAG_SYN | TCP_FLAG_RST, TCP_INPUT_NEXT_RCV_PROCESS,
3760 TCP_ERROR_NONE);
3761 _(CLOSING, TCP_FLAG_SYN | TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS,
3762 TCP_ERROR_NONE);
3763 _(CLOSING, TCP_FLAG_SYN | TCP_FLAG_RST | TCP_FLAG_ACK,
3764 TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
Florin Coras54ddf432018-12-21 13:54:09 -08003765 _(CLOSING, TCP_FLAG_RST, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3766 _(CLOSING, TCP_FLAG_RST | TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS,
3767 TCP_ERROR_NONE);
Florin Corasb56fcf12019-01-02 10:10:08 -08003768 _(CLOSING, TCP_FLAG_FIN, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
Florin Coras3c514d52018-12-22 11:39:33 -08003769 _(CLOSING, TCP_FLAG_FIN | TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS,
3770 TCP_ERROR_NONE);
Florin Corasb56fcf12019-01-02 10:10:08 -08003771 _(CLOSING, TCP_FLAG_FIN | TCP_FLAG_RST, TCP_INPUT_NEXT_RCV_PROCESS,
3772 TCP_ERROR_NONE);
3773 _(CLOSING, TCP_FLAG_FIN | TCP_FLAG_RST | TCP_FLAG_ACK,
3774 TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3775 _(CLOSING, TCP_FLAG_FIN | TCP_FLAG_SYN, TCP_INPUT_NEXT_RCV_PROCESS,
3776 TCP_ERROR_NONE);
3777 _(CLOSING, TCP_FLAG_FIN | TCP_FLAG_SYN | TCP_FLAG_ACK,
3778 TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3779 _(CLOSING, TCP_FLAG_FIN | TCP_FLAG_SYN | TCP_FLAG_RST | TCP_FLAG_ACK,
3780 TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
Dave Barach68b0fb02017-02-28 15:15:56 -05003781 /* FIN confirming that the peer (app) has closed */
3782 _(FIN_WAIT_2, TCP_FLAG_FIN, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
Florin Corasf6d68ed2017-05-07 19:12:02 -07003783 _(FIN_WAIT_2, TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
Dave Barach68b0fb02017-02-28 15:15:56 -05003784 _(FIN_WAIT_2, TCP_FLAG_FIN | TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS,
3785 TCP_ERROR_NONE);
Florin Coras678a6572018-12-17 21:31:25 -08003786 _(FIN_WAIT_2, TCP_FLAG_RST, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3787 _(FIN_WAIT_2, TCP_FLAG_RST | TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS,
3788 TCP_ERROR_NONE);
Florin Corasf03a59a2017-06-09 21:07:32 -07003789 _(CLOSE_WAIT, TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3790 _(CLOSE_WAIT, TCP_FLAG_FIN | TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS,
3791 TCP_ERROR_NONE);
Florin Coras678a6572018-12-17 21:31:25 -08003792 _(CLOSE_WAIT, TCP_FLAG_RST, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3793 _(CLOSE_WAIT, TCP_FLAG_RST | TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS,
3794 TCP_ERROR_NONE);
Florin Corasd12ff502018-12-25 10:55:01 -08003795 _(LAST_ACK, 0, TCP_INPUT_NEXT_DROP, TCP_ERROR_SEGMENT_INVALID);
Dave Barach68b0fb02017-02-28 15:15:56 -05003796 _(LAST_ACK, TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
Florin Coras93992a92017-05-24 18:03:56 -07003797 _(LAST_ACK, TCP_FLAG_FIN, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3798 _(LAST_ACK, TCP_FLAG_FIN | TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS,
3799 TCP_ERROR_NONE);
Florin Corasd12ff502018-12-25 10:55:01 -08003800 _(LAST_ACK, TCP_FLAG_FIN | TCP_FLAG_SYN, TCP_INPUT_NEXT_RCV_PROCESS,
3801 TCP_ERROR_NONE);
3802 _(LAST_ACK, TCP_FLAG_FIN | TCP_FLAG_SYN | TCP_FLAG_ACK,
3803 TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3804 _(LAST_ACK, TCP_FLAG_FIN | TCP_FLAG_RST, TCP_INPUT_NEXT_RCV_PROCESS,
3805 TCP_ERROR_NONE);
3806 _(LAST_ACK, TCP_FLAG_FIN | TCP_FLAG_RST | TCP_FLAG_ACK,
3807 TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3808 _(LAST_ACK, TCP_FLAG_FIN | TCP_FLAG_SYN | TCP_FLAG_RST,
3809 TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3810 _(LAST_ACK, TCP_FLAG_FIN | TCP_FLAG_SYN | TCP_FLAG_RST | TCP_FLAG_ACK,
3811 TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
Florin Corasdb84e572017-05-09 18:54:52 -07003812 _(LAST_ACK, TCP_FLAG_RST, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
Florin Coras678a6572018-12-17 21:31:25 -08003813 _(LAST_ACK, TCP_FLAG_RST | TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS,
3814 TCP_ERROR_NONE);
Florin Corasc01d5782018-10-17 14:53:11 -07003815 _(LAST_ACK, TCP_FLAG_SYN, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
Florin Corasd12ff502018-12-25 10:55:01 -08003816 _(LAST_ACK, TCP_FLAG_SYN | TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS,
3817 TCP_ERROR_NONE);
3818 _(LAST_ACK, TCP_FLAG_SYN | TCP_FLAG_RST, TCP_INPUT_NEXT_RCV_PROCESS,
3819 TCP_ERROR_NONE);
3820 _(LAST_ACK, TCP_FLAG_SYN | TCP_FLAG_RST | TCP_FLAG_ACK,
3821 TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
Florin Corasb56fcf12019-01-02 10:10:08 -08003822 _(TIME_WAIT, TCP_FLAG_SYN, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
Florin Coras93992a92017-05-24 18:03:56 -07003823 _(TIME_WAIT, TCP_FLAG_FIN, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
3824 _(TIME_WAIT, TCP_FLAG_FIN | TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS,
3825 TCP_ERROR_NONE);
Florin Coras3eb50622017-07-13 01:24:57 -04003826 _(TIME_WAIT, TCP_FLAG_RST, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
Florin Coras678a6572018-12-17 21:31:25 -08003827 _(TIME_WAIT, TCP_FLAG_RST | TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS,
3828 TCP_ERROR_NONE);
Florin Coras50958952017-08-29 14:50:13 -07003829 _(TIME_WAIT, TCP_FLAG_ACK, TCP_INPUT_NEXT_RCV_PROCESS, TCP_ERROR_NONE);
Florin Coras678a6572018-12-17 21:31:25 -08003830 /* RFC793 CLOSED: An incoming segment containing a RST is discarded. An
3831 * incoming segment not containing a RST causes a RST to be sent in
3832 * response.*/
Florin Corasdc629cd2017-05-09 00:52:37 -07003833 _(CLOSED, TCP_FLAG_RST, TCP_INPUT_NEXT_DROP, TCP_ERROR_CONNECTION_CLOSED);
Florin Coras678a6572018-12-17 21:31:25 -08003834 _(CLOSED, TCP_FLAG_RST | TCP_FLAG_ACK, TCP_INPUT_NEXT_DROP,
Dave Barach2c25a622017-06-26 11:35:07 -04003835 TCP_ERROR_CONNECTION_CLOSED);
Florin Coras678a6572018-12-17 21:31:25 -08003836 _(CLOSED, TCP_FLAG_ACK, TCP_INPUT_NEXT_RESET, TCP_ERROR_NONE);
3837 _(CLOSED, TCP_FLAG_SYN, TCP_INPUT_NEXT_RESET, TCP_ERROR_NONE);
3838 _(CLOSED, TCP_FLAG_FIN | TCP_FLAG_ACK, TCP_INPUT_NEXT_RESET,
3839 TCP_ERROR_NONE);
Dave Barach68b0fb02017-02-28 15:15:56 -05003840#undef _
3841}
3842
Florin Coras0dbd5172018-06-25 16:19:34 -07003843static clib_error_t *
Dave Barach68b0fb02017-02-28 15:15:56 -05003844tcp_input_init (vlib_main_t * vm)
3845{
3846 clib_error_t *error = 0;
3847 tcp_main_t *tm = vnet_get_tcp_main ();
3848
3849 if ((error = vlib_call_init_function (vm, tcp_init)))
3850 return error;
3851
3852 /* Initialize dispatch table. */
3853 tcp_dispatch_table_init (tm);
3854
3855 return error;
3856}
3857
3858VLIB_INIT_FUNCTION (tcp_input_init);
3859
Filip Tehlare275bed2019-03-06 00:06:56 -08003860#endif /* CLIB_MARCH_VARIANT */
3861
Dave Barach68b0fb02017-02-28 15:15:56 -05003862/*
3863 * fd.io coding-style-patch-verification: ON
3864 *
3865 * Local Variables:
3866 * eval: (c-set-style "gnu")
3867 * End:
3868 */