blob: 239f6035d5f49f13648adaa2bbb0e7758227e91a [file] [log] [blame]
Dave Barach68b0fb02017-02-28 15:15:56 -05001/*
Florin Coras222e1f412019-02-16 20:47:32 -08002 * Copyright (c) 2016-2019 Cisco and/or its affiliates.
Dave Barach68b0fb02017-02-28 15:15:56 -05003 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16#include <vnet/tcp/tcp.h>
Florin Corasb2215d62017-08-01 16:56:58 -070017#include <math.h>
Dave Barach68b0fb02017-02-28 15:15:56 -050018
Dave Barach2c25a622017-06-26 11:35:07 -040019typedef enum _tcp_output_next
Dave Barach68b0fb02017-02-28 15:15:56 -050020{
21 TCP_OUTPUT_NEXT_DROP,
Dave Barach2c25a622017-06-26 11:35:07 -040022 TCP_OUTPUT_NEXT_IP_LOOKUP,
Florin Corasf9d05682018-04-26 08:26:52 -070023 TCP_OUTPUT_NEXT_IP_REWRITE,
24 TCP_OUTPUT_NEXT_IP_ARP,
Dave Barach68b0fb02017-02-28 15:15:56 -050025 TCP_OUTPUT_N_NEXT
26} tcp_output_next_t;
27
28#define foreach_tcp4_output_next \
29 _ (DROP, "error-drop") \
Florin Corasf9d05682018-04-26 08:26:52 -070030 _ (IP_LOOKUP, "ip4-lookup") \
31 _ (IP_REWRITE, "ip4-rewrite") \
32 _ (IP_ARP, "ip4-arp")
Dave Barach68b0fb02017-02-28 15:15:56 -050033
34#define foreach_tcp6_output_next \
35 _ (DROP, "error-drop") \
Florin Corasf9d05682018-04-26 08:26:52 -070036 _ (IP_LOOKUP, "ip6-lookup") \
37 _ (IP_REWRITE, "ip6-rewrite") \
38 _ (IP_ARP, "ip6-discover-neighbor")
Dave Barach68b0fb02017-02-28 15:15:56 -050039
40static char *tcp_error_strings[] = {
41#define tcp_error(n,s) s,
42#include <vnet/tcp/tcp_error.def>
43#undef tcp_error
44};
45
46typedef struct
47{
Clement Durand6cf260c2017-04-13 13:27:04 +020048 tcp_header_t tcp_header;
49 tcp_connection_t tcp_connection;
Dave Barach68b0fb02017-02-28 15:15:56 -050050} tcp_tx_trace_t;
51
Filip Tehlare275bed2019-03-06 00:06:56 -080052static u8 *
Dave Barach68b0fb02017-02-28 15:15:56 -050053format_tcp_tx_trace (u8 * s, va_list * args)
54{
55 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
56 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
Clement Durand6cf260c2017-04-13 13:27:04 +020057 tcp_tx_trace_t *t = va_arg (*args, tcp_tx_trace_t *);
Christophe Fontained3c008d2017-10-02 18:10:54 +020058 u32 indent = format_get_indent (s);
Dave Barach68b0fb02017-02-28 15:15:56 -050059
Clement Durand6cf260c2017-04-13 13:27:04 +020060 s = format (s, "%U\n%U%U",
61 format_tcp_header, &t->tcp_header, 128,
62 format_white_space, indent,
Florin Corasbb292f42017-05-19 09:49:19 -070063 format_tcp_connection, &t->tcp_connection, 1);
Dave Barach68b0fb02017-02-28 15:15:56 -050064
65 return s;
66}
67
Filip Tehlare275bed2019-03-06 00:06:56 -080068#ifndef CLIB_MARCH_VARIANT
Dave Barach68b0fb02017-02-28 15:15:56 -050069static u8
Florin Coras4eeeaaf2017-09-05 14:03:37 -040070tcp_window_compute_scale (u32 window)
Dave Barach68b0fb02017-02-28 15:15:56 -050071{
72 u8 wnd_scale = 0;
Florin Coras4eeeaaf2017-09-05 14:03:37 -040073 while (wnd_scale < TCP_MAX_WND_SCALE && (window >> wnd_scale) > TCP_WND_MAX)
Dave Barach68b0fb02017-02-28 15:15:56 -050074 wnd_scale++;
75 return wnd_scale;
76}
77
78/**
Florin Coras6534b7a2017-07-18 05:38:03 -040079 * TCP's initial window
Florin Corase04c2992017-03-01 08:17:34 -080080 */
81always_inline u32
82tcp_initial_wnd_unscaled (tcp_connection_t * tc)
83{
Florin Coras6534b7a2017-07-18 05:38:03 -040084 /* RFC 6928 recommends the value lower. However at the time our connections
85 * are initialized, fifos may not be allocated. Therefore, advertise the
86 * smallest possible unscaled window size and update once fifos are
87 * assigned to the session.
88 */
89 /*
90 tcp_update_rcv_mss (tc);
91 TCP_IW_N_SEGMENTS * tc->mss;
92 */
Florin Coras9094b5c2019-08-12 14:17:47 -070093 return tcp_cfg.min_rx_fifo;
Florin Corase04c2992017-03-01 08:17:34 -080094}
95
96/**
Dave Barach68b0fb02017-02-28 15:15:56 -050097 * Compute initial window and scale factor. As per RFC1323, window field in
98 * SYN and SYN-ACK segments is never scaled.
99 */
100u32
101tcp_initial_window_to_advertise (tcp_connection_t * tc)
102{
Florin Corase80b5912018-12-12 19:25:43 -0800103 /* Compute rcv wscale only if peer advertised support for it */
104 if (tc->state != TCP_STATE_SYN_RCVD || tcp_opts_wscale (&tc->rcv_opts))
Florin Coras9094b5c2019-08-12 14:17:47 -0700105 tc->rcv_wscale = tcp_window_compute_scale (tcp_cfg.max_rx_fifo);
Florin Corase80b5912018-12-12 19:25:43 -0800106
Florin Corase04c2992017-03-01 08:17:34 -0800107 tc->rcv_wnd = tcp_initial_wnd_unscaled (tc);
Dave Barach68b0fb02017-02-28 15:15:56 -0500108
109 return clib_min (tc->rcv_wnd, TCP_WND_MAX);
110}
111
Florin Corasf20fd1a2019-03-28 13:21:19 -0700112static inline void
Florin Coras6792ec02017-03-13 03:49:51 -0700113tcp_update_rcv_wnd (tcp_connection_t * tc)
114{
Florin Corasf20fd1a2019-03-28 13:21:19 -0700115 u32 available_space, wnd;
Florin Coras6792ec02017-03-13 03:49:51 -0700116 i32 observed_wnd;
Florin Corasf20fd1a2019-03-28 13:21:19 -0700117
118 ASSERT (tc->rcv_opts.mss < transport_rx_fifo_size (&tc->connection));
Florin Coras6792ec02017-03-13 03:49:51 -0700119
Florin Corase04c2992017-03-01 08:17:34 -0800120 /*
121 * Figure out how much space we have available
122 */
Florin Corasd2aab832018-05-22 11:39:59 -0700123 available_space = transport_max_rx_enqueue (&tc->connection);
Florin Corasf20fd1a2019-03-28 13:21:19 -0700124 if (PREDICT_FALSE (available_space < tc->rcv_opts.mss))
Florin Corase04c2992017-03-01 08:17:34 -0800125 available_space = 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500126
Florin Corase04c2992017-03-01 08:17:34 -0800127 /*
128 * Use the above and what we know about what we've previously advertised
129 * to compute the new window
130 */
Florin Coras6792ec02017-03-13 03:49:51 -0700131 observed_wnd = (i32) tc->rcv_wnd - (tc->rcv_nxt - tc->rcv_las);
Florin Corase04c2992017-03-01 08:17:34 -0800132
133 /* Bad. Thou shalt not shrink */
Florin Corasf20fd1a2019-03-28 13:21:19 -0700134 if (PREDICT_FALSE ((i32) available_space < observed_wnd))
Florin Corase04c2992017-03-01 08:17:34 -0800135 {
Florin Corasf20fd1a2019-03-28 13:21:19 -0700136 wnd = clib_max (observed_wnd, 0);
Florin Corasa436a422019-08-20 07:09:31 -0700137 TCP_EVT (TCP_EVT_RCV_WND_SHRUNK, tc, observed_wnd, available_space);
Florin Corase04c2992017-03-01 08:17:34 -0800138 }
Florin Coras6792ec02017-03-13 03:49:51 -0700139 else
Florin Corase04c2992017-03-01 08:17:34 -0800140 {
Florin Coras6792ec02017-03-13 03:49:51 -0700141 wnd = available_space;
Florin Corase04c2992017-03-01 08:17:34 -0800142 }
143
Florin Coras3e350af2017-03-30 02:54:28 -0700144 /* Make sure we have a multiple of rcv_wscale */
145 if (wnd && tc->rcv_wscale)
146 {
Florin Coras679b78f2019-03-06 21:16:26 -0800147 wnd &= ~((1 << tc->rcv_wscale) - 1);
Florin Coras3e350af2017-03-30 02:54:28 -0700148 if (wnd == 0)
149 wnd = 1 << tc->rcv_wscale;
150 }
Florin Coras6792ec02017-03-13 03:49:51 -0700151
152 tc->rcv_wnd = clib_min (wnd, TCP_WND_MAX << tc->rcv_wscale);
Dave Barach68b0fb02017-02-28 15:15:56 -0500153}
154
155/**
Florin Coras0dbd5172018-06-25 16:19:34 -0700156 * Compute and return window to advertise, scaled as per RFC1323
157 */
Florin Coras47596832019-03-12 18:58:54 -0700158static inline u32
Florin Coras0dbd5172018-06-25 16:19:34 -0700159tcp_window_to_advertise (tcp_connection_t * tc, tcp_state_t state)
160{
161 if (state < TCP_STATE_ESTABLISHED)
162 return tcp_initial_window_to_advertise (tc);
163
164 tcp_update_rcv_wnd (tc);
Florin Coras0dbd5172018-06-25 16:19:34 -0700165 return tc->rcv_wnd >> tc->rcv_wscale;
166}
167
168/**
Dave Barach68b0fb02017-02-28 15:15:56 -0500169 * Write TCP options to segment.
170 */
Florin Coras0dbd5172018-06-25 16:19:34 -0700171static u32
Dave Barach68b0fb02017-02-28 15:15:56 -0500172tcp_options_write (u8 * data, tcp_options_t * opts)
173{
174 u32 opts_len = 0;
175 u32 buf, seq_len = 4;
176
177 if (tcp_opts_mss (opts))
178 {
179 *data++ = TCP_OPTION_MSS;
180 *data++ = TCP_OPTION_LEN_MSS;
181 buf = clib_host_to_net_u16 (opts->mss);
Dave Barach178cf492018-11-13 16:34:13 -0500182 clib_memcpy_fast (data, &buf, sizeof (opts->mss));
Dave Barach68b0fb02017-02-28 15:15:56 -0500183 data += sizeof (opts->mss);
184 opts_len += TCP_OPTION_LEN_MSS;
185 }
186
187 if (tcp_opts_wscale (opts))
188 {
189 *data++ = TCP_OPTION_WINDOW_SCALE;
190 *data++ = TCP_OPTION_LEN_WINDOW_SCALE;
191 *data++ = opts->wscale;
192 opts_len += TCP_OPTION_LEN_WINDOW_SCALE;
193 }
194
195 if (tcp_opts_sack_permitted (opts))
196 {
197 *data++ = TCP_OPTION_SACK_PERMITTED;
198 *data++ = TCP_OPTION_LEN_SACK_PERMITTED;
199 opts_len += TCP_OPTION_LEN_SACK_PERMITTED;
200 }
201
202 if (tcp_opts_tstamp (opts))
203 {
204 *data++ = TCP_OPTION_TIMESTAMP;
205 *data++ = TCP_OPTION_LEN_TIMESTAMP;
206 buf = clib_host_to_net_u32 (opts->tsval);
Dave Barach178cf492018-11-13 16:34:13 -0500207 clib_memcpy_fast (data, &buf, sizeof (opts->tsval));
Dave Barach68b0fb02017-02-28 15:15:56 -0500208 data += sizeof (opts->tsval);
209 buf = clib_host_to_net_u32 (opts->tsecr);
Dave Barach178cf492018-11-13 16:34:13 -0500210 clib_memcpy_fast (data, &buf, sizeof (opts->tsecr));
Dave Barach68b0fb02017-02-28 15:15:56 -0500211 data += sizeof (opts->tsecr);
212 opts_len += TCP_OPTION_LEN_TIMESTAMP;
213 }
214
215 if (tcp_opts_sack (opts))
216 {
217 int i;
Dave Barach68b0fb02017-02-28 15:15:56 -0500218
Florin Corase5b17912019-02-21 16:46:24 -0800219 if (opts->n_sack_blocks != 0)
Dave Barach68b0fb02017-02-28 15:15:56 -0500220 {
221 *data++ = TCP_OPTION_SACK_BLOCK;
Florin Corase5b17912019-02-21 16:46:24 -0800222 *data++ = 2 + opts->n_sack_blocks * TCP_OPTION_LEN_SACK_BLOCK;
223 for (i = 0; i < opts->n_sack_blocks; i++)
Dave Barach68b0fb02017-02-28 15:15:56 -0500224 {
225 buf = clib_host_to_net_u32 (opts->sacks[i].start);
Dave Barach178cf492018-11-13 16:34:13 -0500226 clib_memcpy_fast (data, &buf, seq_len);
Dave Barach68b0fb02017-02-28 15:15:56 -0500227 data += seq_len;
228 buf = clib_host_to_net_u32 (opts->sacks[i].end);
Dave Barach178cf492018-11-13 16:34:13 -0500229 clib_memcpy_fast (data, &buf, seq_len);
Dave Barach68b0fb02017-02-28 15:15:56 -0500230 data += seq_len;
231 }
Florin Corase5b17912019-02-21 16:46:24 -0800232 opts_len += 2 + opts->n_sack_blocks * TCP_OPTION_LEN_SACK_BLOCK;
Dave Barach68b0fb02017-02-28 15:15:56 -0500233 }
234 }
235
236 /* Terminate TCP options */
237 if (opts_len % 4)
238 {
239 *data++ = TCP_OPTION_EOL;
240 opts_len += TCP_OPTION_LEN_EOL;
241 }
242
243 /* Pad with zeroes to a u32 boundary */
244 while (opts_len % 4)
245 {
246 *data++ = TCP_OPTION_NOOP;
247 opts_len += TCP_OPTION_LEN_NOOP;
248 }
249 return opts_len;
250}
251
Florin Coras0dbd5172018-06-25 16:19:34 -0700252static int
Florin Corascedcf602019-08-27 12:15:43 -0700253tcp_make_syn_options (tcp_connection_t * tc, tcp_options_t * opts)
Dave Barach68b0fb02017-02-28 15:15:56 -0500254{
255 u8 len = 0;
256
257 opts->flags |= TCP_OPTS_FLAG_MSS;
Florin Corascedcf602019-08-27 12:15:43 -0700258 opts->mss = tc->mss;
Dave Barach68b0fb02017-02-28 15:15:56 -0500259 len += TCP_OPTION_LEN_MSS;
260
261 opts->flags |= TCP_OPTS_FLAG_WSCALE;
Florin Corascedcf602019-08-27 12:15:43 -0700262 opts->wscale = tc->rcv_wscale;
Dave Barach68b0fb02017-02-28 15:15:56 -0500263 len += TCP_OPTION_LEN_WINDOW_SCALE;
264
265 opts->flags |= TCP_OPTS_FLAG_TSTAMP;
266 opts->tsval = tcp_time_now ();
267 opts->tsecr = 0;
268 len += TCP_OPTION_LEN_TIMESTAMP;
269
Florin Coras93992a92017-05-24 18:03:56 -0700270 if (TCP_USE_SACKS)
271 {
272 opts->flags |= TCP_OPTS_FLAG_SACK_PERMITTED;
273 len += TCP_OPTION_LEN_SACK_PERMITTED;
274 }
Dave Barach68b0fb02017-02-28 15:15:56 -0500275
276 /* Align to needed boundary */
277 len += (TCP_OPTS_ALIGN - len % TCP_OPTS_ALIGN) % TCP_OPTS_ALIGN;
278 return len;
279}
280
Florin Coras0dbd5172018-06-25 16:19:34 -0700281static int
Dave Barach68b0fb02017-02-28 15:15:56 -0500282tcp_make_synack_options (tcp_connection_t * tc, tcp_options_t * opts)
283{
284 u8 len = 0;
285
286 opts->flags |= TCP_OPTS_FLAG_MSS;
Florin Corasc8343412017-05-04 14:25:50 -0700287 opts->mss = tc->mss;
Dave Barach68b0fb02017-02-28 15:15:56 -0500288 len += TCP_OPTION_LEN_MSS;
289
Florin Coras93992a92017-05-24 18:03:56 -0700290 if (tcp_opts_wscale (&tc->rcv_opts))
Dave Barach68b0fb02017-02-28 15:15:56 -0500291 {
292 opts->flags |= TCP_OPTS_FLAG_WSCALE;
293 opts->wscale = tc->rcv_wscale;
294 len += TCP_OPTION_LEN_WINDOW_SCALE;
295 }
296
Florin Coras93992a92017-05-24 18:03:56 -0700297 if (tcp_opts_tstamp (&tc->rcv_opts))
Dave Barach68b0fb02017-02-28 15:15:56 -0500298 {
299 opts->flags |= TCP_OPTS_FLAG_TSTAMP;
300 opts->tsval = tcp_time_now ();
301 opts->tsecr = tc->tsval_recent;
302 len += TCP_OPTION_LEN_TIMESTAMP;
303 }
304
Florin Coras93992a92017-05-24 18:03:56 -0700305 if (tcp_opts_sack_permitted (&tc->rcv_opts))
Dave Barach68b0fb02017-02-28 15:15:56 -0500306 {
307 opts->flags |= TCP_OPTS_FLAG_SACK_PERMITTED;
308 len += TCP_OPTION_LEN_SACK_PERMITTED;
309 }
310
311 /* Align to needed boundary */
312 len += (TCP_OPTS_ALIGN - len % TCP_OPTS_ALIGN) % TCP_OPTS_ALIGN;
313 return len;
314}
315
Florin Coras0dbd5172018-06-25 16:19:34 -0700316static int
Dave Barach68b0fb02017-02-28 15:15:56 -0500317tcp_make_established_options (tcp_connection_t * tc, tcp_options_t * opts)
318{
319 u8 len = 0;
320
321 opts->flags = 0;
322
Florin Coras93992a92017-05-24 18:03:56 -0700323 if (tcp_opts_tstamp (&tc->rcv_opts))
Dave Barach68b0fb02017-02-28 15:15:56 -0500324 {
325 opts->flags |= TCP_OPTS_FLAG_TSTAMP;
Vladimir Kropylev1cfcb782019-07-02 11:25:26 +0300326 opts->tsval = tcp_tstamp (tc);
Dave Barach68b0fb02017-02-28 15:15:56 -0500327 opts->tsecr = tc->tsval_recent;
328 len += TCP_OPTION_LEN_TIMESTAMP;
329 }
Florin Coras93992a92017-05-24 18:03:56 -0700330 if (tcp_opts_sack_permitted (&tc->rcv_opts))
Dave Barach68b0fb02017-02-28 15:15:56 -0500331 {
332 if (vec_len (tc->snd_sacks))
333 {
334 opts->flags |= TCP_OPTS_FLAG_SACK;
Florin Corase5b17912019-02-21 16:46:24 -0800335 if (tc->snd_sack_pos >= vec_len (tc->snd_sacks))
336 tc->snd_sack_pos = 0;
337 opts->sacks = &tc->snd_sacks[tc->snd_sack_pos];
338 opts->n_sack_blocks = vec_len (tc->snd_sacks) - tc->snd_sack_pos;
339 opts->n_sack_blocks = clib_min (opts->n_sack_blocks,
Florin Corasc28764f2017-04-26 00:08:42 -0700340 TCP_OPTS_MAX_SACK_BLOCKS);
Florin Corase5b17912019-02-21 16:46:24 -0800341 tc->snd_sack_pos += opts->n_sack_blocks;
Dave Barach68b0fb02017-02-28 15:15:56 -0500342 len += 2 + TCP_OPTION_LEN_SACK_BLOCK * opts->n_sack_blocks;
343 }
344 }
345
346 /* Align to needed boundary */
347 len += (TCP_OPTS_ALIGN - len % TCP_OPTS_ALIGN) % TCP_OPTS_ALIGN;
348 return len;
349}
350
351always_inline int
352tcp_make_options (tcp_connection_t * tc, tcp_options_t * opts,
353 tcp_state_t state)
354{
355 switch (state)
356 {
357 case TCP_STATE_ESTABLISHED:
Florin Coras25579b42018-06-06 17:55:02 -0700358 case TCP_STATE_CLOSE_WAIT:
Florin Coras54ddf432018-12-21 13:54:09 -0800359 case TCP_STATE_FIN_WAIT_1:
360 case TCP_STATE_LAST_ACK:
361 case TCP_STATE_CLOSING:
362 case TCP_STATE_FIN_WAIT_2:
363 case TCP_STATE_TIME_WAIT:
364 case TCP_STATE_CLOSED:
Dave Barach68b0fb02017-02-28 15:15:56 -0500365 return tcp_make_established_options (tc, opts);
366 case TCP_STATE_SYN_RCVD:
367 return tcp_make_synack_options (tc, opts);
368 case TCP_STATE_SYN_SENT:
Florin Corascedcf602019-08-27 12:15:43 -0700369 return tcp_make_syn_options (tc, opts);
Dave Barach68b0fb02017-02-28 15:15:56 -0500370 default:
Florin Coras371ca502018-02-21 12:07:41 -0800371 clib_warning ("State not handled! %d", state);
Dave Barach68b0fb02017-02-28 15:15:56 -0500372 return 0;
373 }
374}
375
Florin Corasc8343412017-05-04 14:25:50 -0700376/**
Florin Corasb26743d2018-06-26 09:31:04 -0700377 * Update burst send vars
378 *
379 * - Updates snd_mss to reflect the effective segment size that we can send
380 * by taking into account all TCP options, including SACKs.
381 * - Cache 'on the wire' options for reuse
382 * - Updates receive window which can be reused for a burst.
383 *
384 * This should *only* be called when doing bursts
Florin Corasc8343412017-05-04 14:25:50 -0700385 */
386void
Florin Corasb26743d2018-06-26 09:31:04 -0700387tcp_update_burst_snd_vars (tcp_connection_t * tc)
Florin Corasc8343412017-05-04 14:25:50 -0700388{
Florin Corasb26743d2018-06-26 09:31:04 -0700389 tcp_main_t *tm = &tcp_main;
390
Florin Corasc8343412017-05-04 14:25:50 -0700391 /* Compute options to be used for connection. These may be reused when
392 * sending data or to compute the effective mss (snd_mss) */
Florin Corasb26743d2018-06-26 09:31:04 -0700393 tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts,
394 TCP_STATE_ESTABLISHED);
Florin Corasc8343412017-05-04 14:25:50 -0700395
396 /* XXX check if MTU has been updated */
Florin Coras93992a92017-05-24 18:03:56 -0700397 tc->snd_mss = clib_min (tc->mss, tc->rcv_opts.mss) - tc->snd_opts_len;
Florin Corasdb84e572017-05-09 18:54:52 -0700398 ASSERT (tc->snd_mss > 0);
Florin Corasb26743d2018-06-26 09:31:04 -0700399
400 tcp_options_write (tm->wrk_ctx[tc->c_thread_index].cached_opts,
401 &tc->snd_opts);
402
403 tcp_update_rcv_wnd (tc);
Florin Coras52814732019-06-12 15:38:19 -0700404
405 if (tc->flags & TCP_CONN_RATE_SAMPLE)
406 tc->flags |= TCP_CONN_TRACK_BURST;
Florin Corasca2831a2019-07-04 17:05:59 -0700407
408 if (tc->snd_una == tc->snd_nxt)
409 tcp_cc_event (tc, TCP_CC_EVT_START_TX);
Florin Corasc8343412017-05-04 14:25:50 -0700410}
411
Filip Tehlare275bed2019-03-06 00:06:56 -0800412#endif /* CLIB_MARCH_VARIANT */
Florin Corasc8343412017-05-04 14:25:50 -0700413
Florin Coras0dbd5172018-06-25 16:19:34 -0700414static void *
Dave Barach68b0fb02017-02-28 15:15:56 -0500415tcp_reuse_buffer (vlib_main_t * vm, vlib_buffer_t * b)
416{
Florin Corasb2215d62017-08-01 16:56:58 -0700417 if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
418 vlib_buffer_free_one (vm, b->next_buffer);
Florin Coras4eeeaaf2017-09-05 14:03:37 -0400419 /* Zero all flags but free list index and trace flag */
420 b->flags &= VLIB_BUFFER_NEXT_PRESENT - 1;
Florin Coras1f152cd2017-08-18 19:28:03 -0700421 b->current_data = 0;
422 b->current_length = 0;
423 b->total_length_not_including_first_buffer = 0;
424 vnet_buffer (b)->tcp.flags = 0;
Florin Corasb2215d62017-08-01 16:56:58 -0700425
Dave Barach68b0fb02017-02-28 15:15:56 -0500426 /* Leave enough space for headers */
Florin Coras1ee78302019-02-05 15:51:15 -0800427 return vlib_buffer_make_headroom (b, TRANSPORT_MAX_HDRS_LEN);
Florin Coras1f152cd2017-08-18 19:28:03 -0700428}
429
Filip Tehlare275bed2019-03-06 00:06:56 -0800430#ifndef CLIB_MARCH_VARIANT
Florin Coras0dbd5172018-06-25 16:19:34 -0700431static void *
Florin Coras1f152cd2017-08-18 19:28:03 -0700432tcp_init_buffer (vlib_main_t * vm, vlib_buffer_t * b)
433{
434 ASSERT ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
Florin Coras4eeeaaf2017-09-05 14:03:37 -0400435 b->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
Florin Coras1f152cd2017-08-18 19:28:03 -0700436 b->total_length_not_including_first_buffer = 0;
Florin Coras24793e32018-05-09 11:34:25 -0700437 b->current_data = 0;
Florin Corasd79b41e2017-03-04 05:37:52 -0800438 vnet_buffer (b)->tcp.flags = 0;
Florin Corasf988e692017-11-27 04:34:14 -0500439 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b);
Florin Coras1f152cd2017-08-18 19:28:03 -0700440 /* Leave enough space for headers */
Florin Coras1ee78302019-02-05 15:51:15 -0800441 return vlib_buffer_make_headroom (b, TRANSPORT_MAX_HDRS_LEN);
Dave Barach68b0fb02017-02-28 15:15:56 -0500442}
443
444/**
445 * Prepare ACK
446 */
Florin Corasbdf7fd62019-01-31 17:31:01 -0800447static inline void
Dave Barach68b0fb02017-02-28 15:15:56 -0500448tcp_make_ack_i (tcp_connection_t * tc, vlib_buffer_t * b, tcp_state_t state,
449 u8 flags)
450{
451 tcp_options_t _snd_opts, *snd_opts = &_snd_opts;
452 u8 tcp_opts_len, tcp_hdr_opts_len;
453 tcp_header_t *th;
454 u16 wnd;
455
456 wnd = tcp_window_to_advertise (tc, state);
457
458 /* Make and write options */
459 tcp_opts_len = tcp_make_established_options (tc, snd_opts);
460 tcp_hdr_opts_len = tcp_opts_len + sizeof (tcp_header_t);
461
462 th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, tc->snd_nxt,
463 tc->rcv_nxt, tcp_hdr_opts_len, flags, wnd);
464
465 tcp_options_write ((u8 *) (th + 1), snd_opts);
Dave Barach68b0fb02017-02-28 15:15:56 -0500466 vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
Vladimir Kropylev398afbd2019-06-25 00:06:52 +0300467
468 if (wnd == 0)
469 tcp_zero_rwnd_sent_on (tc);
470 else
471 tcp_zero_rwnd_sent_off (tc);
Dave Barach68b0fb02017-02-28 15:15:56 -0500472}
473
474/**
475 * Convert buffer to ACK
476 */
Florin Corasbdf7fd62019-01-31 17:31:01 -0800477static inline void
Dave Barach68b0fb02017-02-28 15:15:56 -0500478tcp_make_ack (tcp_connection_t * tc, vlib_buffer_t * b)
479{
Dave Barach68b0fb02017-02-28 15:15:56 -0500480 tcp_make_ack_i (tc, b, TCP_STATE_ESTABLISHED, TCP_FLAG_ACK);
Florin Corasa436a422019-08-20 07:09:31 -0700481 TCP_EVT (TCP_EVT_ACK_SENT, tc);
Florin Coras3e350af2017-03-30 02:54:28 -0700482 tc->rcv_las = tc->rcv_nxt;
Dave Barach68b0fb02017-02-28 15:15:56 -0500483}
484
485/**
486 * Convert buffer to FIN-ACK
487 */
488void
Florin Corasd79b41e2017-03-04 05:37:52 -0800489tcp_make_fin (tcp_connection_t * tc, vlib_buffer_t * b)
Dave Barach68b0fb02017-02-28 15:15:56 -0500490{
Florin Corasbdf7fd62019-01-31 17:31:01 -0800491 tcp_make_ack_i (tc, b, TCP_STATE_ESTABLISHED, TCP_FLAG_FIN | TCP_FLAG_ACK);
Florin Coras4eeeaaf2017-09-05 14:03:37 -0400492}
Dave Barach68b0fb02017-02-28 15:15:56 -0500493
Florin Coras4eeeaaf2017-09-05 14:03:37 -0400494/**
495 * Convert buffer to SYN
496 */
497void
498tcp_make_syn (tcp_connection_t * tc, vlib_buffer_t * b)
499{
500 u8 tcp_hdr_opts_len, tcp_opts_len;
501 tcp_header_t *th;
502 u16 initial_wnd;
503 tcp_options_t snd_opts;
504
505 initial_wnd = tcp_initial_window_to_advertise (tc);
506
507 /* Make and write options */
Dave Barachb7b92992018-10-17 10:38:51 -0400508 clib_memset (&snd_opts, 0, sizeof (snd_opts));
Florin Corascedcf602019-08-27 12:15:43 -0700509 tcp_opts_len = tcp_make_syn_options (tc, &snd_opts);
Florin Coras4eeeaaf2017-09-05 14:03:37 -0400510 tcp_hdr_opts_len = tcp_opts_len + sizeof (tcp_header_t);
511
512 th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, tc->iss,
513 tc->rcv_nxt, tcp_hdr_opts_len, TCP_FLAG_SYN,
514 initial_wnd);
515 vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
516 tcp_options_write ((u8 *) (th + 1), &snd_opts);
Dave Barach68b0fb02017-02-28 15:15:56 -0500517}
518
519/**
520 * Convert buffer to SYN-ACK
521 */
522void
523tcp_make_synack (tcp_connection_t * tc, vlib_buffer_t * b)
524{
Dave Barach68b0fb02017-02-28 15:15:56 -0500525 tcp_options_t _snd_opts, *snd_opts = &_snd_opts;
526 u8 tcp_opts_len, tcp_hdr_opts_len;
527 tcp_header_t *th;
528 u16 initial_wnd;
Dave Barach68b0fb02017-02-28 15:15:56 -0500529
Dave Barachb7b92992018-10-17 10:38:51 -0400530 clib_memset (snd_opts, 0, sizeof (*snd_opts));
Dave Barach68b0fb02017-02-28 15:15:56 -0500531 initial_wnd = tcp_initial_window_to_advertise (tc);
Dave Barach68b0fb02017-02-28 15:15:56 -0500532 tcp_opts_len = tcp_make_synack_options (tc, snd_opts);
533 tcp_hdr_opts_len = tcp_opts_len + sizeof (tcp_header_t);
534
535 th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, tc->iss,
536 tc->rcv_nxt, tcp_hdr_opts_len,
537 TCP_FLAG_SYN | TCP_FLAG_ACK, initial_wnd);
Dave Barach68b0fb02017-02-28 15:15:56 -0500538 tcp_options_write ((u8 *) (th + 1), snd_opts);
539
540 vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
Dave Barach68b0fb02017-02-28 15:15:56 -0500541}
542
543always_inline void
Florin Corasbe72ae62018-11-01 11:23:03 -0700544tcp_enqueue_to_ip_lookup_i (tcp_worker_ctx_t * wrk, vlib_buffer_t * b, u32 bi,
Florin Coras56b39f62018-03-27 17:29:32 -0700545 u8 is_ip4, u32 fib_index, u8 flush)
Dave Barach68b0fb02017-02-28 15:15:56 -0500546{
Florin Corasbe72ae62018-11-01 11:23:03 -0700547 vlib_main_t *vm = wrk->vm;
Dave Barach68b0fb02017-02-28 15:15:56 -0500548 u32 *to_next, next_index;
549 vlib_frame_t *f;
550
Damjan Marion213b5aa2017-07-13 21:19:27 +0200551 b->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
Dave Barach68b0fb02017-02-28 15:15:56 -0500552 b->error = 0;
553
Florin Coras56b39f62018-03-27 17:29:32 -0700554 vnet_buffer (b)->sw_if_index[VLIB_TX] = fib_index;
555 vnet_buffer (b)->sw_if_index[VLIB_RX] = 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500556
557 /* Send to IP lookup */
558 next_index = is_ip4 ? ip4_lookup_node.index : ip6_lookup_node.index;
Florin Corasf988e692017-11-27 04:34:14 -0500559 tcp_trajectory_add_start (b, 1);
Dave Barach68b0fb02017-02-28 15:15:56 -0500560
Florin Corasbe72ae62018-11-01 11:23:03 -0700561 f = wrk->ip_lookup_tx_frames[!is_ip4];
Florin Coras9d063042017-09-14 03:08:00 -0400562 if (!f)
563 {
564 f = vlib_get_frame_to_node (vm, next_index);
565 ASSERT (f);
Florin Corasbe72ae62018-11-01 11:23:03 -0700566 wrk->ip_lookup_tx_frames[!is_ip4] = f;
Florin Coras9d063042017-09-14 03:08:00 -0400567 }
568
Dave Barach68b0fb02017-02-28 15:15:56 -0500569 to_next = vlib_frame_vector_args (f);
Florin Coras9d063042017-09-14 03:08:00 -0400570 to_next[f->n_vectors] = bi;
571 f->n_vectors += 1;
572 if (flush || f->n_vectors == VLIB_FRAME_SIZE)
573 {
574 vlib_put_frame_to_node (vm, next_index, f);
Florin Corasbe72ae62018-11-01 11:23:03 -0700575 wrk->ip_lookup_tx_frames[!is_ip4] = 0;
Florin Coras9d063042017-09-14 03:08:00 -0400576 }
577}
578
Florin Coras0dbd5172018-06-25 16:19:34 -0700579static void
Florin Corasbe72ae62018-11-01 11:23:03 -0700580tcp_enqueue_to_ip_lookup_now (tcp_worker_ctx_t * wrk, vlib_buffer_t * b,
581 u32 bi, u8 is_ip4, u32 fib_index)
Florin Coras9d063042017-09-14 03:08:00 -0400582{
Florin Corasbe72ae62018-11-01 11:23:03 -0700583 tcp_enqueue_to_ip_lookup_i (wrk, b, bi, is_ip4, fib_index, 1);
Florin Coras9d063042017-09-14 03:08:00 -0400584}
585
Florin Coras0dbd5172018-06-25 16:19:34 -0700586static void
Florin Corasbe72ae62018-11-01 11:23:03 -0700587tcp_enqueue_to_ip_lookup (tcp_worker_ctx_t * wrk, vlib_buffer_t * b, u32 bi,
Florin Coras56b39f62018-03-27 17:29:32 -0700588 u8 is_ip4, u32 fib_index)
Florin Coras9d063042017-09-14 03:08:00 -0400589{
Florin Corasbe72ae62018-11-01 11:23:03 -0700590 tcp_enqueue_to_ip_lookup_i (wrk, b, bi, is_ip4, fib_index, 0);
591 if (wrk->vm->thread_index == 0 && vlib_num_workers ())
592 session_flush_frames_main_thread (wrk->vm);
Dave Barach68b0fb02017-02-28 15:15:56 -0500593}
594
Florin Coras1f152cd2017-08-18 19:28:03 -0700595always_inline void
Florin Corasbe72ae62018-11-01 11:23:03 -0700596tcp_enqueue_to_output_i (tcp_worker_ctx_t * wrk, vlib_buffer_t * b, u32 bi,
Florin Coras1f152cd2017-08-18 19:28:03 -0700597 u8 is_ip4, u8 flush)
598{
Florin Coras1f152cd2017-08-18 19:28:03 -0700599 u32 *to_next, next_index;
600 vlib_frame_t *f;
601
602 b->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
603 b->error = 0;
604
605 /* Decide where to send the packet */
606 next_index = is_ip4 ? tcp4_output_node.index : tcp6_output_node.index;
Florin Corasf988e692017-11-27 04:34:14 -0500607 tcp_trajectory_add_start (b, 2);
Florin Coras1f152cd2017-08-18 19:28:03 -0700608
609 /* Get frame to v4/6 output node */
Florin Corasbe72ae62018-11-01 11:23:03 -0700610 f = wrk->tx_frames[!is_ip4];
Florin Coras1f152cd2017-08-18 19:28:03 -0700611 if (!f)
612 {
Florin Corasbe72ae62018-11-01 11:23:03 -0700613 f = vlib_get_frame_to_node (wrk->vm, next_index);
Florin Coras1f152cd2017-08-18 19:28:03 -0700614 ASSERT (f);
Florin Corasbe72ae62018-11-01 11:23:03 -0700615 wrk->tx_frames[!is_ip4] = f;
Florin Coras1f152cd2017-08-18 19:28:03 -0700616 }
617 to_next = vlib_frame_vector_args (f);
618 to_next[f->n_vectors] = bi;
619 f->n_vectors += 1;
620 if (flush || f->n_vectors == VLIB_FRAME_SIZE)
621 {
Florin Corasbe72ae62018-11-01 11:23:03 -0700622 vlib_put_frame_to_node (wrk->vm, next_index, f);
623 wrk->tx_frames[!is_ip4] = 0;
Florin Coras1f152cd2017-08-18 19:28:03 -0700624 }
625}
626
Florin Coras0dbd5172018-06-25 16:19:34 -0700627static void
Florin Corasbe72ae62018-11-01 11:23:03 -0700628tcp_enqueue_to_output (tcp_worker_ctx_t * wrk, vlib_buffer_t * b, u32 bi,
629 u8 is_ip4)
Florin Coras1f152cd2017-08-18 19:28:03 -0700630{
Florin Corasbe72ae62018-11-01 11:23:03 -0700631 tcp_enqueue_to_output_i (wrk, b, bi, is_ip4, 0);
Florin Coras1f152cd2017-08-18 19:28:03 -0700632}
633
Florin Coras0dbd5172018-06-25 16:19:34 -0700634static void
Florin Corasbe72ae62018-11-01 11:23:03 -0700635tcp_enqueue_to_output_now (tcp_worker_ctx_t * wrk, vlib_buffer_t * b, u32 bi,
Florin Coras1f152cd2017-08-18 19:28:03 -0700636 u8 is_ip4)
637{
Florin Corasbe72ae62018-11-01 11:23:03 -0700638 tcp_enqueue_to_output_i (wrk, b, bi, is_ip4, 1);
Florin Coras1f152cd2017-08-18 19:28:03 -0700639}
Filip Tehlare275bed2019-03-06 00:06:56 -0800640#endif /* CLIB_MARCH_VARIANT */
Florin Coras1f152cd2017-08-18 19:28:03 -0700641
Florin Coras0dbd5172018-06-25 16:19:34 -0700642static int
Dave Barach68b0fb02017-02-28 15:15:56 -0500643tcp_make_reset_in_place (vlib_main_t * vm, vlib_buffer_t * b0,
Florin Corasdc629cd2017-05-09 00:52:37 -0700644 tcp_state_t state, u8 thread_index, u8 is_ip4)
Dave Barach68b0fb02017-02-28 15:15:56 -0500645{
Dave Barach68b0fb02017-02-28 15:15:56 -0500646 ip4_header_t *ih4;
647 ip6_header_t *ih6;
648 tcp_header_t *th0;
Florin Corasdc629cd2017-05-09 00:52:37 -0700649 ip4_address_t src_ip40, dst_ip40;
650 ip6_address_t src_ip60, dst_ip60;
651 u16 src_port, dst_port;
Dave Barach68b0fb02017-02-28 15:15:56 -0500652 u32 tmp;
Florin Corasdc629cd2017-05-09 00:52:37 -0700653 u32 seq, ack;
654 u8 flags;
Dave Barach68b0fb02017-02-28 15:15:56 -0500655
656 /* Find IP and TCP headers */
Florin Corasdc629cd2017-05-09 00:52:37 -0700657 th0 = tcp_buffer_hdr (b0);
658
659 /* Save src and dst ip */
Dave Barach68b0fb02017-02-28 15:15:56 -0500660 if (is_ip4)
661 {
662 ih4 = vlib_buffer_get_current (b0);
Florin Corasdc629cd2017-05-09 00:52:37 -0700663 ASSERT ((ih4->ip_version_and_header_length & 0xF0) == 0x40);
664 src_ip40.as_u32 = ih4->src_address.as_u32;
665 dst_ip40.as_u32 = ih4->dst_address.as_u32;
Dave Barach68b0fb02017-02-28 15:15:56 -0500666 }
667 else
668 {
669 ih6 = vlib_buffer_get_current (b0);
Dave Barach68b0fb02017-02-28 15:15:56 -0500670 ASSERT ((ih6->ip_version_traffic_class_and_flow_label & 0xF0) == 0x60);
Dave Barach178cf492018-11-13 16:34:13 -0500671 clib_memcpy_fast (&src_ip60, &ih6->src_address, sizeof (ip6_address_t));
672 clib_memcpy_fast (&dst_ip60, &ih6->dst_address, sizeof (ip6_address_t));
Dave Barach68b0fb02017-02-28 15:15:56 -0500673 }
674
Florin Corasdc629cd2017-05-09 00:52:37 -0700675 src_port = th0->src_port;
676 dst_port = th0->dst_port;
677
678 /* Try to determine what/why we're actually resetting */
Dave Barach68b0fb02017-02-28 15:15:56 -0500679 if (state == TCP_STATE_CLOSED)
680 {
681 if (!tcp_syn (th0))
682 return -1;
683
684 tmp = clib_net_to_host_u32 (th0->seq_number);
685
686 /* Got a SYN for no listener. */
Florin Corasdc629cd2017-05-09 00:52:37 -0700687 flags = TCP_FLAG_RST | TCP_FLAG_ACK;
688 ack = clib_host_to_net_u32 (tmp + 1);
689 seq = 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500690 }
Florin Corasdc629cd2017-05-09 00:52:37 -0700691 else
Dave Barach68b0fb02017-02-28 15:15:56 -0500692 {
Florin Corasdc629cd2017-05-09 00:52:37 -0700693 flags = TCP_FLAG_RST;
694 seq = th0->ack_number;
695 ack = 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500696 }
697
Florin Corasdc629cd2017-05-09 00:52:37 -0700698 tcp_reuse_buffer (vm, b0);
Florin Corasf988e692017-11-27 04:34:14 -0500699 tcp_trajectory_add_start (b0, 4);
Florin Corasdc629cd2017-05-09 00:52:37 -0700700 th0 = vlib_buffer_push_tcp_net_order (b0, dst_port, src_port, seq, ack,
701 sizeof (tcp_header_t), flags, 0);
Dave Barach68b0fb02017-02-28 15:15:56 -0500702
Dave Barach68b0fb02017-02-28 15:15:56 -0500703 if (is_ip4)
704 {
Florin Corasdc629cd2017-05-09 00:52:37 -0700705 ih4 = vlib_buffer_push_ip4 (vm, b0, &dst_ip40, &src_ip40,
Florin Corasfdbc3822017-07-27 00:34:12 -0700706 IP_PROTOCOL_TCP, 1);
Dave Barach68b0fb02017-02-28 15:15:56 -0500707 th0->checksum = ip4_tcp_udp_compute_checksum (vm, b0, ih4);
708 }
709 else
710 {
711 int bogus = ~0;
Florin Corasdc629cd2017-05-09 00:52:37 -0700712 ih6 = vlib_buffer_push_ip6 (vm, b0, &dst_ip60, &src_ip60,
713 IP_PROTOCOL_TCP);
Dave Barach68b0fb02017-02-28 15:15:56 -0500714 th0->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b0, ih6, &bogus);
715 ASSERT (!bogus);
716 }
717
718 return 0;
719}
720
Filip Tehlare275bed2019-03-06 00:06:56 -0800721#ifndef CLIB_MARCH_VARIANT
Dave Barach68b0fb02017-02-28 15:15:56 -0500722/**
723 * Send reset without reusing existing buffer
Florin Coras1f152cd2017-08-18 19:28:03 -0700724 *
725 * It extracts connection info out of original packet
Dave Barach68b0fb02017-02-28 15:15:56 -0500726 */
727void
Florin Corasd4c49be2019-02-07 00:15:53 -0800728tcp_send_reset_w_pkt (tcp_connection_t * tc, vlib_buffer_t * pkt,
729 u32 thread_index, u8 is_ip4)
Dave Barach68b0fb02017-02-28 15:15:56 -0500730{
Florin Corasd4c49be2019-02-07 00:15:53 -0800731 tcp_worker_ctx_t *wrk = tcp_get_worker (thread_index);
Florin Corasbe72ae62018-11-01 11:23:03 -0700732 vlib_main_t *vm = wrk->vm;
Dave Barach68b0fb02017-02-28 15:15:56 -0500733 vlib_buffer_t *b;
Florin Coras56b39f62018-03-27 17:29:32 -0700734 u32 bi, sw_if_index, fib_index;
Dave Barach68b0fb02017-02-28 15:15:56 -0500735 u8 tcp_hdr_len, flags = 0;
736 tcp_header_t *th, *pkt_th;
737 u32 seq, ack;
738 ip4_header_t *ih4, *pkt_ih4;
739 ip6_header_t *ih6, *pkt_ih6;
Florin Coras56b39f62018-03-27 17:29:32 -0700740 fib_protocol_t fib_proto;
Dave Barach68b0fb02017-02-28 15:15:56 -0500741
Florin Corasbdf7fd62019-01-31 17:31:01 -0800742 if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
Florin Coras66b11312017-07-31 17:18:03 -0700743 return;
744
Dave Barach68b0fb02017-02-28 15:15:56 -0500745 b = vlib_get_buffer (vm, bi);
Florin Coras56b39f62018-03-27 17:29:32 -0700746 sw_if_index = vnet_buffer (pkt)->sw_if_index[VLIB_RX];
747 fib_proto = is_ip4 ? FIB_PROTOCOL_IP4 : FIB_PROTOCOL_IP6;
748 fib_index = fib_table_get_index_for_sw_if_index (fib_proto, sw_if_index);
Florin Coras1f152cd2017-08-18 19:28:03 -0700749 tcp_init_buffer (vm, b);
Dave Barach68b0fb02017-02-28 15:15:56 -0500750
751 /* Make and write options */
752 tcp_hdr_len = sizeof (tcp_header_t);
753
754 if (is_ip4)
755 {
756 pkt_ih4 = vlib_buffer_get_current (pkt);
757 pkt_th = ip4_next_header (pkt_ih4);
758 }
759 else
760 {
761 pkt_ih6 = vlib_buffer_get_current (pkt);
762 pkt_th = ip6_next_header (pkt_ih6);
763 }
764
765 if (tcp_ack (pkt_th))
766 {
767 flags = TCP_FLAG_RST;
768 seq = pkt_th->ack_number;
Florin Coras776f3d82018-11-02 08:23:58 -0700769 ack = (tc->state >= TCP_STATE_SYN_RCVD) ? tc->rcv_nxt : 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500770 }
771 else
772 {
773 flags = TCP_FLAG_RST | TCP_FLAG_ACK;
774 seq = 0;
775 ack = clib_host_to_net_u32 (vnet_buffer (pkt)->tcp.seq_end);
776 }
777
778 th = vlib_buffer_push_tcp_net_order (b, pkt_th->dst_port, pkt_th->src_port,
779 seq, ack, tcp_hdr_len, flags, 0);
780
781 /* Swap src and dst ip */
782 if (is_ip4)
783 {
784 ASSERT ((pkt_ih4->ip_version_and_header_length & 0xF0) == 0x40);
785 ih4 = vlib_buffer_push_ip4 (vm, b, &pkt_ih4->dst_address,
Florin Corasfdbc3822017-07-27 00:34:12 -0700786 &pkt_ih4->src_address, IP_PROTOCOL_TCP, 1);
Dave Barach68b0fb02017-02-28 15:15:56 -0500787 th->checksum = ip4_tcp_udp_compute_checksum (vm, b, ih4);
788 }
789 else
790 {
791 int bogus = ~0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500792 ASSERT ((pkt_ih6->ip_version_traffic_class_and_flow_label & 0xF0) ==
793 0x60);
rootc9d1c5b2017-08-15 12:58:31 -0400794 ih6 = vlib_buffer_push_ip6 (vm, b, &pkt_ih6->dst_address,
795 &pkt_ih6->src_address, IP_PROTOCOL_TCP);
Dave Barach68b0fb02017-02-28 15:15:56 -0500796 th->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ih6, &bogus);
797 ASSERT (!bogus);
798 }
799
Florin Corasbe72ae62018-11-01 11:23:03 -0700800 tcp_enqueue_to_ip_lookup_now (wrk, b, bi, is_ip4, fib_index);
Florin Corasa436a422019-08-20 07:09:31 -0700801 TCP_EVT (TCP_EVT_RST_SENT, tc);
Florin Coras1f421012019-07-26 10:18:51 -0700802 vlib_node_increment_counter (vm, tcp_node_index (output, tc->c_is_ip4),
803 TCP_ERROR_RST_SENT, 1);
Dave Barach68b0fb02017-02-28 15:15:56 -0500804}
805
Florin Coras1f152cd2017-08-18 19:28:03 -0700806/**
807 * Build and set reset packet for connection
808 */
809void
810tcp_send_reset (tcp_connection_t * tc)
811{
Florin Corasbe72ae62018-11-01 11:23:03 -0700812 tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
813 vlib_main_t *vm = wrk->vm;
Florin Coras1f152cd2017-08-18 19:28:03 -0700814 vlib_buffer_t *b;
815 u32 bi;
816 tcp_header_t *th;
817 u16 tcp_hdr_opts_len, advertise_wnd, opts_write_len;
818 u8 flags;
819
Florin Corasbdf7fd62019-01-31 17:31:01 -0800820 if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
Florin Coras1f152cd2017-08-18 19:28:03 -0700821 return;
822 b = vlib_get_buffer (vm, bi);
823 tcp_init_buffer (vm, b);
824
825 tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts, tc->state);
826 tcp_hdr_opts_len = tc->snd_opts_len + sizeof (tcp_header_t);
827 advertise_wnd = tcp_window_to_advertise (tc, TCP_STATE_ESTABLISHED);
828 flags = TCP_FLAG_RST;
829 th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, tc->snd_nxt,
830 tc->rcv_nxt, tcp_hdr_opts_len, flags,
831 advertise_wnd);
832 opts_write_len = tcp_options_write ((u8 *) (th + 1), &tc->snd_opts);
833 ASSERT (opts_write_len == tc->snd_opts_len);
834 vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
Florin Corasdf36f492019-08-18 18:09:28 -0700835 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Florin Corasa436a422019-08-20 07:09:31 -0700836 TCP_EVT (TCP_EVT_RST_SENT, tc);
Florin Coras1f421012019-07-26 10:18:51 -0700837 vlib_node_increment_counter (vm, tcp_node_index (output, tc->c_is_ip4),
838 TCP_ERROR_RST_SENT, 1);
Florin Coras1f152cd2017-08-18 19:28:03 -0700839}
840
Florin Coras0dbd5172018-06-25 16:19:34 -0700841static void
Florin Corasbe72ae62018-11-01 11:23:03 -0700842tcp_push_ip_hdr (tcp_worker_ctx_t * wrk, tcp_connection_t * tc,
843 vlib_buffer_t * b)
Dave Barach68b0fb02017-02-28 15:15:56 -0500844{
845 tcp_header_t *th = vlib_buffer_get_current (b);
Florin Corasbe72ae62018-11-01 11:23:03 -0700846 vlib_main_t *vm = wrk->vm;
Dave Barach68b0fb02017-02-28 15:15:56 -0500847 if (tc->c_is_ip4)
848 {
849 ip4_header_t *ih;
Dave Barach2c25a622017-06-26 11:35:07 -0400850 ih = vlib_buffer_push_ip4 (vm, b, &tc->c_lcl_ip4,
Florin Corasfdbc3822017-07-27 00:34:12 -0700851 &tc->c_rmt_ip4, IP_PROTOCOL_TCP, 1);
Dave Barach2c25a622017-06-26 11:35:07 -0400852 th->checksum = ip4_tcp_udp_compute_checksum (vm, b, ih);
Dave Barach68b0fb02017-02-28 15:15:56 -0500853 }
854 else
855 {
856 ip6_header_t *ih;
857 int bogus = ~0;
858
Dave Barach2c25a622017-06-26 11:35:07 -0400859 ih = vlib_buffer_push_ip6 (vm, b, &tc->c_lcl_ip6,
Dave Barach68b0fb02017-02-28 15:15:56 -0500860 &tc->c_rmt_ip6, IP_PROTOCOL_TCP);
Dave Barach2c25a622017-06-26 11:35:07 -0400861 th->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ih, &bogus);
Dave Barach68b0fb02017-02-28 15:15:56 -0500862 ASSERT (!bogus);
863 }
864}
865
866/**
867 * Send SYN
868 *
869 * Builds a SYN packet for a half-open connection and sends it to ipx_lookup.
870 * The packet is not forwarded through tcpx_output to avoid doing lookups
871 * in the half_open pool.
872 */
873void
874tcp_send_syn (tcp_connection_t * tc)
875{
Florin Corasbe72ae62018-11-01 11:23:03 -0700876 tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
877 vlib_main_t *vm = wrk->vm;
Dave Barach68b0fb02017-02-28 15:15:56 -0500878 vlib_buffer_t *b;
879 u32 bi;
Dave Barach68b0fb02017-02-28 15:15:56 -0500880
Florin Corasf988e692017-11-27 04:34:14 -0500881 /*
882 * Setup retransmit and establish timers before requesting buffer
883 * such that we can return if we've ran out.
884 */
Florin Corasf988e692017-11-27 04:34:14 -0500885 tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN,
886 tc->rto * TCP_TO_TIMER_TICK);
887
Florin Corasbdf7fd62019-01-31 17:31:01 -0800888 if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
Florin Coras222e1f412019-02-16 20:47:32 -0800889 {
890 tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN, 1);
891 return;
892 }
Florin Coras66b11312017-07-31 17:18:03 -0700893
Dave Barach68b0fb02017-02-28 15:15:56 -0500894 b = vlib_get_buffer (vm, bi);
Florin Coras1f152cd2017-08-18 19:28:03 -0700895 tcp_init_buffer (vm, b);
Florin Coras4eeeaaf2017-09-05 14:03:37 -0400896 tcp_make_syn (tc, b);
Dave Barach68b0fb02017-02-28 15:15:56 -0500897
898 /* Measure RTT with this */
Florin Corasefefc6b2018-11-07 12:49:19 -0800899 tc->rtt_ts = tcp_time_now_us (vlib_num_workers ()? 1 : 0);
Dave Barach68b0fb02017-02-28 15:15:56 -0500900 tc->rtt_seq = tc->snd_nxt;
Dave Barach68b0fb02017-02-28 15:15:56 -0500901 tc->rto_boff = 0;
902
Florin Corasbe72ae62018-11-01 11:23:03 -0700903 tcp_push_ip_hdr (wrk, tc, b);
904 tcp_enqueue_to_ip_lookup (wrk, b, bi, tc->c_is_ip4, tc->c_fib_index);
Florin Corasa436a422019-08-20 07:09:31 -0700905 TCP_EVT (TCP_EVT_SYN_SENT, tc);
Dave Barach68b0fb02017-02-28 15:15:56 -0500906}
907
Florin Coras7ac053b2018-11-05 15:57:21 -0800908void
909tcp_send_synack (tcp_connection_t * tc)
910{
911 tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
912 vlib_main_t *vm = wrk->vm;
913 vlib_buffer_t *b;
914 u32 bi;
915
Florin Coras222e1f412019-02-16 20:47:32 -0800916 tcp_retransmit_timer_force_update (tc);
917
Florin Corasbdf7fd62019-01-31 17:31:01 -0800918 if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
Florin Coras222e1f412019-02-16 20:47:32 -0800919 {
920 tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1);
921 return;
922 }
Florin Coras7ac053b2018-11-05 15:57:21 -0800923
Florin Corasdf084782018-12-06 17:41:10 -0800924 tc->rtt_ts = tcp_time_now_us (tc->c_thread_index);
Florin Coras7ac053b2018-11-05 15:57:21 -0800925 b = vlib_get_buffer (vm, bi);
Florin Corasbdf7fd62019-01-31 17:31:01 -0800926 tcp_init_buffer (vm, b);
Florin Coras7ac053b2018-11-05 15:57:21 -0800927 tcp_make_synack (tc, b);
928 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Florin Corasa436a422019-08-20 07:09:31 -0700929 TCP_EVT (TCP_EVT_SYNACK_SENT, tc);
Florin Coras7ac053b2018-11-05 15:57:21 -0800930}
931
Florin Coras66b11312017-07-31 17:18:03 -0700932/**
933 * Flush tx frame populated by retransmits and timer pops
934 */
935void
Florin Corasbe72ae62018-11-01 11:23:03 -0700936tcp_flush_frame_to_output (tcp_worker_ctx_t * wrk, u8 is_ip4)
Florin Coras66b11312017-07-31 17:18:03 -0700937{
Florin Corasbe72ae62018-11-01 11:23:03 -0700938 if (wrk->tx_frames[!is_ip4])
Florin Coras66b11312017-07-31 17:18:03 -0700939 {
940 u32 next_index;
941 next_index = is_ip4 ? tcp4_output_node.index : tcp6_output_node.index;
Florin Corasbe72ae62018-11-01 11:23:03 -0700942 vlib_put_frame_to_node (wrk->vm, next_index, wrk->tx_frames[!is_ip4]);
943 wrk->tx_frames[!is_ip4] = 0;
Florin Coras66b11312017-07-31 17:18:03 -0700944 }
945}
946
947/**
Florin Coras9d063042017-09-14 03:08:00 -0400948 * Flush ip lookup tx frames populated by timer pops
949 */
Florin Coras0dbd5172018-06-25 16:19:34 -0700950static void
Florin Corasbe72ae62018-11-01 11:23:03 -0700951tcp_flush_frame_to_ip_lookup (tcp_worker_ctx_t * wrk, u8 is_ip4)
Florin Coras9d063042017-09-14 03:08:00 -0400952{
Florin Corasbe72ae62018-11-01 11:23:03 -0700953 if (wrk->ip_lookup_tx_frames[!is_ip4])
Florin Coras9d063042017-09-14 03:08:00 -0400954 {
955 u32 next_index;
956 next_index = is_ip4 ? ip4_lookup_node.index : ip6_lookup_node.index;
Florin Corasbe72ae62018-11-01 11:23:03 -0700957 vlib_put_frame_to_node (wrk->vm, next_index,
958 wrk->ip_lookup_tx_frames[!is_ip4]);
959 wrk->ip_lookup_tx_frames[!is_ip4] = 0;
Florin Coras9d063042017-09-14 03:08:00 -0400960 }
961}
962
963/**
964 * Flush v4 and v6 tcp and ip-lookup tx frames for thread index
Florin Coras66b11312017-07-31 17:18:03 -0700965 */
966void
Florin Corasbe72ae62018-11-01 11:23:03 -0700967tcp_flush_frames_to_output (tcp_worker_ctx_t * wrk)
Florin Coras66b11312017-07-31 17:18:03 -0700968{
Florin Corasbe72ae62018-11-01 11:23:03 -0700969 tcp_flush_frame_to_output (wrk, 1);
970 tcp_flush_frame_to_output (wrk, 0);
971 tcp_flush_frame_to_ip_lookup (wrk, 1);
972 tcp_flush_frame_to_ip_lookup (wrk, 0);
Dave Barach68b0fb02017-02-28 15:15:56 -0500973}
974
975/**
976 * Send FIN
977 */
978void
979tcp_send_fin (tcp_connection_t * tc)
980{
Florin Corasbe72ae62018-11-01 11:23:03 -0700981 tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
982 vlib_main_t *vm = wrk->vm;
Florin Coras9d063042017-09-14 03:08:00 -0400983 vlib_buffer_t *b;
984 u32 bi;
985 u8 fin_snt = 0;
986
Florin Corasc977e7c2018-10-16 20:30:31 -0700987 fin_snt = tc->flags & TCP_CONN_FINSNT;
988 if (fin_snt)
Florin Coras47596832019-03-12 18:58:54 -0700989 tc->snd_nxt -= 1;
Florin Corasc977e7c2018-10-16 20:30:31 -0700990
Florin Corasbdf7fd62019-01-31 17:31:01 -0800991 if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
Florin Coraseb97e5f2018-10-15 21:35:42 -0700992 {
993 /* Out of buffers so program fin retransmit ASAP */
994 tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1);
Florin Coras85a3ddd2018-12-24 16:54:34 -0800995 if (fin_snt)
Florin Coras47596832019-03-12 18:58:54 -0700996 tc->snd_nxt += 1;
Florin Coras222e1f412019-02-16 20:47:32 -0800997 else
998 /* Make sure retransmit retries a fin not data */
999 tc->flags |= TCP_CONN_FINSNT;
Florin Coras85a3ddd2018-12-24 16:54:34 -08001000 return;
Florin Coraseb97e5f2018-10-15 21:35:42 -07001001 }
1002
Florin Corasc977e7c2018-10-16 20:30:31 -07001003 tcp_retransmit_timer_force_update (tc);
Dave Barach68b0fb02017-02-28 15:15:56 -05001004 b = vlib_get_buffer (vm, bi);
Florin Coras24793e32018-05-09 11:34:25 -07001005 tcp_init_buffer (vm, b);
Florin Corasd79b41e2017-03-04 05:37:52 -08001006 tcp_make_fin (tc, b);
Florin Corasbe72ae62018-11-01 11:23:03 -07001007 tcp_enqueue_to_output_now (wrk, b, bi, tc->c_is_ip4);
Florin Corasa436a422019-08-20 07:09:31 -07001008 TCP_EVT (TCP_EVT_FIN_SENT, tc);
Florin Coras47596832019-03-12 18:58:54 -07001009 /* Account for the FIN */
1010 tc->snd_nxt += 1;
Florin Coras9d063042017-09-14 03:08:00 -04001011 if (!fin_snt)
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001012 {
1013 tc->flags |= TCP_CONN_FINSNT;
1014 tc->flags &= ~TCP_CONN_FINPNDG;
Florin Coras47596832019-03-12 18:58:54 -07001015 tc->snd_una_max = seq_max (tc->snd_una_max, tc->snd_nxt);
Florin Corasa096f2d2017-09-28 23:49:42 -04001016 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001017}
1018
Dave Barach68b0fb02017-02-28 15:15:56 -05001019/**
Florin Corasfd247442019-03-12 16:56:26 -07001020 * Push TCP header and update connection variables. Should only be called
1021 * for segments with data, not for 'control' packets.
Dave Barach68b0fb02017-02-28 15:15:56 -05001022 */
Florin Coras0dbd5172018-06-25 16:19:34 -07001023always_inline void
Florin Coras47596832019-03-12 18:58:54 -07001024tcp_push_hdr_i (tcp_connection_t * tc, vlib_buffer_t * b, u32 snd_nxt,
1025 u8 compute_opts, u8 maybe_burst, u8 update_snd_nxt)
Dave Barach68b0fb02017-02-28 15:15:56 -05001026{
Florin Corasfd247442019-03-12 16:56:26 -07001027 u8 tcp_hdr_opts_len, flags = TCP_FLAG_ACK;
Dave Barach68b0fb02017-02-28 15:15:56 -05001028 u32 advertise_wnd, data_len;
Florin Corasb26743d2018-06-26 09:31:04 -07001029 tcp_main_t *tm = &tcp_main;
Dave Barach68b0fb02017-02-28 15:15:56 -05001030 tcp_header_t *th;
1031
Florin Corasb26743d2018-06-26 09:31:04 -07001032 data_len = b->current_length;
1033 if (PREDICT_FALSE (b->flags & VLIB_BUFFER_NEXT_PRESENT))
1034 data_len += b->total_length_not_including_first_buffer;
1035
Dave Barach68b0fb02017-02-28 15:15:56 -05001036 vnet_buffer (b)->tcp.flags = 0;
Florin Corasb26743d2018-06-26 09:31:04 -07001037 vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
Dave Barach68b0fb02017-02-28 15:15:56 -05001038
Florin Corasc8343412017-05-04 14:25:50 -07001039 if (compute_opts)
1040 tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts, tc->state);
1041
Florin Corasc8343412017-05-04 14:25:50 -07001042 tcp_hdr_opts_len = tc->snd_opts_len + sizeof (tcp_header_t);
Florin Corasb26743d2018-06-26 09:31:04 -07001043
1044 if (maybe_burst)
1045 advertise_wnd = tc->rcv_wnd >> tc->rcv_wscale;
1046 else
Florin Coras47596832019-03-12 18:58:54 -07001047 advertise_wnd = tcp_window_to_advertise (tc, TCP_STATE_ESTABLISHED);
Florin Corasb26743d2018-06-26 09:31:04 -07001048
Florin Coras42ceddb2018-12-12 10:56:01 -08001049 if (PREDICT_FALSE (tc->flags & TCP_CONN_PSH_PENDING))
1050 {
Florin Coras47596832019-03-12 18:58:54 -07001051 if (seq_geq (tc->psh_seq, snd_nxt)
1052 && seq_lt (tc->psh_seq, snd_nxt + data_len))
Florin Coras42ceddb2018-12-12 10:56:01 -08001053 flags |= TCP_FLAG_PSH;
1054 }
Florin Coras47596832019-03-12 18:58:54 -07001055 th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, snd_nxt,
Dave Barach68b0fb02017-02-28 15:15:56 -05001056 tc->rcv_nxt, tcp_hdr_opts_len, flags,
1057 advertise_wnd);
Dave Barach68b0fb02017-02-28 15:15:56 -05001058
Florin Corasb26743d2018-06-26 09:31:04 -07001059 if (maybe_burst)
1060 {
Dave Barach178cf492018-11-13 16:34:13 -05001061 clib_memcpy_fast ((u8 *) (th + 1),
1062 tm->wrk_ctx[tc->c_thread_index].cached_opts,
1063 tc->snd_opts_len);
Florin Corasb26743d2018-06-26 09:31:04 -07001064 }
1065 else
1066 {
1067 u8 len = tcp_options_write ((u8 *) (th + 1), &tc->snd_opts);
1068 ASSERT (len == tc->snd_opts_len);
1069 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001070
Florin Coras93992a92017-05-24 18:03:56 -07001071 /*
1072 * Update connection variables
1073 */
1074
Florin Coras47596832019-03-12 18:58:54 -07001075 if (update_snd_nxt)
1076 tc->snd_nxt += data_len;
Florin Corasc28764f2017-04-26 00:08:42 -07001077 tc->rcv_las = tc->rcv_nxt;
Florin Coras3e350af2017-03-30 02:54:28 -07001078
Florin Corasedfe0ee2019-07-29 18:13:25 -07001079 tc->bytes_out += data_len;
1080 tc->data_segs_out += 1;
1081
Florin Corasa436a422019-08-20 07:09:31 -07001082 TCP_EVT (TCP_EVT_PKTIZE, tc);
Dave Barach68b0fb02017-02-28 15:15:56 -05001083}
1084
Florin Coras0dbd5172018-06-25 16:19:34 -07001085u32
Florin Coras14ed6df2019-03-06 21:13:42 -08001086tcp_session_push_header (transport_connection_t * tconn, vlib_buffer_t * b)
Florin Coras0dbd5172018-06-25 16:19:34 -07001087{
Florin Coras14ed6df2019-03-06 21:13:42 -08001088 tcp_connection_t *tc = (tcp_connection_t *) tconn;
Florin Coras52814732019-06-12 15:38:19 -07001089
1090 if (tc->flags & TCP_CONN_TRACK_BURST)
1091 {
1092 tcp_bt_check_app_limited (tc);
1093 tcp_bt_track_tx (tc);
1094 tc->flags &= ~TCP_CONN_TRACK_BURST;
1095 }
1096
Florin Coras47596832019-03-12 18:58:54 -07001097 tcp_push_hdr_i (tc, b, tc->snd_nxt, /* compute opts */ 0, /* burst */ 1,
1098 /* update_snd_nxt */ 1);
Florin Coras52814732019-06-12 15:38:19 -07001099
Florin Coras47596832019-03-12 18:58:54 -07001100 tc->snd_una_max = seq_max (tc->snd_nxt, tc->snd_una_max);
Florin Coras0dbd5172018-06-25 16:19:34 -07001101 tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una);
1102 /* If not tracking an ACK, start tracking */
1103 if (tc->rtt_ts == 0 && !tcp_in_cong_recovery (tc))
1104 {
Florin Corasd67f1122018-05-21 17:47:40 -07001105 tc->rtt_ts = tcp_time_now_us (tc->c_thread_index);
Florin Coras0dbd5172018-06-25 16:19:34 -07001106 tc->rtt_seq = tc->snd_nxt;
1107 }
1108 if (PREDICT_FALSE (!tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT)))
1109 {
1110 tcp_retransmit_timer_set (tc);
1111 tc->rto_boff = 0;
1112 }
1113 tcp_trajectory_add_start (b, 3);
1114 return 0;
1115}
1116
Dave Barach68b0fb02017-02-28 15:15:56 -05001117void
Florin Coras6792ec02017-03-13 03:49:51 -07001118tcp_send_ack (tcp_connection_t * tc)
Dave Barach68b0fb02017-02-28 15:15:56 -05001119{
Florin Corasbe72ae62018-11-01 11:23:03 -07001120 tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
1121 vlib_main_t *vm = wrk->vm;
Dave Barach68b0fb02017-02-28 15:15:56 -05001122 vlib_buffer_t *b;
1123 u32 bi;
1124
Florin Corasbdf7fd62019-01-31 17:31:01 -08001125 if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
Florin Coras222e1f412019-02-16 20:47:32 -08001126 {
1127 tcp_update_rcv_wnd (tc);
1128 return;
1129 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001130 b = vlib_get_buffer (vm, bi);
Florin Coras24793e32018-05-09 11:34:25 -07001131 tcp_init_buffer (vm, b);
Dave Barach68b0fb02017-02-28 15:15:56 -05001132 tcp_make_ack (tc, b);
Florin Corasbe72ae62018-11-01 11:23:03 -07001133 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Dave Barach68b0fb02017-02-28 15:15:56 -05001134}
1135
Florin Coras7ac053b2018-11-05 15:57:21 -08001136void
Florin Coras26dd6de2019-07-23 23:54:47 -07001137tcp_program_ack (tcp_connection_t * tc)
Florin Coras7ac053b2018-11-05 15:57:21 -08001138{
1139 if (!(tc->flags & TCP_CONN_SNDACK))
1140 {
Florin Coras26dd6de2019-07-23 23:54:47 -07001141 session_add_self_custom_tx_evt (&tc->connection, 1);
Florin Coras7ac053b2018-11-05 15:57:21 -08001142 tc->flags |= TCP_CONN_SNDACK;
1143 }
1144}
1145
1146void
Florin Coras26dd6de2019-07-23 23:54:47 -07001147tcp_program_dupack (tcp_connection_t * tc)
Florin Coras7ac053b2018-11-05 15:57:21 -08001148{
1149 if (!(tc->flags & TCP_CONN_SNDACK))
1150 {
Florin Coras26dd6de2019-07-23 23:54:47 -07001151 session_add_self_custom_tx_evt (&tc->connection, 1);
Florin Coras7ac053b2018-11-05 15:57:21 -08001152 tc->flags |= TCP_CONN_SNDACK;
1153 }
1154 if (tc->pending_dupacks < 255)
1155 tc->pending_dupacks += 1;
1156}
1157
1158void
Florin Coras26dd6de2019-07-23 23:54:47 -07001159tcp_program_fastretransmit (tcp_connection_t * tc)
Florin Coras7ac053b2018-11-05 15:57:21 -08001160{
Florin Coras26dd6de2019-07-23 23:54:47 -07001161 if (!(tc->flags & TCP_CONN_FRXT_PENDING))
Florin Coras7ac053b2018-11-05 15:57:21 -08001162 {
Florin Coras26dd6de2019-07-23 23:54:47 -07001163 session_add_self_custom_tx_evt (&tc->connection, 0);
1164 tc->flags |= TCP_CONN_FRXT_PENDING;
Florin Coras7ac053b2018-11-05 15:57:21 -08001165 }
Florin Coras7ac053b2018-11-05 15:57:21 -08001166}
1167
Florin Corasb2215d62017-08-01 16:56:58 -07001168/**
1169 * Delayed ack timer handler
1170 *
1171 * Sends delayed ACK when timer expires
1172 */
Florin Coras6792ec02017-03-13 03:49:51 -07001173void
1174tcp_timer_delack_handler (u32 index)
1175{
Damjan Marion586afd72017-04-05 19:18:20 +02001176 u32 thread_index = vlib_get_thread_index ();
Florin Coras6792ec02017-03-13 03:49:51 -07001177 tcp_connection_t *tc;
1178
1179 tc = tcp_connection_get (index, thread_index);
1180 tc->timers[TCP_TIMER_DELACK] = TCP_TIMER_HANDLE_INVALID;
Florin Coras6792ec02017-03-13 03:49:51 -07001181 tcp_send_ack (tc);
1182}
1183
Florin Corasb2215d62017-08-01 16:56:58 -07001184/**
Vladimir Kropylev398afbd2019-06-25 00:06:52 +03001185 * Send Window Update ACK,
1186 * ensuring that it will be sent once, if RWND became non-zero,
1187 * after zero RWND has been advertised in ACK before
1188 */
1189void
1190tcp_send_window_update_ack (tcp_connection_t * tc)
1191{
Vladimir Kropylev398afbd2019-06-25 00:06:52 +03001192 u32 win;
1193
1194 if (tcp_zero_rwnd_sent (tc))
1195 {
1196 win = tcp_window_to_advertise (tc, tc->state);
1197 if (win > 0)
1198 {
1199 tcp_zero_rwnd_sent_off (tc);
Florin Coras26dd6de2019-07-23 23:54:47 -07001200 tcp_program_ack (tc);
Vladimir Kropylev398afbd2019-06-25 00:06:52 +03001201 }
1202 }
1203}
1204
1205/**
Florin Coras36ee9f12018-11-02 12:52:10 -07001206 * Allocate a new buffer and build a new tcp segment
Dave Barach68b0fb02017-02-28 15:15:56 -05001207 *
Florin Coras36ee9f12018-11-02 12:52:10 -07001208 * @param wrk tcp worker
1209 * @param tc connection for which the segment will be allocated
1210 * @param offset offset of the first byte in the tx fifo
1211 * @param max_deq_byte segment size
1212 * @param[out] b pointer to buffer allocated
1213 *
1214 * @return the number of bytes in the segment or 0 if buffer cannot be
1215 * allocated or no data available
Florin Coras93992a92017-05-24 18:03:56 -07001216 */
Florin Coras36ee9f12018-11-02 12:52:10 -07001217static int
1218tcp_prepare_segment (tcp_worker_ctx_t * wrk, tcp_connection_t * tc,
1219 u32 offset, u32 max_deq_bytes, vlib_buffer_t ** b)
Dave Barach68b0fb02017-02-28 15:15:56 -05001220{
Florin Corasbe72ae62018-11-01 11:23:03 -07001221 u32 bytes_per_buffer = vnet_get_tcp_main ()->bytes_per_buffer;
1222 vlib_main_t *vm = wrk->vm;
Florin Corasbdf7fd62019-01-31 17:31:01 -08001223 u32 bi, seg_size;
Florin Coras93992a92017-05-24 18:03:56 -07001224 int n_bytes = 0;
Florin Corase87216f2017-08-17 16:59:22 -07001225 u8 *data;
Dave Barach68b0fb02017-02-28 15:15:56 -05001226
Florin Coras1ee78302019-02-05 15:51:15 -08001227 seg_size = max_deq_bytes + TRANSPORT_MAX_HDRS_LEN;
Florin Coras1f152cd2017-08-18 19:28:03 -07001228
Florin Corasb2215d62017-08-01 16:56:58 -07001229 /*
1230 * Prepare options
1231 */
Florin Corasc8343412017-05-04 14:25:50 -07001232 tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts, tc->state);
1233
Florin Corasb2215d62017-08-01 16:56:58 -07001234 /*
1235 * Allocate and fill in buffer(s)
1236 */
Dave Barach68b0fb02017-02-28 15:15:56 -05001237
Florin Corasb2215d62017-08-01 16:56:58 -07001238 /* Easy case, buffer size greater than mss */
Florin Corasbe72ae62018-11-01 11:23:03 -07001239 if (PREDICT_TRUE (seg_size <= bytes_per_buffer))
Florin Corasb2215d62017-08-01 16:56:58 -07001240 {
Florin Corasbdf7fd62019-01-31 17:31:01 -08001241 if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
Florin Coras24793e32018-05-09 11:34:25 -07001242 return 0;
1243 *b = vlib_get_buffer (vm, bi);
1244 data = tcp_init_buffer (vm, *b);
Florin Coras31c99552019-03-01 13:00:58 -08001245 n_bytes = session_tx_fifo_peek_bytes (&tc->connection, data, offset,
1246 max_deq_bytes);
Florin Corasb2215d62017-08-01 16:56:58 -07001247 ASSERT (n_bytes == max_deq_bytes);
1248 b[0]->current_length = n_bytes;
Florin Coras47596832019-03-12 18:58:54 -07001249 tcp_push_hdr_i (tc, *b, tc->snd_una + offset, /* compute opts */ 0,
1250 /* burst */ 0, /* update_snd_nxt */ 0);
Florin Corasb2215d62017-08-01 16:56:58 -07001251 }
1252 /* Split mss into multiple buffers */
1253 else
1254 {
Florin Corasbdf7fd62019-01-31 17:31:01 -08001255 u32 chain_bi = ~0, n_bufs_per_seg, n_bufs;
1256 u16 n_peeked, len_to_deq;
Florin Corasb2215d62017-08-01 16:56:58 -07001257 vlib_buffer_t *chain_b, *prev_b;
Florin Corasb2215d62017-08-01 16:56:58 -07001258 int i;
1259
Florin Corasb2215d62017-08-01 16:56:58 -07001260 /* Make sure we have enough buffers */
Florin Corasbe72ae62018-11-01 11:23:03 -07001261 n_bufs_per_seg = ceil ((double) seg_size / bytes_per_buffer);
Florin Corasbdf7fd62019-01-31 17:31:01 -08001262 vec_validate_aligned (wrk->tx_buffers, n_bufs_per_seg - 1,
1263 CLIB_CACHE_LINE_BYTES);
1264 n_bufs = vlib_buffer_alloc (vm, wrk->tx_buffers, n_bufs_per_seg);
1265 if (PREDICT_FALSE (n_bufs != n_bufs_per_seg))
Florin Corasb2215d62017-08-01 16:56:58 -07001266 {
Florin Corasbdf7fd62019-01-31 17:31:01 -08001267 if (n_bufs)
1268 vlib_buffer_free (vm, wrk->tx_buffers, n_bufs);
1269 return 0;
Florin Corasb2215d62017-08-01 16:56:58 -07001270 }
1271
Florin Corasbdf7fd62019-01-31 17:31:01 -08001272 *b = vlib_get_buffer (vm, wrk->tx_buffers[--n_bufs]);
Florin Coras24793e32018-05-09 11:34:25 -07001273 data = tcp_init_buffer (vm, *b);
Florin Coras31c99552019-03-01 13:00:58 -08001274 n_bytes = session_tx_fifo_peek_bytes (&tc->connection, data, offset,
1275 bytes_per_buffer -
1276 TRANSPORT_MAX_HDRS_LEN);
Florin Corasb2215d62017-08-01 16:56:58 -07001277 b[0]->current_length = n_bytes;
1278 b[0]->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
1279 b[0]->total_length_not_including_first_buffer = 0;
Florin Corasb2215d62017-08-01 16:56:58 -07001280 max_deq_bytes -= n_bytes;
1281
1282 chain_b = *b;
1283 for (i = 1; i < n_bufs_per_seg; i++)
1284 {
1285 prev_b = chain_b;
Florin Corasbe72ae62018-11-01 11:23:03 -07001286 len_to_deq = clib_min (max_deq_bytes, bytes_per_buffer);
Florin Corasbdf7fd62019-01-31 17:31:01 -08001287 chain_bi = wrk->tx_buffers[--n_bufs];
Florin Corasb2215d62017-08-01 16:56:58 -07001288 chain_b = vlib_get_buffer (vm, chain_bi);
1289 chain_b->current_data = 0;
Florin Corase87216f2017-08-17 16:59:22 -07001290 data = vlib_buffer_get_current (chain_b);
Florin Coras31c99552019-03-01 13:00:58 -08001291 n_peeked = session_tx_fifo_peek_bytes (&tc->connection, data,
1292 offset + n_bytes,
1293 len_to_deq);
Florin Corasb2215d62017-08-01 16:56:58 -07001294 ASSERT (n_peeked == len_to_deq);
Florin Coras1f152cd2017-08-18 19:28:03 -07001295 n_bytes += n_peeked;
Florin Corasb2215d62017-08-01 16:56:58 -07001296 chain_b->current_length = n_peeked;
Florin Coras1f152cd2017-08-18 19:28:03 -07001297 chain_b->next_buffer = 0;
Florin Corasb2215d62017-08-01 16:56:58 -07001298
1299 /* update previous buffer */
1300 prev_b->next_buffer = chain_bi;
1301 prev_b->flags |= VLIB_BUFFER_NEXT_PRESENT;
1302
Florin Corasb2215d62017-08-01 16:56:58 -07001303 max_deq_bytes -= n_peeked;
Florin Coras1f152cd2017-08-18 19:28:03 -07001304 b[0]->total_length_not_including_first_buffer += n_peeked;
Florin Corasb2215d62017-08-01 16:56:58 -07001305 }
Florin Coras1f152cd2017-08-18 19:28:03 -07001306
Florin Coras47596832019-03-12 18:58:54 -07001307 tcp_push_hdr_i (tc, *b, tc->snd_una + offset, /* compute opts */ 0,
1308 /* burst */ 0, /* update_snd_nxt */ 0);
Florin Corasbdf7fd62019-01-31 17:31:01 -08001309
1310 if (PREDICT_FALSE (n_bufs))
1311 {
1312 clib_warning ("not all buffers consumed");
1313 vlib_buffer_free (vm, wrk->tx_buffers, n_bufs);
1314 }
Florin Corasb2215d62017-08-01 16:56:58 -07001315 }
1316
Florin Coras93992a92017-05-24 18:03:56 -07001317 ASSERT (n_bytes > 0);
Florin Corasbe72ae62018-11-01 11:23:03 -07001318 ASSERT (((*b)->current_data + (*b)->current_length) <= bytes_per_buffer);
Florin Corasbb292f42017-05-19 09:49:19 -07001319
Florin Coras36ee9f12018-11-02 12:52:10 -07001320 return n_bytes;
1321}
1322
1323/**
1324 * Build a retransmit segment
1325 *
1326 * @return the number of bytes in the segment or 0 if there's nothing to
1327 * retransmit
1328 */
1329static u32
1330tcp_prepare_retransmit_segment (tcp_worker_ctx_t * wrk,
1331 tcp_connection_t * tc, u32 offset,
1332 u32 max_deq_bytes, vlib_buffer_t ** b)
1333{
1334 u32 start, available_bytes;
1335 int n_bytes = 0;
1336
1337 ASSERT (tc->state >= TCP_STATE_ESTABLISHED);
1338 ASSERT (max_deq_bytes != 0);
1339
1340 /*
1341 * Make sure we can retransmit something
1342 */
Florin Coras31c99552019-03-01 13:00:58 -08001343 available_bytes = transport_max_tx_dequeue (&tc->connection);
Florin Coras36ee9f12018-11-02 12:52:10 -07001344 ASSERT (available_bytes >= offset);
1345 available_bytes -= offset;
1346 if (!available_bytes)
1347 return 0;
1348
1349 max_deq_bytes = clib_min (tc->snd_mss, max_deq_bytes);
1350 max_deq_bytes = clib_min (available_bytes, max_deq_bytes);
1351
1352 /* Start is beyond snd_congestion */
1353 start = tc->snd_una + offset;
1354 if (seq_geq (start, tc->snd_congestion))
Florin Corasedfe0ee2019-07-29 18:13:25 -07001355 return 0;
Florin Coras36ee9f12018-11-02 12:52:10 -07001356
1357 /* Don't overshoot snd_congestion */
1358 if (seq_gt (start + max_deq_bytes, tc->snd_congestion))
1359 {
1360 max_deq_bytes = tc->snd_congestion - start;
1361 if (max_deq_bytes == 0)
Florin Corasedfe0ee2019-07-29 18:13:25 -07001362 return 0;
Florin Coras36ee9f12018-11-02 12:52:10 -07001363 }
1364
1365 n_bytes = tcp_prepare_segment (wrk, tc, offset, max_deq_bytes, b);
1366 if (!n_bytes)
1367 return 0;
1368
Florin Coras93992a92017-05-24 18:03:56 -07001369 if (tcp_in_fastrecovery (tc))
Florin Coras52814732019-06-12 15:38:19 -07001370 {
1371 tc->snd_rxt_bytes += n_bytes;
1372 if (tc->flags & TCP_CONN_RATE_SAMPLE)
1373 tcp_bt_track_rxt (tc, start, start + n_bytes);
1374 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001375
Florin Corasedfe0ee2019-07-29 18:13:25 -07001376 tc->bytes_retrans += n_bytes;
1377 tc->segs_retrans += 1;
Florin Corasa436a422019-08-20 07:09:31 -07001378 TCP_EVT (TCP_EVT_CC_RTX, tc, offset, n_bytes);
Dave Barach68b0fb02017-02-28 15:15:56 -05001379 return n_bytes;
1380}
1381
Florin Coras6792ec02017-03-13 03:49:51 -07001382/**
1383 * Reset congestion control, switch cwnd to loss window and try again.
1384 */
1385static void
Florin Corasa3c32652019-07-03 17:47:22 -07001386tcp_cc_init_rxt_timeout (tcp_connection_t * tc)
Florin Coras6792ec02017-03-13 03:49:51 -07001387{
Florin Corasa436a422019-08-20 07:09:31 -07001388 TCP_EVT (TCP_EVT_CC_EVT, tc, 6);
Florin Coras93992a92017-05-24 18:03:56 -07001389 tc->prev_ssthresh = tc->ssthresh;
1390 tc->prev_cwnd = tc->cwnd;
1391
Florin Corasa3c32652019-07-03 17:47:22 -07001392 /* Clear fast recovery state if needed */
Florin Coras6792ec02017-03-13 03:49:51 -07001393 if (tcp_in_fastrecovery (tc))
Florin Corasa3c32652019-07-03 17:47:22 -07001394 tcp_cc_fastrecovery_clear (tc);
1395
1396 /* Let cc algo decide loss cwnd and ssthresh */
1397 tcp_cc_loss (tc);
Florin Coras6792ec02017-03-13 03:49:51 -07001398
1399 /* Start again from the beginning */
Florin Coras47596832019-03-12 18:58:54 -07001400 tc->snd_congestion = tc->snd_nxt;
Florin Corasa3c32652019-07-03 17:47:22 -07001401 tc->rcv_dupacks = 0;
Florin Corasf1762d62017-09-24 19:43:08 -04001402 tc->rtt_ts = 0;
Florin Corasd2aab832018-05-22 11:39:59 -07001403 tc->cwnd_acc_bytes = 0;
Florin Corasedfe0ee2019-07-29 18:13:25 -07001404 tc->tr_occurences += 1;
Florin Corasc44a5582018-11-01 16:30:54 -07001405 tcp_connection_tx_pacer_reset (tc, tc->cwnd, 2 * tc->snd_mss);
Florin Coras3af90fc2017-05-03 21:09:42 -07001406 tcp_recovery_on (tc);
Florin Coras6792ec02017-03-13 03:49:51 -07001407}
1408
Florin Coras75c48c12019-08-02 15:17:21 -07001409void
1410tcp_timer_retransmit_handler (u32 tc_index)
Dave Barach68b0fb02017-02-28 15:15:56 -05001411{
Damjan Marion586afd72017-04-05 19:18:20 +02001412 u32 thread_index = vlib_get_thread_index ();
Florin Corasbe72ae62018-11-01 11:23:03 -07001413 tcp_worker_ctx_t *wrk = tcp_get_worker (thread_index);
1414 vlib_main_t *vm = wrk->vm;
Dave Barach68b0fb02017-02-28 15:15:56 -05001415 tcp_connection_t *tc;
Florin Corasb2215d62017-08-01 16:56:58 -07001416 vlib_buffer_t *b = 0;
Florin Coras3af90fc2017-05-03 21:09:42 -07001417 u32 bi, n_bytes;
Dave Barach68b0fb02017-02-28 15:15:56 -05001418
Florin Coras75c48c12019-08-02 15:17:21 -07001419 tc = tcp_connection_get (tc_index, thread_index);
1420
1421 /* Note: the connection may have been closed and pool_put */
1422 if (PREDICT_FALSE (tc == 0 || tc->state == TCP_STATE_SYN_SENT))
1423 return;
1424
1425 tc->timers[TCP_TIMER_RETRANSMIT] = TCP_TIMER_HANDLE_INVALID;
1426
1427 /* Wait-close and retransmit could pop at the same time */
1428 if (tc->state == TCP_STATE_CLOSED)
1429 return;
Dave Barach68b0fb02017-02-28 15:15:56 -05001430
Florin Corase04c2992017-03-01 08:17:34 -08001431 if (tc->state >= TCP_STATE_ESTABLISHED)
Dave Barach68b0fb02017-02-28 15:15:56 -05001432 {
Florin Corasa436a422019-08-20 07:09:31 -07001433 TCP_EVT (TCP_EVT_CC_EVT, tc, 2);
Florin Corase55a6d72018-10-31 23:09:22 -07001434
Florin Coras93992a92017-05-24 18:03:56 -07001435 /* Lost FIN, retransmit and return */
Florin Coras222e1f412019-02-16 20:47:32 -08001436 if (tc->flags & TCP_CONN_FINSNT)
Florin Coras93992a92017-05-24 18:03:56 -07001437 {
1438 tcp_send_fin (tc);
Florin Corasf988e692017-11-27 04:34:14 -05001439 tc->rto_boff += 1;
1440 tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
Florin Coras93992a92017-05-24 18:03:56 -07001441 return;
1442 }
1443
Florin Coras3ec66b02018-08-23 16:27:05 -07001444 /* Shouldn't be here. This condition is tricky because it has to take
1445 * into account boff > 0 due to persist timeout. */
Florin Coras47596832019-03-12 18:58:54 -07001446 if ((tc->rto_boff == 0 && tc->snd_una == tc->snd_nxt)
Florin Coras3ec66b02018-08-23 16:27:05 -07001447 || (tc->rto_boff > 0 && seq_geq (tc->snd_una, tc->snd_congestion)
1448 && !tcp_flight_size (tc)))
Florin Corasa096f2d2017-09-28 23:49:42 -04001449 {
Florin Coras3ec66b02018-08-23 16:27:05 -07001450 ASSERT (!tcp_in_recovery (tc));
1451 tc->rto_boff = 0;
Florin Corasa096f2d2017-09-28 23:49:42 -04001452 return;
1453 }
1454
Florin Coras3ec66b02018-08-23 16:27:05 -07001455 /* We're not in recovery so make sure rto_boff is 0. Can be non 0 due
1456 * to persist timer timeout */
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001457 if (!tcp_in_recovery (tc) && tc->rto_boff > 0)
1458 {
1459 tc->rto_boff = 0;
1460 tcp_update_rto (tc);
1461 }
1462
Florin Coras548f7572019-06-07 15:31:06 -07001463 /* Peer is dead or network connectivity is lost. Close connection.
1464 * RFC 1122 section 4.2.3.5 recommends a value of at least 100s. For
1465 * a min rto of 0.2s we need to retry about 8 times. */
1466 if (tc->rto_boff >= TCP_RTO_BOFF_MAX)
1467 {
1468 tcp_send_reset (tc);
1469 tcp_connection_set_state (tc, TCP_STATE_CLOSED);
1470 session_transport_closing_notify (&tc->connection);
1471 tcp_connection_timers_reset (tc);
Florin Coras9094b5c2019-08-12 14:17:47 -07001472 tcp_timer_update (tc, TCP_TIMER_WAITCLOSE, tcp_cfg.closewait_time);
Florin Coras548f7572019-06-07 15:31:06 -07001473 return;
1474 }
1475
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001476 /* Increment RTO backoff (also equal to number of retries) and go back
1477 * to first un-acked byte */
1478 tc->rto_boff += 1;
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001479
Florin Corasa3c32652019-07-03 17:47:22 -07001480 /* TODO be less aggressive about clearing scoreboard */
1481 scoreboard_clear (&tc->sack_sb);
1482
Florin Coras6792ec02017-03-13 03:49:51 -07001483 /* First retransmit timeout */
1484 if (tc->rto_boff == 1)
Florin Coras75c48c12019-08-02 15:17:21 -07001485 {
1486 tcp_cc_init_rxt_timeout (tc);
1487 /* Record timestamp. Eifel detection algorithm RFC3522 */
1488 tc->snd_rxt_ts = tcp_tstamp (tc);
1489 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001490
Florin Coras52814732019-06-12 15:38:19 -07001491 if (tc->flags & TCP_CONN_RATE_SAMPLE)
1492 tcp_bt_flush_samples (tc);
1493
Florin Coras3ec66b02018-08-23 16:27:05 -07001494 /* If we've sent beyond snd_congestion, update it */
Florin Coras47596832019-03-12 18:58:54 -07001495 tc->snd_congestion = seq_max (tc->snd_nxt, tc->snd_congestion);
Florin Coras47596832019-03-12 18:58:54 -07001496 tc->snd_nxt = tc->snd_una;
Dave Barach68b0fb02017-02-28 15:15:56 -05001497
Florin Coras75c48c12019-08-02 15:17:21 -07001498 /* Send one segment. n_bytes may be zero due to buffer shortfall */
Florin Corasbe72ae62018-11-01 11:23:03 -07001499 n_bytes = tcp_prepare_retransmit_segment (wrk, tc, 0, tc->snd_mss, &b);
Florin Coras85a3ddd2018-12-24 16:54:34 -08001500 if (!n_bytes)
Florin Corase04c2992017-03-01 08:17:34 -08001501 {
Florin Coras222e1f412019-02-16 20:47:32 -08001502 tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1);
Florin Corase04c2992017-03-01 08:17:34 -08001503 return;
1504 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001505
Dave Barachb7f1faa2017-08-29 11:43:37 -04001506 bi = vlib_get_buffer_index (vm, b);
Florin Corasbe72ae62018-11-01 11:23:03 -07001507 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Florin Coras75c48c12019-08-02 15:17:21 -07001508
1509 tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
Florin Coras3ec66b02018-08-23 16:27:05 -07001510 tcp_retransmit_timer_force_update (tc);
Florin Coras93992a92017-05-24 18:03:56 -07001511 }
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001512 /* Retransmit SYN-ACK */
1513 else if (tc->state == TCP_STATE_SYN_RCVD)
1514 {
Florin Corasa436a422019-08-20 07:09:31 -07001515 TCP_EVT (TCP_EVT_CC_EVT, tc, 2);
Florin Corase55a6d72018-10-31 23:09:22 -07001516
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001517 tc->rtt_ts = 0;
1518
Florin Coras75c48c12019-08-02 15:17:21 -07001519 /* Passive open establish timeout */
1520 if (tc->rto > TCP_ESTABLISH_TIME >> 1)
1521 {
1522 tcp_connection_set_state (tc, TCP_STATE_CLOSED);
1523 tcp_connection_timers_reset (tc);
Florin Coras9094b5c2019-08-12 14:17:47 -07001524 tcp_timer_update (tc, TCP_TIMER_WAITCLOSE, tcp_cfg.cleanup_time);
Florin Coras75c48c12019-08-02 15:17:21 -07001525 return;
1526 }
Florin Coras222e1f412019-02-16 20:47:32 -08001527
Florin Corasbdf7fd62019-01-31 17:31:01 -08001528 if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
Florin Corasf988e692017-11-27 04:34:14 -05001529 {
Florin Coras222e1f412019-02-16 20:47:32 -08001530 tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1);
Florin Corasf988e692017-11-27 04:34:14 -05001531 return;
1532 }
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001533
Florin Coras75c48c12019-08-02 15:17:21 -07001534 tc->rto_boff += 1;
1535 if (tc->rto_boff > TCP_RTO_SYN_RETRIES)
1536 tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
1537
1538 tcp_retransmit_timer_force_update (tc);
1539
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001540 b = vlib_get_buffer (vm, bi);
Florin Coras24793e32018-05-09 11:34:25 -07001541 tcp_init_buffer (vm, b);
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001542 tcp_make_synack (tc, b);
Florin Corasa436a422019-08-20 07:09:31 -07001543 TCP_EVT (TCP_EVT_SYN_RXT, tc, 1);
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001544
1545 /* Retransmit timer already updated, just enqueue to output */
Florin Corasbe72ae62018-11-01 11:23:03 -07001546 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Dave Barach68b0fb02017-02-28 15:15:56 -05001547 }
Florin Coras93992a92017-05-24 18:03:56 -07001548 else
1549 {
1550 ASSERT (tc->state == TCP_STATE_CLOSED);
Florin Coras93992a92017-05-24 18:03:56 -07001551 return;
1552 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001553}
1554
Florin Coras75c48c12019-08-02 15:17:21 -07001555/**
1556 * SYN retransmit timer handler. Active open only.
1557 */
Dave Barach68b0fb02017-02-28 15:15:56 -05001558void
Florin Coras75c48c12019-08-02 15:17:21 -07001559tcp_timer_retransmit_syn_handler (u32 tc_index)
Dave Barach68b0fb02017-02-28 15:15:56 -05001560{
Florin Coras75c48c12019-08-02 15:17:21 -07001561 u32 thread_index = vlib_get_thread_index ();
1562 tcp_worker_ctx_t *wrk = tcp_get_worker (thread_index);
1563 vlib_main_t *vm = wrk->vm;
1564 tcp_connection_t *tc;
1565 vlib_buffer_t *b = 0;
1566 u32 bi;
Dave Barach68b0fb02017-02-28 15:15:56 -05001567
Florin Coras75c48c12019-08-02 15:17:21 -07001568 tc = tcp_half_open_connection_get (tc_index);
1569
1570 /* Note: the connection may have transitioned to ESTABLISHED... */
1571 if (PREDICT_FALSE (tc == 0 || tc->state != TCP_STATE_SYN_SENT))
1572 return;
1573
1574 tc->timers[TCP_TIMER_RETRANSMIT_SYN] = TCP_TIMER_HANDLE_INVALID;
1575
1576 /* Half-open connection actually moved to established but we were
1577 * waiting for syn retransmit to pop to call cleanup from the right
1578 * thread. */
1579 if (tc->flags & TCP_CONN_HALF_OPEN_DONE)
1580 {
1581 if (tcp_half_open_connection_cleanup (tc))
1582 TCP_DBG ("could not remove half-open connection");
1583 return;
1584 }
1585
Florin Corasa436a422019-08-20 07:09:31 -07001586 TCP_EVT (TCP_EVT_CC_EVT, tc, 2);
Florin Coras75c48c12019-08-02 15:17:21 -07001587 tc->rtt_ts = 0;
1588
1589 /* Active open establish timeout */
1590 if (tc->rto >= TCP_ESTABLISH_TIME >> 1)
1591 {
1592 session_stream_connect_notify (&tc->connection, 1 /* fail */ );
1593 tcp_connection_cleanup (tc);
1594 return;
1595 }
1596
1597 if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
1598 {
1599 tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN, 1);
1600 return;
1601 }
1602
1603 /* Try without increasing RTO a number of times. If this fails,
1604 * start growing RTO exponentially */
1605 tc->rto_boff += 1;
1606 if (tc->rto_boff > TCP_RTO_SYN_RETRIES)
1607 tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
1608
1609 b = vlib_get_buffer (vm, bi);
1610 tcp_init_buffer (vm, b);
1611 tcp_make_syn (tc, b);
1612
Florin Corasa436a422019-08-20 07:09:31 -07001613 TCP_EVT (TCP_EVT_SYN_RXT, tc, 0);
Florin Coras75c48c12019-08-02 15:17:21 -07001614
1615 /* This goes straight to ipx_lookup */
1616 tcp_push_ip_hdr (wrk, tc, b);
1617 tcp_enqueue_to_ip_lookup (wrk, b, bi, tc->c_is_ip4, tc->c_fib_index);
1618
1619 tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN,
1620 tc->rto * TCP_TO_TIMER_TICK);
Dave Barach68b0fb02017-02-28 15:15:56 -05001621}
1622
1623/**
Florin Coras3e350af2017-03-30 02:54:28 -07001624 * Got 0 snd_wnd from peer, try to do something about it.
1625 *
1626 */
1627void
1628tcp_timer_persist_handler (u32 index)
1629{
Damjan Marion586afd72017-04-05 19:18:20 +02001630 u32 thread_index = vlib_get_thread_index ();
Florin Corasbe72ae62018-11-01 11:23:03 -07001631 tcp_worker_ctx_t *wrk = tcp_get_worker (thread_index);
1632 u32 bi, max_snd_bytes, available_bytes, offset;
1633 tcp_main_t *tm = vnet_get_tcp_main ();
1634 vlib_main_t *vm = wrk->vm;
Florin Coras3e350af2017-03-30 02:54:28 -07001635 tcp_connection_t *tc;
1636 vlib_buffer_t *b;
Florin Coras93992a92017-05-24 18:03:56 -07001637 int n_bytes = 0;
Florin Coras1f152cd2017-08-18 19:28:03 -07001638 u8 *data;
Florin Coras3e350af2017-03-30 02:54:28 -07001639
Florin Corasdb84e572017-05-09 18:54:52 -07001640 tc = tcp_connection_get_if_valid (index, thread_index);
Florin Corasdb84e572017-05-09 18:54:52 -07001641 if (!tc)
1642 return;
Florin Coras3e350af2017-03-30 02:54:28 -07001643
1644 /* Make sure timer handle is set to invalid */
1645 tc->timers[TCP_TIMER_PERSIST] = TCP_TIMER_HANDLE_INVALID;
1646
1647 /* Problem already solved or worse */
Florin Coras7e74bf32019-03-06 16:51:58 -08001648 if (tc->state == TCP_STATE_CLOSED || tc->snd_wnd > tc->snd_mss
1649 || (tc->flags & TCP_CONN_FINSNT))
Florin Coras3e350af2017-03-30 02:54:28 -07001650 return;
1651
Florin Coras31c99552019-03-01 13:00:58 -08001652 available_bytes = transport_max_tx_dequeue (&tc->connection);
Florin Coras47596832019-03-12 18:58:54 -07001653 offset = tc->snd_nxt - tc->snd_una;
Florin Coras50958952017-08-29 14:50:13 -07001654
1655 /* Reprogram persist if no new bytes available to send. We may have data
1656 * next time */
1657 if (!available_bytes)
1658 {
1659 tcp_persist_timer_set (tc);
1660 return;
1661 }
1662
1663 if (available_bytes <= offset)
1664 {
1665 ASSERT (tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT));
1666 return;
1667 }
1668
Florin Coras3e350af2017-03-30 02:54:28 -07001669 /* Increment RTO backoff */
1670 tc->rto_boff += 1;
1671 tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
1672
Florin Corasb2215d62017-08-01 16:56:58 -07001673 /*
1674 * Try to force the first unsent segment (or buffer)
1675 */
Florin Corasbdf7fd62019-01-31 17:31:01 -08001676 if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
Florin Coras3ec66b02018-08-23 16:27:05 -07001677 {
1678 tcp_persist_timer_set (tc);
1679 return;
1680 }
Florin Coras3e350af2017-03-30 02:54:28 -07001681 b = vlib_get_buffer (vm, bi);
Florin Coras1f152cd2017-08-18 19:28:03 -07001682 data = tcp_init_buffer (vm, b);
Florin Coras93992a92017-05-24 18:03:56 -07001683
Florin Coras1f152cd2017-08-18 19:28:03 -07001684 tcp_validate_txf_size (tc, offset);
Florin Corasc8343412017-05-04 14:25:50 -07001685 tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts, tc->state);
Florin Coras7e74bf32019-03-06 16:51:58 -08001686 max_snd_bytes = clib_min (tc->snd_mss,
1687 tm->bytes_per_buffer - TRANSPORT_MAX_HDRS_LEN);
1688 n_bytes = session_tx_fifo_peek_bytes (&tc->connection, data, offset,
1689 max_snd_bytes);
Florin Coras3e350af2017-03-30 02:54:28 -07001690 b->current_length = n_bytes;
Florin Coras84275e92017-09-26 12:30:40 -04001691 ASSERT (n_bytes != 0 && (tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT)
1692 || tc->snd_nxt == tc->snd_una_max
1693 || tc->rto_boff > 1));
Florin Coras93992a92017-05-24 18:03:56 -07001694
Florin Coras47596832019-03-12 18:58:54 -07001695 tcp_push_hdr_i (tc, b, tc->snd_nxt, /* compute opts */ 0,
1696 /* burst */ 0, /* update_snd_nxt */ 1);
1697 tc->snd_una_max = seq_max (tc->snd_nxt, tc->snd_una_max);
Florin Coras8b20bf52018-06-14 14:55:50 -07001698 tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una);
Florin Corasbe72ae62018-11-01 11:23:03 -07001699 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Florin Coras3e350af2017-03-30 02:54:28 -07001700
Florin Coras1f152cd2017-08-18 19:28:03 -07001701 /* Just sent new data, enable retransmit */
1702 tcp_retransmit_timer_update (tc);
Florin Coras3e350af2017-03-30 02:54:28 -07001703}
1704
1705/**
Florin Coras6792ec02017-03-13 03:49:51 -07001706 * Retransmit first unacked segment
1707 */
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001708int
Florin Corasbe72ae62018-11-01 11:23:03 -07001709tcp_retransmit_first_unacked (tcp_worker_ctx_t * wrk, tcp_connection_t * tc)
Dave Barach68b0fb02017-02-28 15:15:56 -05001710{
Florin Corasbe72ae62018-11-01 11:23:03 -07001711 vlib_main_t *vm = wrk->vm;
1712 vlib_buffer_t *b;
Florin Coras47596832019-03-12 18:58:54 -07001713 u32 bi, n_bytes;
Dave Barach68b0fb02017-02-28 15:15:56 -05001714
Florin Corasa436a422019-08-20 07:09:31 -07001715 TCP_EVT (TCP_EVT_CC_EVT, tc, 1);
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001716
Florin Corasbe72ae62018-11-01 11:23:03 -07001717 n_bytes = tcp_prepare_retransmit_segment (wrk, tc, 0, tc->snd_mss, &b);
Florin Corasb2215d62017-08-01 16:56:58 -07001718 if (!n_bytes)
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001719 return -1;
1720
Florin Corasb2215d62017-08-01 16:56:58 -07001721 bi = vlib_get_buffer_index (vm, b);
Florin Corasbe72ae62018-11-01 11:23:03 -07001722 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001723
1724 return 0;
Florin Coras6792ec02017-03-13 03:49:51 -07001725}
1726
Florin Coras36ee9f12018-11-02 12:52:10 -07001727static int
1728tcp_fast_retransmit_unsent (tcp_worker_ctx_t * wrk, tcp_connection_t * tc,
1729 u32 burst_size)
1730{
Florin Coras635acbf2019-07-31 14:11:05 -07001731 u32 offset, n_segs = 0, n_written, bi, available_wnd;
Florin Coras36ee9f12018-11-02 12:52:10 -07001732 vlib_main_t *vm = wrk->vm;
1733 vlib_buffer_t *b = 0;
1734
Florin Coras47596832019-03-12 18:58:54 -07001735 offset = tc->snd_nxt - tc->snd_una;
Florin Coras635acbf2019-07-31 14:11:05 -07001736 available_wnd = tc->snd_wnd - offset;
1737 burst_size = clib_min (burst_size, available_wnd / tc->snd_mss);
1738
Florin Coras36ee9f12018-11-02 12:52:10 -07001739 while (n_segs < burst_size)
1740 {
1741 n_written = tcp_prepare_segment (wrk, tc, offset, tc->snd_mss, &b);
1742 if (!n_written)
1743 goto done;
1744
1745 bi = vlib_get_buffer_index (vm, b);
1746 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
1747 offset += n_written;
1748 n_segs += 1;
Florin Coras47596832019-03-12 18:58:54 -07001749
1750 tc->snd_nxt += n_written;
1751 tc->snd_una_max = seq_max (tc->snd_nxt, tc->snd_una_max);
Florin Coras36ee9f12018-11-02 12:52:10 -07001752 }
1753
1754done:
1755 return n_segs;
1756}
1757
1758#define scoreboard_rescue_rxt_valid(_sb, _tc) \
1759 (seq_geq (_sb->rescue_rxt, _tc->snd_una) \
1760 && seq_leq (_sb->rescue_rxt, _tc->snd_congestion))
1761
Florin Coras6792ec02017-03-13 03:49:51 -07001762/**
Florin Coras93992a92017-05-24 18:03:56 -07001763 * Do fast retransmit with SACKs
Florin Coras6792ec02017-03-13 03:49:51 -07001764 */
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001765int
Florin Corasbe72ae62018-11-01 11:23:03 -07001766tcp_fast_retransmit_sack (tcp_worker_ctx_t * wrk, tcp_connection_t * tc,
1767 u32 burst_size)
Dave Barach68b0fb02017-02-28 15:15:56 -05001768{
Florin Coras36ee9f12018-11-02 12:52:10 -07001769 u32 n_written = 0, offset, max_bytes, n_segs = 0, n_segs_now;
Florin Coras93992a92017-05-24 18:03:56 -07001770 sack_scoreboard_hole_t *hole;
Florin Coras36ee9f12018-11-02 12:52:10 -07001771 vlib_main_t *vm = wrk->vm;
1772 vlib_buffer_t *b = 0;
Florin Coras93992a92017-05-24 18:03:56 -07001773 sack_scoreboard_t *sb;
Florin Coras47596832019-03-12 18:58:54 -07001774 u32 bi, max_deq;
Florin Coras93992a92017-05-24 18:03:56 -07001775 int snd_space;
1776 u8 snd_limited = 0, can_rescue = 0;
Dave Barach68b0fb02017-02-28 15:15:56 -05001777
1778 ASSERT (tcp_in_fastrecovery (tc));
Dave Barach68b0fb02017-02-28 15:15:56 -05001779
Florin Corasca1c8f32018-05-23 21:01:30 -07001780 snd_space = tcp_available_cc_snd_space (tc);
Florin Corasca1c8f32018-05-23 21:01:30 -07001781 if (snd_space < tc->snd_mss)
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001782 {
Florin Coras26dd6de2019-07-23 23:54:47 -07001783 tcp_program_fastretransmit (tc);
Florin Coras36ee9f12018-11-02 12:52:10 -07001784 return 0;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001785 }
Florin Corasca1c8f32018-05-23 21:01:30 -07001786
Florin Corasa436a422019-08-20 07:09:31 -07001787 TCP_EVT (TCP_EVT_CC_EVT, tc, 0);
Florin Coras36ee9f12018-11-02 12:52:10 -07001788 sb = &tc->sack_sb;
1789 hole = scoreboard_get_hole (sb, sb->cur_rxt_hole);
1790
Florin Coras31c99552019-03-01 13:00:58 -08001791 max_deq = transport_max_tx_dequeue (&tc->connection);
Florin Coras47596832019-03-12 18:58:54 -07001792 max_deq -= tc->snd_nxt - tc->snd_una;
Florin Coras36ee9f12018-11-02 12:52:10 -07001793
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001794 while (snd_space > 0 && n_segs < burst_size)
Florin Coras93992a92017-05-24 18:03:56 -07001795 {
Florin Coras36ee9f12018-11-02 12:52:10 -07001796 hole = scoreboard_next_rxt_hole (sb, hole, max_deq, &can_rescue,
1797 &snd_limited);
Florin Coras93992a92017-05-24 18:03:56 -07001798 if (!hole)
1799 {
Florin Coras36ee9f12018-11-02 12:52:10 -07001800 if (max_deq)
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001801 {
Florin Coras36ee9f12018-11-02 12:52:10 -07001802 snd_space = clib_min (max_deq, snd_space);
1803 burst_size = clib_min (burst_size - n_segs,
1804 snd_space / tc->snd_mss);
1805 n_segs_now = tcp_fast_retransmit_unsent (wrk, tc, burst_size);
1806 if (max_deq > n_segs_now * tc->snd_mss)
Florin Coras26dd6de2019-07-23 23:54:47 -07001807 tcp_program_fastretransmit (tc);
Florin Coras36ee9f12018-11-02 12:52:10 -07001808 n_segs += n_segs_now;
1809 goto done;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001810 }
Florin Coras93992a92017-05-24 18:03:56 -07001811
Florin Coras36ee9f12018-11-02 12:52:10 -07001812 if (!can_rescue || scoreboard_rescue_rxt_valid (sb, tc))
1813 break;
1814
Florin Coras93992a92017-05-24 18:03:56 -07001815 /* If rescue rxt undefined or less than snd_una then one segment of
1816 * up to SMSS octets that MUST include the highest outstanding
1817 * unSACKed sequence number SHOULD be returned, and RescueRxt set to
1818 * RecoveryPoint. HighRxt MUST NOT be updated.
1819 */
Florin Coras1f152cd2017-08-18 19:28:03 -07001820 max_bytes = clib_min (tc->snd_mss,
1821 tc->snd_congestion - tc->snd_una);
1822 max_bytes = clib_min (max_bytes, snd_space);
Florin Coras93992a92017-05-24 18:03:56 -07001823 offset = tc->snd_congestion - tc->snd_una - max_bytes;
1824 sb->rescue_rxt = tc->snd_congestion;
Florin Corasbe72ae62018-11-01 11:23:03 -07001825 n_written = tcp_prepare_retransmit_segment (wrk, tc, offset,
1826 max_bytes, &b);
Florin Coras24793e32018-05-09 11:34:25 -07001827 if (!n_written)
1828 goto done;
1829
Florin Corasb2215d62017-08-01 16:56:58 -07001830 bi = vlib_get_buffer_index (vm, b);
Florin Corasbe72ae62018-11-01 11:23:03 -07001831 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001832 n_segs += 1;
Florin Coras93992a92017-05-24 18:03:56 -07001833 break;
1834 }
1835
Florin Coras1f152cd2017-08-18 19:28:03 -07001836 max_bytes = clib_min (hole->end - sb->high_rxt, snd_space);
1837 max_bytes = snd_limited ? clib_min (max_bytes, tc->snd_mss) : max_bytes;
1838 if (max_bytes == 0)
1839 break;
Florin Coras36ee9f12018-11-02 12:52:10 -07001840
Florin Coras93992a92017-05-24 18:03:56 -07001841 offset = sb->high_rxt - tc->snd_una;
Florin Corasbe72ae62018-11-01 11:23:03 -07001842 n_written = tcp_prepare_retransmit_segment (wrk, tc, offset, max_bytes,
1843 &b);
Florin Coras36ee9f12018-11-02 12:52:10 -07001844 ASSERT (n_written <= snd_space);
Florin Coras93992a92017-05-24 18:03:56 -07001845
1846 /* Nothing left to retransmit */
1847 if (n_written == 0)
Florin Corasb2215d62017-08-01 16:56:58 -07001848 break;
Florin Coras93992a92017-05-24 18:03:56 -07001849
Florin Corasb2215d62017-08-01 16:56:58 -07001850 bi = vlib_get_buffer_index (vm, b);
Florin Corasbe72ae62018-11-01 11:23:03 -07001851 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Florin Coras36ee9f12018-11-02 12:52:10 -07001852
1853 sb->high_rxt += n_written;
Florin Coras93992a92017-05-24 18:03:56 -07001854 snd_space -= n_written;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001855 n_segs += 1;
Florin Coras93992a92017-05-24 18:03:56 -07001856 }
1857
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001858 if (hole)
Florin Coras26dd6de2019-07-23 23:54:47 -07001859 tcp_program_fastretransmit (tc);
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001860
Florin Coras24793e32018-05-09 11:34:25 -07001861done:
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001862 return n_segs;
Florin Coras93992a92017-05-24 18:03:56 -07001863}
1864
1865/**
1866 * Fast retransmit without SACK info
1867 */
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001868int
Florin Corasbe72ae62018-11-01 11:23:03 -07001869tcp_fast_retransmit_no_sack (tcp_worker_ctx_t * wrk, tcp_connection_t * tc,
1870 u32 burst_size)
Florin Coras93992a92017-05-24 18:03:56 -07001871{
Florin Coras47596832019-03-12 18:58:54 -07001872 u32 n_written = 0, offset = 0, bi, max_deq, n_segs_now;
Florin Corasbe72ae62018-11-01 11:23:03 -07001873 vlib_main_t *vm = wrk->vm;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001874 int snd_space, n_segs = 0;
Florin Coras93992a92017-05-24 18:03:56 -07001875 vlib_buffer_t *b;
1876
1877 ASSERT (tcp_in_fastrecovery (tc));
Florin Corasa436a422019-08-20 07:09:31 -07001878 TCP_EVT (TCP_EVT_CC_EVT, tc, 0);
Florin Coras6792ec02017-03-13 03:49:51 -07001879
Florin Coras87a9bf82019-06-12 11:26:35 -07001880 snd_space = tcp_available_cc_snd_space (tc);
1881
Florin Coras36ee9f12018-11-02 12:52:10 -07001882 if (!tcp_fastrecovery_first (tc))
1883 goto send_unsent;
1884
1885 /* RFC 6582: [If a partial ack], retransmit the first unacknowledged
1886 * segment. */
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001887 while (snd_space > 0 && n_segs < burst_size)
Dave Barach68b0fb02017-02-28 15:15:56 -05001888 {
Florin Coras36ee9f12018-11-02 12:52:10 -07001889 n_written = tcp_prepare_retransmit_segment (wrk, tc, offset,
1890 tc->snd_mss, &b);
Dave Barach68b0fb02017-02-28 15:15:56 -05001891
1892 /* Nothing left to retransmit */
Florin Coras6792ec02017-03-13 03:49:51 -07001893 if (n_written == 0)
Florin Corasb2215d62017-08-01 16:56:58 -07001894 break;
Dave Barach68b0fb02017-02-28 15:15:56 -05001895
Florin Corasb2215d62017-08-01 16:56:58 -07001896 bi = vlib_get_buffer_index (vm, b);
Florin Corasbe72ae62018-11-01 11:23:03 -07001897 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Florin Coras6792ec02017-03-13 03:49:51 -07001898 snd_space -= n_written;
Florin Coras36ee9f12018-11-02 12:52:10 -07001899 offset += n_written;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001900 n_segs += 1;
Dave Barach68b0fb02017-02-28 15:15:56 -05001901 }
1902
Florin Coras36ee9f12018-11-02 12:52:10 -07001903 if (n_segs == burst_size)
1904 goto done;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001905
Florin Coras36ee9f12018-11-02 12:52:10 -07001906send_unsent:
1907
1908 /* RFC 6582: Send a new segment if permitted by the new value of cwnd. */
Florin Coras9ece3c02018-11-05 11:06:53 -08001909 if (snd_space < tc->snd_mss || tc->snd_mss == 0)
Florin Coras36ee9f12018-11-02 12:52:10 -07001910 goto done;
1911
Florin Coras31c99552019-03-01 13:00:58 -08001912 max_deq = transport_max_tx_dequeue (&tc->connection);
Florin Coras47596832019-03-12 18:58:54 -07001913 max_deq -= tc->snd_nxt - tc->snd_una;
Florin Coras36ee9f12018-11-02 12:52:10 -07001914 if (max_deq)
1915 {
1916 snd_space = clib_min (max_deq, snd_space);
1917 burst_size = clib_min (burst_size - n_segs, snd_space / tc->snd_mss);
1918 n_segs_now = tcp_fast_retransmit_unsent (wrk, tc, burst_size);
1919 if (max_deq > n_segs_now * tc->snd_mss)
Florin Coras26dd6de2019-07-23 23:54:47 -07001920 tcp_program_fastretransmit (tc);
Florin Coras36ee9f12018-11-02 12:52:10 -07001921 n_segs += n_segs_now;
1922 }
1923
Florin Coras36ee9f12018-11-02 12:52:10 -07001924done:
1925 tcp_fastrecovery_first_off (tc);
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001926 return n_segs;
Florin Coras93992a92017-05-24 18:03:56 -07001927}
1928
1929/**
1930 * Do fast retransmit
1931 */
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001932int
Florin Corasbe72ae62018-11-01 11:23:03 -07001933tcp_fast_retransmit (tcp_worker_ctx_t * wrk, tcp_connection_t * tc,
1934 u32 burst_size)
Florin Coras93992a92017-05-24 18:03:56 -07001935{
Florin Corasca1c8f32018-05-23 21:01:30 -07001936 if (tcp_opts_sack_permitted (&tc->rcv_opts))
Florin Corasbe72ae62018-11-01 11:23:03 -07001937 return tcp_fast_retransmit_sack (wrk, tc, burst_size);
Florin Coras93992a92017-05-24 18:03:56 -07001938 else
Florin Corasbe72ae62018-11-01 11:23:03 -07001939 return tcp_fast_retransmit_no_sack (wrk, tc, burst_size);
Dave Barach68b0fb02017-02-28 15:15:56 -05001940}
Florin Coras26dd6de2019-07-23 23:54:47 -07001941
1942static int
1943tcp_send_acks (tcp_connection_t * tc, u32 max_burst_size)
1944{
1945 int j, n_acks;
1946
1947 if (!tc->pending_dupacks)
1948 {
1949 tcp_send_ack (tc);
1950 return 1;
1951 }
1952
1953 /* If we're supposed to send dupacks but have no ooo data
1954 * send only one ack */
1955 if (!vec_len (tc->snd_sacks))
1956 {
1957 tcp_send_ack (tc);
1958 return 1;
1959 }
1960
1961 /* Start with first sack block */
1962 tc->snd_sack_pos = 0;
1963
1964 /* Generate enough dupacks to cover all sack blocks. Do not generate
1965 * more sacks than the number of packets received. But do generate at
1966 * least 3, i.e., the number needed to signal congestion, if needed. */
1967 n_acks = vec_len (tc->snd_sacks) / TCP_OPTS_MAX_SACK_BLOCKS;
1968 n_acks = clib_min (n_acks, tc->pending_dupacks);
1969 n_acks = clib_max (n_acks, clib_min (tc->pending_dupacks, 3));
1970 for (j = 0; j < clib_min (n_acks, max_burst_size); j++)
1971 tcp_send_ack (tc);
1972
1973 if (n_acks < max_burst_size)
1974 {
1975 tc->pending_dupacks = 0;
1976 tc->snd_sack_pos = 0;
Florin Corasedfe0ee2019-07-29 18:13:25 -07001977 tc->dupacks_out += n_acks;
Florin Coras26dd6de2019-07-23 23:54:47 -07001978 return n_acks;
1979 }
1980 else
1981 {
1982 TCP_DBG ("constrained by burst size");
1983 tc->pending_dupacks = n_acks - max_burst_size;
Florin Corasedfe0ee2019-07-29 18:13:25 -07001984 tc->dupacks_out += max_burst_size;
Florin Coras26dd6de2019-07-23 23:54:47 -07001985 tcp_program_dupack (tc);
1986 return max_burst_size;
1987 }
1988}
1989
1990static int
1991tcp_do_fastretransmit (tcp_connection_t * tc, u32 max_burst_size)
1992{
1993 u32 n_segs = 0, burst_size, sent_bytes, burst_bytes;
1994 tcp_worker_ctx_t *wrk;
1995
1996 wrk = tcp_get_worker (tc->c_thread_index);
1997 burst_bytes = transport_connection_tx_pacer_burst (&tc->connection,
1998 wrk->vm->
1999 clib_time.last_cpu_time);
2000 burst_size = clib_min (max_burst_size, burst_bytes / tc->snd_mss);
2001 if (!burst_size)
2002 {
2003 tcp_program_fastretransmit (tc);
2004 return 0;
2005 }
2006
2007 n_segs = tcp_fast_retransmit (wrk, tc, burst_size);
2008 sent_bytes = clib_min (n_segs * tc->snd_mss, burst_bytes);
2009 transport_connection_tx_pacer_update_bytes (&tc->connection, sent_bytes);
2010 return n_segs;
2011}
2012
2013int
2014tcp_session_custom_tx (void *conn, u32 max_burst_size)
2015{
2016 tcp_connection_t *tc = (tcp_connection_t *) conn;
2017 u32 n_segs = 0;
2018
2019 if (tcp_in_fastrecovery (tc) && (tc->flags & TCP_CONN_FRXT_PENDING))
2020 {
2021 tc->flags &= ~TCP_CONN_FRXT_PENDING;
2022 n_segs = tcp_do_fastretransmit (tc, max_burst_size);
2023 max_burst_size -= n_segs;
2024 }
2025
2026 if (!(tc->flags & TCP_CONN_SNDACK))
2027 return n_segs;
2028
2029 tc->flags &= ~TCP_CONN_SNDACK;
2030
2031 /* We have retransmitted packets and no dupack */
2032 if (n_segs && !tc->pending_dupacks)
2033 return n_segs;
2034
2035 if (!max_burst_size)
2036 {
2037 tcp_program_ack (tc);
2038 return max_burst_size;
2039 }
2040
2041 n_segs += tcp_send_acks (tc, max_burst_size);
2042
2043 return n_segs;
2044}
Filip Tehlare275bed2019-03-06 00:06:56 -08002045#endif /* CLIB_MARCH_VARIANT */
Dave Barach68b0fb02017-02-28 15:15:56 -05002046
Florin Corasf9d05682018-04-26 08:26:52 -07002047static void
2048tcp_output_handle_link_local (tcp_connection_t * tc0, vlib_buffer_t * b0,
Florin Coras8b20bf52018-06-14 14:55:50 -07002049 u16 * next0, u32 * error0)
Florin Corasf9d05682018-04-26 08:26:52 -07002050{
2051 ip_adjacency_t *adj;
2052 adj_index_t ai;
2053
Florin Coras1c8ff632018-05-17 13:28:34 -07002054 /* Not thread safe but as long as the connection exists the adj should
2055 * not be removed */
Florin Corasf9d05682018-04-26 08:26:52 -07002056 ai = adj_nbr_find (FIB_PROTOCOL_IP6, VNET_LINK_IP6, &tc0->c_rmt_ip,
2057 tc0->sw_if_index);
2058 if (ai == ADJ_INDEX_INVALID)
2059 {
Florin Corasf9d05682018-04-26 08:26:52 -07002060 vnet_buffer (b0)->sw_if_index[VLIB_TX] = ~0;
2061 *next0 = TCP_OUTPUT_NEXT_DROP;
2062 *error0 = TCP_ERROR_LINK_LOCAL_RW;
2063 return;
2064 }
2065
2066 adj = adj_get (ai);
Florin Coras1c8ff632018-05-17 13:28:34 -07002067 if (PREDICT_TRUE (adj->lookup_next_index == IP_LOOKUP_NEXT_REWRITE))
Florin Corasf9d05682018-04-26 08:26:52 -07002068 *next0 = TCP_OUTPUT_NEXT_IP_REWRITE;
2069 else if (adj->lookup_next_index == IP_LOOKUP_NEXT_ARP)
2070 *next0 = TCP_OUTPUT_NEXT_IP_ARP;
Florin Coras1c8ff632018-05-17 13:28:34 -07002071 else
2072 {
2073 *next0 = TCP_OUTPUT_NEXT_DROP;
2074 *error0 = TCP_ERROR_LINK_LOCAL_RW;
2075 }
2076 vnet_buffer (b0)->ip.adj_index[VLIB_TX] = ai;
Florin Corasf9d05682018-04-26 08:26:52 -07002077}
2078
Florin Coras8b20bf52018-06-14 14:55:50 -07002079static void
2080tcp46_output_trace_frame (vlib_main_t * vm, vlib_node_runtime_t * node,
2081 u32 * to_next, u32 n_bufs)
Dave Barach68b0fb02017-02-28 15:15:56 -05002082{
Florin Coras8b20bf52018-06-14 14:55:50 -07002083 u32 n_trace = vlib_get_trace_count (vm, node);
2084 tcp_connection_t *tc;
2085 tcp_tx_trace_t *t;
2086 vlib_buffer_t *b;
2087 tcp_header_t *th;
2088 int i;
Dave Barach68b0fb02017-02-28 15:15:56 -05002089
Florin Coras8b20bf52018-06-14 14:55:50 -07002090 for (i = 0; i < clib_min (n_trace, n_bufs); i++)
Dave Barach68b0fb02017-02-28 15:15:56 -05002091 {
Florin Coras8b20bf52018-06-14 14:55:50 -07002092 b = vlib_get_buffer (vm, to_next[i]);
2093 th = vlib_buffer_get_current (b);
2094 tc = tcp_connection_get (vnet_buffer (b)->tcp.connection_index,
2095 vm->thread_index);
2096 t = vlib_add_trace (vm, node, b, sizeof (*t));
Dave Barach178cf492018-11-13 16:34:13 -05002097 clib_memcpy_fast (&t->tcp_header, th, sizeof (t->tcp_header));
2098 clib_memcpy_fast (&t->tcp_connection, tc, sizeof (t->tcp_connection));
Florin Coras8b20bf52018-06-14 14:55:50 -07002099 }
2100}
Dave Barach68b0fb02017-02-28 15:15:56 -05002101
Florin Coras0dbd5172018-06-25 16:19:34 -07002102always_inline void
Florin Coras8b20bf52018-06-14 14:55:50 -07002103tcp_output_push_ip (vlib_main_t * vm, vlib_buffer_t * b0,
2104 tcp_connection_t * tc0, u8 is_ip4)
2105{
2106 tcp_header_t *th0 = 0;
Dave Barach68b0fb02017-02-28 15:15:56 -05002107
Florin Coras8b20bf52018-06-14 14:55:50 -07002108 th0 = vlib_buffer_get_current (b0);
Florin Corasa436a422019-08-20 07:09:31 -07002109 TCP_EVT (TCP_EVT_OUTPUT, tc0, th0->flags, b0->current_length);
Florin Coras8b20bf52018-06-14 14:55:50 -07002110 if (is_ip4)
2111 {
2112 vlib_buffer_push_ip4 (vm, b0, &tc0->c_lcl_ip4, &tc0->c_rmt_ip4,
2113 IP_PROTOCOL_TCP, 1);
2114 b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
2115 vnet_buffer (b0)->l4_hdr_offset = (u8 *) th0 - b0->data;
2116 th0->checksum = 0;
2117 }
2118 else
2119 {
2120 ip6_header_t *ih0;
2121 ih0 = vlib_buffer_push_ip6 (vm, b0, &tc0->c_lcl_ip6,
2122 &tc0->c_rmt_ip6, IP_PROTOCOL_TCP);
2123 b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
2124 vnet_buffer (b0)->l3_hdr_offset = (u8 *) ih0 - b0->data;
2125 vnet_buffer (b0)->l4_hdr_offset = (u8 *) th0 - b0->data;
2126 th0->checksum = 0;
2127 }
2128}
Dave Barach68b0fb02017-02-28 15:15:56 -05002129
Florin Coras0dbd5172018-06-25 16:19:34 -07002130always_inline void
Florin Coras8b20bf52018-06-14 14:55:50 -07002131tcp_output_handle_packet (tcp_connection_t * tc0, vlib_buffer_t * b0,
Florin Corasdf36f492019-08-18 18:09:28 -07002132 vlib_node_runtime_t * error_node, u16 * next0,
2133 u8 is_ip4)
Florin Coras8b20bf52018-06-14 14:55:50 -07002134{
Florin Coras540a8da2019-06-18 08:55:14 -07002135 /* If next_index is not drop use it */
Florin Coras07beade2019-06-20 12:18:31 -07002136 if (tc0->next_node_index)
2137 {
2138 *next0 = tc0->next_node_index;
2139 vnet_buffer (b0)->tcp.next_node_opaque = tc0->next_node_opaque;
2140 }
Florin Coras540a8da2019-06-18 08:55:14 -07002141
Florin Coras8b20bf52018-06-14 14:55:50 -07002142 vnet_buffer (b0)->sw_if_index[VLIB_TX] = tc0->c_fib_index;
2143 vnet_buffer (b0)->sw_if_index[VLIB_RX] = 0;
2144
2145 if (!is_ip4)
2146 {
Florin Corasdf36f492019-08-18 18:09:28 -07002147 u32 error0 = 0;
2148
Florin Coras8b20bf52018-06-14 14:55:50 -07002149 if (PREDICT_FALSE (ip6_address_is_link_local_unicast (&tc0->c_rmt_ip6)))
Florin Corasdf36f492019-08-18 18:09:28 -07002150 tcp_output_handle_link_local (tc0, b0, next0, &error0);
2151
2152 if (PREDICT_FALSE (error0))
2153 {
2154 b0->error = error_node->errors[error0];
2155 return;
2156 }
Florin Coras8b20bf52018-06-14 14:55:50 -07002157 }
2158
Florin Coras8b20bf52018-06-14 14:55:50 -07002159 if (!TCP_ALWAYS_ACK)
2160 tcp_timer_reset (tc0, TCP_TIMER_DELACK);
Florin Corasedfe0ee2019-07-29 18:13:25 -07002161
2162 tc0->segs_out += 1;
Florin Coras8b20bf52018-06-14 14:55:50 -07002163}
2164
2165always_inline uword
2166tcp46_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
2167 vlib_frame_t * frame, int is_ip4)
2168{
2169 u32 n_left_from, *from, thread_index = vm->thread_index;
2170 vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
2171 u16 nexts[VLIB_FRAME_SIZE], *next;
Florin Corasdf36f492019-08-18 18:09:28 -07002172 vlib_node_runtime_t *error_node;
2173
2174 error_node = vlib_node_get_runtime (vm, tcp_node_index (output, is_ip4));
Florin Coras8b20bf52018-06-14 14:55:50 -07002175
2176 from = vlib_frame_vector_args (frame);
2177 n_left_from = frame->n_vectors;
Florin Corasbe72ae62018-11-01 11:23:03 -07002178 tcp_set_time_now (tcp_get_worker (thread_index));
Florin Coras8b20bf52018-06-14 14:55:50 -07002179
2180 if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
2181 tcp46_output_trace_frame (vm, node, from, n_left_from);
2182
2183 vlib_get_buffers (vm, from, bufs, n_left_from);
2184 b = bufs;
2185 next = nexts;
2186
2187 while (n_left_from >= 4)
2188 {
Florin Coras8b20bf52018-06-14 14:55:50 -07002189 tcp_connection_t *tc0, *tc1;
2190
2191 {
2192 vlib_prefetch_buffer_header (b[2], STORE);
2193 CLIB_PREFETCH (b[2]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE);
2194
2195 vlib_prefetch_buffer_header (b[3], STORE);
2196 CLIB_PREFETCH (b[3]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE);
2197 }
2198
2199 next[0] = next[1] = TCP_OUTPUT_NEXT_IP_LOOKUP;
2200
2201 tc0 = tcp_connection_get (vnet_buffer (b[0])->tcp.connection_index,
2202 thread_index);
2203 tc1 = tcp_connection_get (vnet_buffer (b[1])->tcp.connection_index,
2204 thread_index);
2205
2206 tcp_output_push_ip (vm, b[0], tc0, is_ip4);
2207 tcp_output_push_ip (vm, b[1], tc1, is_ip4);
2208
Florin Corasdf36f492019-08-18 18:09:28 -07002209 tcp_output_handle_packet (tc0, b[0], error_node, &next[0], is_ip4);
2210 tcp_output_handle_packet (tc1, b[1], error_node, &next[1], is_ip4);
Florin Coras8b20bf52018-06-14 14:55:50 -07002211
2212 b += 2;
2213 next += 2;
2214 n_left_from -= 2;
2215 }
2216 while (n_left_from > 0)
2217 {
Florin Coras8b20bf52018-06-14 14:55:50 -07002218 tcp_connection_t *tc0;
2219
2220 if (n_left_from > 1)
2221 {
Florin Coras91ca4622018-06-30 11:27:59 -07002222 vlib_prefetch_buffer_header (b[1], STORE);
2223 CLIB_PREFETCH (b[1]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE);
Florin Coras8b20bf52018-06-14 14:55:50 -07002224 }
2225
2226 next[0] = TCP_OUTPUT_NEXT_IP_LOOKUP;
2227 tc0 = tcp_connection_get (vnet_buffer (b[0])->tcp.connection_index,
2228 thread_index);
2229
2230 tcp_output_push_ip (vm, b[0], tc0, is_ip4);
Florin Corasdf36f492019-08-18 18:09:28 -07002231 tcp_output_handle_packet (tc0, b[0], error_node, &next[0], is_ip4);
Florin Coras8b20bf52018-06-14 14:55:50 -07002232
2233 b += 1;
2234 next += 1;
2235 n_left_from -= 1;
2236 }
2237
2238 vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
Florin Corasdf36f492019-08-18 18:09:28 -07002239 vlib_node_increment_counter (vm, tcp_node_index (output, is_ip4),
2240 TCP_ERROR_PKTS_SENT, frame->n_vectors);
Florin Coras8b20bf52018-06-14 14:55:50 -07002241 return frame->n_vectors;
Dave Barach68b0fb02017-02-28 15:15:56 -05002242}
2243
Filip Tehlare275bed2019-03-06 00:06:56 -08002244VLIB_NODE_FN (tcp4_output_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
2245 vlib_frame_t * from_frame)
Dave Barach68b0fb02017-02-28 15:15:56 -05002246{
2247 return tcp46_output_inline (vm, node, from_frame, 1 /* is_ip4 */ );
2248}
2249
Filip Tehlare275bed2019-03-06 00:06:56 -08002250VLIB_NODE_FN (tcp6_output_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
2251 vlib_frame_t * from_frame)
Dave Barach68b0fb02017-02-28 15:15:56 -05002252{
2253 return tcp46_output_inline (vm, node, from_frame, 0 /* is_ip4 */ );
2254}
2255
Florin Corase69f4952017-03-07 10:06:24 -08002256/* *INDENT-OFF* */
Dave Barach68b0fb02017-02-28 15:15:56 -05002257VLIB_REGISTER_NODE (tcp4_output_node) =
2258{
Dave Barachf970d2f2018-11-26 18:34:33 -05002259 .name = "tcp4-output",
2260 /* Takes a vector of packets. */
2261 .vector_size = sizeof (u32),
2262 .n_errors = TCP_N_ERROR,
Dave Barach7fff3d22018-11-27 16:52:59 -05002263 .protocol_hint = VLIB_NODE_PROTO_HINT_TCP,
Dave Barachf970d2f2018-11-26 18:34:33 -05002264 .error_strings = tcp_error_strings,
2265 .n_next_nodes = TCP_OUTPUT_N_NEXT,
2266 .next_nodes = {
Dave Barach68b0fb02017-02-28 15:15:56 -05002267#define _(s,n) [TCP_OUTPUT_NEXT_##s] = n,
2268 foreach_tcp4_output_next
2269#undef _
Dave Barachf970d2f2018-11-26 18:34:33 -05002270 },
2271 .format_buffer = format_tcp_header,
2272 .format_trace = format_tcp_tx_trace,
Florin Corase69f4952017-03-07 10:06:24 -08002273};
2274/* *INDENT-ON* */
Dave Barach68b0fb02017-02-28 15:15:56 -05002275
Florin Corase69f4952017-03-07 10:06:24 -08002276/* *INDENT-OFF* */
Dave Barach68b0fb02017-02-28 15:15:56 -05002277VLIB_REGISTER_NODE (tcp6_output_node) =
2278{
Florin Corase69f4952017-03-07 10:06:24 -08002279 .name = "tcp6-output",
Dave Barach68b0fb02017-02-28 15:15:56 -05002280 /* Takes a vector of packets. */
Florin Corase69f4952017-03-07 10:06:24 -08002281 .vector_size = sizeof (u32),
2282 .n_errors = TCP_N_ERROR,
Dave Barach7fff3d22018-11-27 16:52:59 -05002283 .protocol_hint = VLIB_NODE_PROTO_HINT_TCP,
Florin Corase69f4952017-03-07 10:06:24 -08002284 .error_strings = tcp_error_strings,
2285 .n_next_nodes = TCP_OUTPUT_N_NEXT,
2286 .next_nodes = {
Dave Barach68b0fb02017-02-28 15:15:56 -05002287#define _(s,n) [TCP_OUTPUT_NEXT_##s] = n,
2288 foreach_tcp6_output_next
2289#undef _
Florin Corase69f4952017-03-07 10:06:24 -08002290 },
2291 .format_buffer = format_tcp_header,
2292 .format_trace = format_tcp_tx_trace,
2293};
2294/* *INDENT-ON* */
Dave Barach68b0fb02017-02-28 15:15:56 -05002295
Dave Barach68b0fb02017-02-28 15:15:56 -05002296typedef enum _tcp_reset_next
2297{
2298 TCP_RESET_NEXT_DROP,
2299 TCP_RESET_NEXT_IP_LOOKUP,
2300 TCP_RESET_N_NEXT
2301} tcp_reset_next_t;
2302
2303#define foreach_tcp4_reset_next \
2304 _(DROP, "error-drop") \
2305 _(IP_LOOKUP, "ip4-lookup")
2306
2307#define foreach_tcp6_reset_next \
2308 _(DROP, "error-drop") \
2309 _(IP_LOOKUP, "ip6-lookup")
2310
2311static uword
2312tcp46_send_reset_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
2313 vlib_frame_t * from_frame, u8 is_ip4)
2314{
2315 u32 n_left_from, next_index, *from, *to_next;
Damjan Marion586afd72017-04-05 19:18:20 +02002316 u32 my_thread_index = vm->thread_index;
Dave Barach68b0fb02017-02-28 15:15:56 -05002317
2318 from = vlib_frame_vector_args (from_frame);
2319 n_left_from = from_frame->n_vectors;
2320
2321 next_index = node->cached_next_index;
2322
2323 while (n_left_from > 0)
2324 {
2325 u32 n_left_to_next;
2326
2327 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2328
2329 while (n_left_from > 0 && n_left_to_next > 0)
2330 {
2331 u32 bi0;
2332 vlib_buffer_t *b0;
Clement Durand6cf260c2017-04-13 13:27:04 +02002333 tcp_tx_trace_t *t0;
2334 tcp_header_t *th0;
Dave Barach68b0fb02017-02-28 15:15:56 -05002335 u32 error0 = TCP_ERROR_RST_SENT, next0 = TCP_RESET_NEXT_IP_LOOKUP;
2336
2337 bi0 = from[0];
2338 to_next[0] = bi0;
2339 from += 1;
2340 to_next += 1;
2341 n_left_from -= 1;
2342 n_left_to_next -= 1;
2343
2344 b0 = vlib_get_buffer (vm, bi0);
2345
2346 if (tcp_make_reset_in_place (vm, b0, vnet_buffer (b0)->tcp.flags,
2347 my_thread_index, is_ip4))
2348 {
2349 error0 = TCP_ERROR_LOOKUP_DROPS;
2350 next0 = TCP_RESET_NEXT_DROP;
2351 goto done;
2352 }
2353
2354 /* Prepare to send to IP lookup */
Florin Corasf988e692017-11-27 04:34:14 -05002355 vnet_buffer (b0)->sw_if_index[VLIB_TX] = ~0;
Dave Barach68b0fb02017-02-28 15:15:56 -05002356 next0 = TCP_RESET_NEXT_IP_LOOKUP;
2357
2358 done:
Florin Corase69f4952017-03-07 10:06:24 -08002359 b0->error = node->errors[error0];
Damjan Marion213b5aa2017-07-13 21:19:27 +02002360 b0->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
Dave Barach68b0fb02017-02-28 15:15:56 -05002361 if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
2362 {
Clement Durand6cf260c2017-04-13 13:27:04 +02002363 th0 = vlib_buffer_get_current (b0);
2364 if (is_ip4)
2365 th0 = ip4_next_header ((ip4_header_t *) th0);
2366 else
2367 th0 = ip6_next_header ((ip6_header_t *) th0);
Clement Durand6cf260c2017-04-13 13:27:04 +02002368 t0 = vlib_add_trace (vm, node, b0, sizeof (*t0));
Dave Barach178cf492018-11-13 16:34:13 -05002369 clib_memcpy_fast (&t0->tcp_header, th0,
2370 sizeof (t0->tcp_header));
Dave Barach68b0fb02017-02-28 15:15:56 -05002371 }
2372
2373 vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
2374 n_left_to_next, bi0, next0);
2375 }
2376 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2377 }
2378 return from_frame->n_vectors;
2379}
2380
Filip Tehlare275bed2019-03-06 00:06:56 -08002381VLIB_NODE_FN (tcp4_reset_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
2382 vlib_frame_t * from_frame)
Dave Barach68b0fb02017-02-28 15:15:56 -05002383{
2384 return tcp46_send_reset_inline (vm, node, from_frame, 1);
2385}
2386
Filip Tehlare275bed2019-03-06 00:06:56 -08002387VLIB_NODE_FN (tcp6_reset_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
2388 vlib_frame_t * from_frame)
Dave Barach68b0fb02017-02-28 15:15:56 -05002389{
2390 return tcp46_send_reset_inline (vm, node, from_frame, 0);
2391}
2392
2393/* *INDENT-OFF* */
2394VLIB_REGISTER_NODE (tcp4_reset_node) = {
Dave Barach68b0fb02017-02-28 15:15:56 -05002395 .name = "tcp4-reset",
2396 .vector_size = sizeof (u32),
2397 .n_errors = TCP_N_ERROR,
2398 .error_strings = tcp_error_strings,
2399 .n_next_nodes = TCP_RESET_N_NEXT,
2400 .next_nodes = {
2401#define _(s,n) [TCP_RESET_NEXT_##s] = n,
2402 foreach_tcp4_reset_next
2403#undef _
2404 },
Clement Durand6cf260c2017-04-13 13:27:04 +02002405 .format_trace = format_tcp_tx_trace,
Dave Barach68b0fb02017-02-28 15:15:56 -05002406};
2407/* *INDENT-ON* */
2408
2409/* *INDENT-OFF* */
2410VLIB_REGISTER_NODE (tcp6_reset_node) = {
Dave Barach68b0fb02017-02-28 15:15:56 -05002411 .name = "tcp6-reset",
2412 .vector_size = sizeof (u32),
2413 .n_errors = TCP_N_ERROR,
2414 .error_strings = tcp_error_strings,
2415 .n_next_nodes = TCP_RESET_N_NEXT,
2416 .next_nodes = {
2417#define _(s,n) [TCP_RESET_NEXT_##s] = n,
2418 foreach_tcp6_reset_next
2419#undef _
2420 },
Clement Durand6cf260c2017-04-13 13:27:04 +02002421 .format_trace = format_tcp_tx_trace,
Dave Barach68b0fb02017-02-28 15:15:56 -05002422};
2423/* *INDENT-ON* */
2424
Dave Barach68b0fb02017-02-28 15:15:56 -05002425/*
2426 * fd.io coding-style-patch-verification: ON
2427 *
2428 * Local Variables:
2429 * eval: (c-set-style "gnu")
2430 * End:
2431 */