blob: 29a919bd160bd89bf4820ec628788c4f0a53c9d5 [file] [log] [blame]
Dave Barach68b0fb02017-02-28 15:15:56 -05001/*
2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16#include <vnet/tcp/tcp.h>
17#include <vnet/lisp-cp/packets.h>
Florin Corasb2215d62017-08-01 16:56:58 -070018#include <math.h>
Dave Barach68b0fb02017-02-28 15:15:56 -050019
20vlib_node_registration_t tcp4_output_node;
21vlib_node_registration_t tcp6_output_node;
22
Dave Barach2c25a622017-06-26 11:35:07 -040023typedef enum _tcp_output_next
Dave Barach68b0fb02017-02-28 15:15:56 -050024{
25 TCP_OUTPUT_NEXT_DROP,
Dave Barach2c25a622017-06-26 11:35:07 -040026 TCP_OUTPUT_NEXT_IP_LOOKUP,
Florin Corasf9d05682018-04-26 08:26:52 -070027 TCP_OUTPUT_NEXT_IP_REWRITE,
28 TCP_OUTPUT_NEXT_IP_ARP,
Dave Barach68b0fb02017-02-28 15:15:56 -050029 TCP_OUTPUT_N_NEXT
30} tcp_output_next_t;
31
32#define foreach_tcp4_output_next \
33 _ (DROP, "error-drop") \
Florin Corasf9d05682018-04-26 08:26:52 -070034 _ (IP_LOOKUP, "ip4-lookup") \
35 _ (IP_REWRITE, "ip4-rewrite") \
36 _ (IP_ARP, "ip4-arp")
Dave Barach68b0fb02017-02-28 15:15:56 -050037
38#define foreach_tcp6_output_next \
39 _ (DROP, "error-drop") \
Florin Corasf9d05682018-04-26 08:26:52 -070040 _ (IP_LOOKUP, "ip6-lookup") \
41 _ (IP_REWRITE, "ip6-rewrite") \
42 _ (IP_ARP, "ip6-discover-neighbor")
Dave Barach68b0fb02017-02-28 15:15:56 -050043
44static char *tcp_error_strings[] = {
45#define tcp_error(n,s) s,
46#include <vnet/tcp/tcp_error.def>
47#undef tcp_error
48};
49
50typedef struct
51{
Clement Durand6cf260c2017-04-13 13:27:04 +020052 tcp_header_t tcp_header;
53 tcp_connection_t tcp_connection;
Dave Barach68b0fb02017-02-28 15:15:56 -050054} tcp_tx_trace_t;
55
Florin Corasf6d68ed2017-05-07 19:12:02 -070056u16 dummy_mtu = 1460;
Dave Barach68b0fb02017-02-28 15:15:56 -050057
58u8 *
59format_tcp_tx_trace (u8 * s, va_list * args)
60{
61 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
62 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
Clement Durand6cf260c2017-04-13 13:27:04 +020063 tcp_tx_trace_t *t = va_arg (*args, tcp_tx_trace_t *);
Christophe Fontained3c008d2017-10-02 18:10:54 +020064 u32 indent = format_get_indent (s);
Dave Barach68b0fb02017-02-28 15:15:56 -050065
Clement Durand6cf260c2017-04-13 13:27:04 +020066 s = format (s, "%U\n%U%U",
67 format_tcp_header, &t->tcp_header, 128,
68 format_white_space, indent,
Florin Corasbb292f42017-05-19 09:49:19 -070069 format_tcp_connection, &t->tcp_connection, 1);
Dave Barach68b0fb02017-02-28 15:15:56 -050070
71 return s;
72}
73
Dave Barach68b0fb02017-02-28 15:15:56 -050074static u8
Florin Coras4eeeaaf2017-09-05 14:03:37 -040075tcp_window_compute_scale (u32 window)
Dave Barach68b0fb02017-02-28 15:15:56 -050076{
77 u8 wnd_scale = 0;
Florin Coras4eeeaaf2017-09-05 14:03:37 -040078 while (wnd_scale < TCP_MAX_WND_SCALE && (window >> wnd_scale) > TCP_WND_MAX)
Dave Barach68b0fb02017-02-28 15:15:56 -050079 wnd_scale++;
80 return wnd_scale;
81}
82
83/**
Florin Coras6534b7a2017-07-18 05:38:03 -040084 * Update max segment size we're able to process.
85 *
86 * The value is constrained by our interface's MTU and IP options. It is
87 * also what we advertise to our peer.
88 */
89void
90tcp_update_rcv_mss (tcp_connection_t * tc)
91{
92 /* TODO find our iface MTU */
Florin Corasb2215d62017-08-01 16:56:58 -070093 tc->mss = dummy_mtu - sizeof (tcp_header_t);
Florin Coras6534b7a2017-07-18 05:38:03 -040094}
95
96/**
97 * TCP's initial window
Florin Corase04c2992017-03-01 08:17:34 -080098 */
99always_inline u32
100tcp_initial_wnd_unscaled (tcp_connection_t * tc)
101{
Florin Coras6534b7a2017-07-18 05:38:03 -0400102 /* RFC 6928 recommends the value lower. However at the time our connections
103 * are initialized, fifos may not be allocated. Therefore, advertise the
104 * smallest possible unscaled window size and update once fifos are
105 * assigned to the session.
106 */
107 /*
108 tcp_update_rcv_mss (tc);
109 TCP_IW_N_SEGMENTS * tc->mss;
110 */
111 return TCP_MIN_RX_FIFO_SIZE;
Florin Corase04c2992017-03-01 08:17:34 -0800112}
113
114/**
Dave Barach68b0fb02017-02-28 15:15:56 -0500115 * Compute initial window and scale factor. As per RFC1323, window field in
116 * SYN and SYN-ACK segments is never scaled.
117 */
118u32
119tcp_initial_window_to_advertise (tcp_connection_t * tc)
120{
Florin Corasca031862018-09-24 13:58:05 -0700121 tcp_main_t *tm = &tcp_main;
Florin Corase04c2992017-03-01 08:17:34 -0800122 u32 max_fifo;
Dave Barach68b0fb02017-02-28 15:15:56 -0500123
124 /* Initial wnd for SYN. Fifos are not allocated yet.
Florin Corase04c2992017-03-01 08:17:34 -0800125 * Use some predefined value. For SYN-ACK we still want the
126 * scale to be computed in the same way */
Florin Corasca031862018-09-24 13:58:05 -0700127 max_fifo = tm->max_rx_fifo ? tm->max_rx_fifo : TCP_MAX_RX_FIFO_SIZE;
Dave Barach68b0fb02017-02-28 15:15:56 -0500128
Florin Corase04c2992017-03-01 08:17:34 -0800129 tc->rcv_wscale = tcp_window_compute_scale (max_fifo);
130 tc->rcv_wnd = tcp_initial_wnd_unscaled (tc);
Dave Barach68b0fb02017-02-28 15:15:56 -0500131
132 return clib_min (tc->rcv_wnd, TCP_WND_MAX);
133}
134
Florin Coras0dbd5172018-06-25 16:19:34 -0700135static void
Florin Coras6792ec02017-03-13 03:49:51 -0700136tcp_update_rcv_wnd (tcp_connection_t * tc)
137{
138 i32 observed_wnd;
139 u32 available_space, max_fifo, wnd;
140
Florin Corase04c2992017-03-01 08:17:34 -0800141 /*
142 * Figure out how much space we have available
143 */
Florin Corasd2aab832018-05-22 11:39:59 -0700144 available_space = transport_max_rx_enqueue (&tc->connection);
145 max_fifo = transport_rx_fifo_size (&tc->connection);
Dave Barach68b0fb02017-02-28 15:15:56 -0500146
Florin Coras93992a92017-05-24 18:03:56 -0700147 ASSERT (tc->rcv_opts.mss < max_fifo);
148 if (available_space < tc->rcv_opts.mss && available_space < max_fifo >> 3)
Florin Corase04c2992017-03-01 08:17:34 -0800149 available_space = 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500150
Florin Corase04c2992017-03-01 08:17:34 -0800151 /*
152 * Use the above and what we know about what we've previously advertised
153 * to compute the new window
154 */
Florin Coras6792ec02017-03-13 03:49:51 -0700155 observed_wnd = (i32) tc->rcv_wnd - (tc->rcv_nxt - tc->rcv_las);
156 if (observed_wnd < 0)
157 observed_wnd = 0;
Florin Corase04c2992017-03-01 08:17:34 -0800158
159 /* Bad. Thou shalt not shrink */
160 if (available_space < observed_wnd)
161 {
Florin Coras6792ec02017-03-13 03:49:51 -0700162 wnd = observed_wnd;
Florin Coras3e350af2017-03-30 02:54:28 -0700163 TCP_EVT_DBG (TCP_EVT_RCV_WND_SHRUNK, tc, observed_wnd, available_space);
Florin Corase04c2992017-03-01 08:17:34 -0800164 }
Florin Coras6792ec02017-03-13 03:49:51 -0700165 else
Florin Corase04c2992017-03-01 08:17:34 -0800166 {
Florin Coras6792ec02017-03-13 03:49:51 -0700167 wnd = available_space;
Florin Corase04c2992017-03-01 08:17:34 -0800168 }
169
Florin Coras3e350af2017-03-30 02:54:28 -0700170 /* Make sure we have a multiple of rcv_wscale */
171 if (wnd && tc->rcv_wscale)
172 {
173 wnd &= ~(1 << tc->rcv_wscale);
174 if (wnd == 0)
175 wnd = 1 << tc->rcv_wscale;
176 }
Florin Coras6792ec02017-03-13 03:49:51 -0700177
178 tc->rcv_wnd = clib_min (wnd, TCP_WND_MAX << tc->rcv_wscale);
Dave Barach68b0fb02017-02-28 15:15:56 -0500179}
180
181/**
Florin Coras0dbd5172018-06-25 16:19:34 -0700182 * Compute and return window to advertise, scaled as per RFC1323
183 */
184static u32
185tcp_window_to_advertise (tcp_connection_t * tc, tcp_state_t state)
186{
187 if (state < TCP_STATE_ESTABLISHED)
188 return tcp_initial_window_to_advertise (tc);
189
190 tcp_update_rcv_wnd (tc);
191
192 if (tc->rcv_wnd == 0)
193 {
194 tc->flags |= TCP_CONN_SENT_RCV_WND0;
195 }
196 else
197 {
198 tc->flags &= ~TCP_CONN_SENT_RCV_WND0;
199 }
200
201 return tc->rcv_wnd >> tc->rcv_wscale;
202}
203
204/**
Dave Barach68b0fb02017-02-28 15:15:56 -0500205 * Write TCP options to segment.
206 */
Florin Coras0dbd5172018-06-25 16:19:34 -0700207static u32
Dave Barach68b0fb02017-02-28 15:15:56 -0500208tcp_options_write (u8 * data, tcp_options_t * opts)
209{
210 u32 opts_len = 0;
211 u32 buf, seq_len = 4;
212
213 if (tcp_opts_mss (opts))
214 {
215 *data++ = TCP_OPTION_MSS;
216 *data++ = TCP_OPTION_LEN_MSS;
217 buf = clib_host_to_net_u16 (opts->mss);
218 clib_memcpy (data, &buf, sizeof (opts->mss));
219 data += sizeof (opts->mss);
220 opts_len += TCP_OPTION_LEN_MSS;
221 }
222
223 if (tcp_opts_wscale (opts))
224 {
225 *data++ = TCP_OPTION_WINDOW_SCALE;
226 *data++ = TCP_OPTION_LEN_WINDOW_SCALE;
227 *data++ = opts->wscale;
228 opts_len += TCP_OPTION_LEN_WINDOW_SCALE;
229 }
230
231 if (tcp_opts_sack_permitted (opts))
232 {
233 *data++ = TCP_OPTION_SACK_PERMITTED;
234 *data++ = TCP_OPTION_LEN_SACK_PERMITTED;
235 opts_len += TCP_OPTION_LEN_SACK_PERMITTED;
236 }
237
238 if (tcp_opts_tstamp (opts))
239 {
240 *data++ = TCP_OPTION_TIMESTAMP;
241 *data++ = TCP_OPTION_LEN_TIMESTAMP;
242 buf = clib_host_to_net_u32 (opts->tsval);
243 clib_memcpy (data, &buf, sizeof (opts->tsval));
244 data += sizeof (opts->tsval);
245 buf = clib_host_to_net_u32 (opts->tsecr);
246 clib_memcpy (data, &buf, sizeof (opts->tsecr));
247 data += sizeof (opts->tsecr);
248 opts_len += TCP_OPTION_LEN_TIMESTAMP;
249 }
250
251 if (tcp_opts_sack (opts))
252 {
253 int i;
254 u32 n_sack_blocks = clib_min (vec_len (opts->sacks),
255 TCP_OPTS_MAX_SACK_BLOCKS);
256
257 if (n_sack_blocks != 0)
258 {
259 *data++ = TCP_OPTION_SACK_BLOCK;
260 *data++ = 2 + n_sack_blocks * TCP_OPTION_LEN_SACK_BLOCK;
261 for (i = 0; i < n_sack_blocks; i++)
262 {
263 buf = clib_host_to_net_u32 (opts->sacks[i].start);
264 clib_memcpy (data, &buf, seq_len);
265 data += seq_len;
266 buf = clib_host_to_net_u32 (opts->sacks[i].end);
267 clib_memcpy (data, &buf, seq_len);
268 data += seq_len;
269 }
270 opts_len += 2 + n_sack_blocks * TCP_OPTION_LEN_SACK_BLOCK;
271 }
272 }
273
274 /* Terminate TCP options */
275 if (opts_len % 4)
276 {
277 *data++ = TCP_OPTION_EOL;
278 opts_len += TCP_OPTION_LEN_EOL;
279 }
280
281 /* Pad with zeroes to a u32 boundary */
282 while (opts_len % 4)
283 {
284 *data++ = TCP_OPTION_NOOP;
285 opts_len += TCP_OPTION_LEN_NOOP;
286 }
287 return opts_len;
288}
289
Florin Coras0dbd5172018-06-25 16:19:34 -0700290static int
Florin Corase04c2992017-03-01 08:17:34 -0800291tcp_make_syn_options (tcp_options_t * opts, u8 wnd_scale)
Dave Barach68b0fb02017-02-28 15:15:56 -0500292{
293 u8 len = 0;
294
295 opts->flags |= TCP_OPTS_FLAG_MSS;
296 opts->mss = dummy_mtu; /*XXX discover that */
297 len += TCP_OPTION_LEN_MSS;
298
299 opts->flags |= TCP_OPTS_FLAG_WSCALE;
Florin Corase04c2992017-03-01 08:17:34 -0800300 opts->wscale = wnd_scale;
Dave Barach68b0fb02017-02-28 15:15:56 -0500301 len += TCP_OPTION_LEN_WINDOW_SCALE;
302
303 opts->flags |= TCP_OPTS_FLAG_TSTAMP;
304 opts->tsval = tcp_time_now ();
305 opts->tsecr = 0;
306 len += TCP_OPTION_LEN_TIMESTAMP;
307
Florin Coras93992a92017-05-24 18:03:56 -0700308 if (TCP_USE_SACKS)
309 {
310 opts->flags |= TCP_OPTS_FLAG_SACK_PERMITTED;
311 len += TCP_OPTION_LEN_SACK_PERMITTED;
312 }
Dave Barach68b0fb02017-02-28 15:15:56 -0500313
314 /* Align to needed boundary */
315 len += (TCP_OPTS_ALIGN - len % TCP_OPTS_ALIGN) % TCP_OPTS_ALIGN;
316 return len;
317}
318
Florin Coras0dbd5172018-06-25 16:19:34 -0700319static int
Dave Barach68b0fb02017-02-28 15:15:56 -0500320tcp_make_synack_options (tcp_connection_t * tc, tcp_options_t * opts)
321{
322 u8 len = 0;
323
324 opts->flags |= TCP_OPTS_FLAG_MSS;
Florin Corasc8343412017-05-04 14:25:50 -0700325 opts->mss = tc->mss;
Dave Barach68b0fb02017-02-28 15:15:56 -0500326 len += TCP_OPTION_LEN_MSS;
327
Florin Coras93992a92017-05-24 18:03:56 -0700328 if (tcp_opts_wscale (&tc->rcv_opts))
Dave Barach68b0fb02017-02-28 15:15:56 -0500329 {
330 opts->flags |= TCP_OPTS_FLAG_WSCALE;
331 opts->wscale = tc->rcv_wscale;
332 len += TCP_OPTION_LEN_WINDOW_SCALE;
333 }
334
Florin Coras93992a92017-05-24 18:03:56 -0700335 if (tcp_opts_tstamp (&tc->rcv_opts))
Dave Barach68b0fb02017-02-28 15:15:56 -0500336 {
337 opts->flags |= TCP_OPTS_FLAG_TSTAMP;
338 opts->tsval = tcp_time_now ();
339 opts->tsecr = tc->tsval_recent;
340 len += TCP_OPTION_LEN_TIMESTAMP;
341 }
342
Florin Coras93992a92017-05-24 18:03:56 -0700343 if (tcp_opts_sack_permitted (&tc->rcv_opts))
Dave Barach68b0fb02017-02-28 15:15:56 -0500344 {
345 opts->flags |= TCP_OPTS_FLAG_SACK_PERMITTED;
346 len += TCP_OPTION_LEN_SACK_PERMITTED;
347 }
348
349 /* Align to needed boundary */
350 len += (TCP_OPTS_ALIGN - len % TCP_OPTS_ALIGN) % TCP_OPTS_ALIGN;
351 return len;
352}
353
Florin Coras0dbd5172018-06-25 16:19:34 -0700354static int
Dave Barach68b0fb02017-02-28 15:15:56 -0500355tcp_make_established_options (tcp_connection_t * tc, tcp_options_t * opts)
356{
357 u8 len = 0;
358
359 opts->flags = 0;
360
Florin Coras93992a92017-05-24 18:03:56 -0700361 if (tcp_opts_tstamp (&tc->rcv_opts))
Dave Barach68b0fb02017-02-28 15:15:56 -0500362 {
363 opts->flags |= TCP_OPTS_FLAG_TSTAMP;
Florin Corasbe72ae62018-11-01 11:23:03 -0700364 opts->tsval = tcp_time_now_w_thread (tc->c_thread_index);
Dave Barach68b0fb02017-02-28 15:15:56 -0500365 opts->tsecr = tc->tsval_recent;
366 len += TCP_OPTION_LEN_TIMESTAMP;
367 }
Florin Coras93992a92017-05-24 18:03:56 -0700368 if (tcp_opts_sack_permitted (&tc->rcv_opts))
Dave Barach68b0fb02017-02-28 15:15:56 -0500369 {
370 if (vec_len (tc->snd_sacks))
371 {
372 opts->flags |= TCP_OPTS_FLAG_SACK;
373 opts->sacks = tc->snd_sacks;
Florin Corasc28764f2017-04-26 00:08:42 -0700374 opts->n_sack_blocks = clib_min (vec_len (tc->snd_sacks),
375 TCP_OPTS_MAX_SACK_BLOCKS);
Dave Barach68b0fb02017-02-28 15:15:56 -0500376 len += 2 + TCP_OPTION_LEN_SACK_BLOCK * opts->n_sack_blocks;
377 }
378 }
379
380 /* Align to needed boundary */
381 len += (TCP_OPTS_ALIGN - len % TCP_OPTS_ALIGN) % TCP_OPTS_ALIGN;
382 return len;
383}
384
385always_inline int
386tcp_make_options (tcp_connection_t * tc, tcp_options_t * opts,
387 tcp_state_t state)
388{
389 switch (state)
390 {
391 case TCP_STATE_ESTABLISHED:
392 case TCP_STATE_FIN_WAIT_1:
Florin Corasca1c8f32018-05-23 21:01:30 -0700393 case TCP_STATE_CLOSED:
Florin Coras25579b42018-06-06 17:55:02 -0700394 case TCP_STATE_CLOSE_WAIT:
Dave Barach68b0fb02017-02-28 15:15:56 -0500395 return tcp_make_established_options (tc, opts);
396 case TCP_STATE_SYN_RCVD:
397 return tcp_make_synack_options (tc, opts);
398 case TCP_STATE_SYN_SENT:
Florin Corase04c2992017-03-01 08:17:34 -0800399 return tcp_make_syn_options (opts, tc->rcv_wscale);
Dave Barach68b0fb02017-02-28 15:15:56 -0500400 default:
Florin Coras371ca502018-02-21 12:07:41 -0800401 clib_warning ("State not handled! %d", state);
Dave Barach68b0fb02017-02-28 15:15:56 -0500402 return 0;
403 }
404}
405
Florin Corasc8343412017-05-04 14:25:50 -0700406/**
Florin Corasb26743d2018-06-26 09:31:04 -0700407 * Update burst send vars
408 *
409 * - Updates snd_mss to reflect the effective segment size that we can send
410 * by taking into account all TCP options, including SACKs.
411 * - Cache 'on the wire' options for reuse
412 * - Updates receive window which can be reused for a burst.
413 *
414 * This should *only* be called when doing bursts
Florin Corasc8343412017-05-04 14:25:50 -0700415 */
416void
Florin Corasb26743d2018-06-26 09:31:04 -0700417tcp_update_burst_snd_vars (tcp_connection_t * tc)
Florin Corasc8343412017-05-04 14:25:50 -0700418{
Florin Corasb26743d2018-06-26 09:31:04 -0700419 tcp_main_t *tm = &tcp_main;
420
Florin Corasc8343412017-05-04 14:25:50 -0700421 /* Compute options to be used for connection. These may be reused when
422 * sending data or to compute the effective mss (snd_mss) */
Florin Corasb26743d2018-06-26 09:31:04 -0700423 tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts,
424 TCP_STATE_ESTABLISHED);
Florin Corasc8343412017-05-04 14:25:50 -0700425
426 /* XXX check if MTU has been updated */
Florin Coras93992a92017-05-24 18:03:56 -0700427 tc->snd_mss = clib_min (tc->mss, tc->rcv_opts.mss) - tc->snd_opts_len;
Florin Corasdb84e572017-05-09 18:54:52 -0700428 ASSERT (tc->snd_mss > 0);
Florin Corasb26743d2018-06-26 09:31:04 -0700429
430 tcp_options_write (tm->wrk_ctx[tc->c_thread_index].cached_opts,
431 &tc->snd_opts);
432
433 tcp_update_rcv_wnd (tc);
Florin Corasc8343412017-05-04 14:25:50 -0700434}
435
436void
437tcp_init_mss (tcp_connection_t * tc)
438{
Florin Corasdb84e572017-05-09 18:54:52 -0700439 u16 default_min_mss = 536;
Florin Corasc8343412017-05-04 14:25:50 -0700440 tcp_update_rcv_mss (tc);
441
442 /* TODO cache mss and consider PMTU discovery */
Florin Coras93992a92017-05-24 18:03:56 -0700443 tc->snd_mss = clib_min (tc->rcv_opts.mss, tc->mss);
Florin Corasc8343412017-05-04 14:25:50 -0700444
Florin Corasdb84e572017-05-09 18:54:52 -0700445 if (tc->snd_mss < 45)
Florin Corasc8343412017-05-04 14:25:50 -0700446 {
447 clib_warning ("snd mss is 0");
Florin Corasdb84e572017-05-09 18:54:52 -0700448 /* Assume that at least the min default mss works */
449 tc->snd_mss = default_min_mss;
Florin Coras93992a92017-05-24 18:03:56 -0700450 tc->rcv_opts.mss = default_min_mss;
Florin Corasc8343412017-05-04 14:25:50 -0700451 }
452
453 /* We should have enough space for 40 bytes of options */
454 ASSERT (tc->snd_mss > 45);
455
456 /* If we use timestamp option, account for it */
Florin Coras93992a92017-05-24 18:03:56 -0700457 if (tcp_opts_tstamp (&tc->rcv_opts))
Florin Corasc8343412017-05-04 14:25:50 -0700458 tc->snd_mss -= TCP_OPTION_LEN_TIMESTAMP;
459}
460
Florin Coras0dbd5172018-06-25 16:19:34 -0700461static int
Florin Corasbe72ae62018-11-01 11:23:03 -0700462tcp_alloc_tx_buffers (tcp_worker_ctx_t * wrk, u16 * n_bufs, u32 wanted)
Florin Corasb2215d62017-08-01 16:56:58 -0700463{
Florin Coras2f9b0c02017-09-11 20:54:15 -0400464 vlib_main_t *vm = vlib_get_main ();
Florin Coras24793e32018-05-09 11:34:25 -0700465 u32 n_alloc;
Dave Barachb7f1faa2017-08-29 11:43:37 -0400466
Florin Coras24793e32018-05-09 11:34:25 -0700467 ASSERT (wanted > *n_bufs);
Florin Corasbe72ae62018-11-01 11:23:03 -0700468 vec_validate_aligned (wrk->tx_buffers, wanted - 1, CLIB_CACHE_LINE_BYTES);
469 n_alloc = vlib_buffer_alloc (vm, &wrk->tx_buffers[*n_bufs],
Florin Coras24793e32018-05-09 11:34:25 -0700470 wanted - *n_bufs);
471 *n_bufs += n_alloc;
Florin Corasbe72ae62018-11-01 11:23:03 -0700472 _vec_len (wrk->tx_buffers) = *n_bufs;
Florin Coras24793e32018-05-09 11:34:25 -0700473 return n_alloc;
Florin Corasb2215d62017-08-01 16:56:58 -0700474}
475
476always_inline int
Florin Corasbe72ae62018-11-01 11:23:03 -0700477tcp_get_free_buffer_index (tcp_worker_ctx_t * wrk, u32 * bidx)
Florin Coras66b11312017-07-31 17:18:03 -0700478{
Florin Corasbe72ae62018-11-01 11:23:03 -0700479 u16 n_bufs = vec_len (wrk->tx_buffers);
Florin Corasf988e692017-11-27 04:34:14 -0500480
Florin Corasbe72ae62018-11-01 11:23:03 -0700481 TCP_DBG_BUFFER_ALLOC_MAYBE_FAIL (wrk->vm->thread_index);
Florin Corasf988e692017-11-27 04:34:14 -0500482
Florin Coras24793e32018-05-09 11:34:25 -0700483 if (PREDICT_FALSE (!n_bufs))
Florin Coras66b11312017-07-31 17:18:03 -0700484 {
Florin Corasbe72ae62018-11-01 11:23:03 -0700485 if (!tcp_alloc_tx_buffers (wrk, &n_bufs, VLIB_FRAME_SIZE))
Florin Coras24793e32018-05-09 11:34:25 -0700486 {
487 *bidx = ~0;
488 return -1;
489 }
Florin Coras66b11312017-07-31 17:18:03 -0700490 }
Florin Corasbe72ae62018-11-01 11:23:03 -0700491 *bidx = wrk->tx_buffers[--n_bufs];
492 _vec_len (wrk->tx_buffers) = n_bufs;
Florin Coras66b11312017-07-31 17:18:03 -0700493 return 0;
494}
Dave Barach68b0fb02017-02-28 15:15:56 -0500495
Florin Coras0dbd5172018-06-25 16:19:34 -0700496static void *
Dave Barach68b0fb02017-02-28 15:15:56 -0500497tcp_reuse_buffer (vlib_main_t * vm, vlib_buffer_t * b)
498{
Florin Corasb2215d62017-08-01 16:56:58 -0700499 if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
500 vlib_buffer_free_one (vm, b->next_buffer);
Florin Coras4eeeaaf2017-09-05 14:03:37 -0400501 /* Zero all flags but free list index and trace flag */
502 b->flags &= VLIB_BUFFER_NEXT_PRESENT - 1;
Florin Coras1f152cd2017-08-18 19:28:03 -0700503 b->current_data = 0;
504 b->current_length = 0;
505 b->total_length_not_including_first_buffer = 0;
506 vnet_buffer (b)->tcp.flags = 0;
Florin Corasb2215d62017-08-01 16:56:58 -0700507
Dave Barach68b0fb02017-02-28 15:15:56 -0500508 /* Leave enough space for headers */
Florin Coras1f152cd2017-08-18 19:28:03 -0700509 return vlib_buffer_make_headroom (b, MAX_HDRS_LEN);
510}
511
Florin Coras0dbd5172018-06-25 16:19:34 -0700512static void *
Florin Coras1f152cd2017-08-18 19:28:03 -0700513tcp_init_buffer (vlib_main_t * vm, vlib_buffer_t * b)
514{
515 ASSERT ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
Damjan Mariondac03522018-02-01 15:30:13 +0100516 b->flags &= VLIB_BUFFER_NON_DEFAULT_FREELIST;
Florin Coras4eeeaaf2017-09-05 14:03:37 -0400517 b->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
Florin Coras1f152cd2017-08-18 19:28:03 -0700518 b->total_length_not_including_first_buffer = 0;
Florin Coras24793e32018-05-09 11:34:25 -0700519 b->current_data = 0;
Florin Corasd79b41e2017-03-04 05:37:52 -0800520 vnet_buffer (b)->tcp.flags = 0;
Florin Corasf988e692017-11-27 04:34:14 -0500521 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b);
Florin Coras1f152cd2017-08-18 19:28:03 -0700522 /* Leave enough space for headers */
523 return vlib_buffer_make_headroom (b, MAX_HDRS_LEN);
Dave Barach68b0fb02017-02-28 15:15:56 -0500524}
525
526/**
527 * Prepare ACK
528 */
Florin Coras0dbd5172018-06-25 16:19:34 -0700529static void
Dave Barach68b0fb02017-02-28 15:15:56 -0500530tcp_make_ack_i (tcp_connection_t * tc, vlib_buffer_t * b, tcp_state_t state,
531 u8 flags)
532{
533 tcp_options_t _snd_opts, *snd_opts = &_snd_opts;
534 u8 tcp_opts_len, tcp_hdr_opts_len;
535 tcp_header_t *th;
536 u16 wnd;
537
538 wnd = tcp_window_to_advertise (tc, state);
539
540 /* Make and write options */
541 tcp_opts_len = tcp_make_established_options (tc, snd_opts);
542 tcp_hdr_opts_len = tcp_opts_len + sizeof (tcp_header_t);
543
544 th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, tc->snd_nxt,
545 tc->rcv_nxt, tcp_hdr_opts_len, flags, wnd);
546
547 tcp_options_write ((u8 *) (th + 1), snd_opts);
Dave Barach68b0fb02017-02-28 15:15:56 -0500548 vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
549}
550
551/**
552 * Convert buffer to ACK
553 */
554void
555tcp_make_ack (tcp_connection_t * tc, vlib_buffer_t * b)
556{
Florin Coras6792ec02017-03-13 03:49:51 -0700557 vlib_main_t *vm = vlib_get_main ();
Dave Barach68b0fb02017-02-28 15:15:56 -0500558
559 tcp_reuse_buffer (vm, b);
560 tcp_make_ack_i (tc, b, TCP_STATE_ESTABLISHED, TCP_FLAG_ACK);
Florin Coras6792ec02017-03-13 03:49:51 -0700561 TCP_EVT_DBG (TCP_EVT_ACK_SENT, tc);
Florin Coras3e350af2017-03-30 02:54:28 -0700562 vnet_buffer (b)->tcp.flags = TCP_BUF_FLAG_ACK;
563 tc->rcv_las = tc->rcv_nxt;
Dave Barach68b0fb02017-02-28 15:15:56 -0500564}
565
566/**
567 * Convert buffer to FIN-ACK
568 */
569void
Florin Corasd79b41e2017-03-04 05:37:52 -0800570tcp_make_fin (tcp_connection_t * tc, vlib_buffer_t * b)
Dave Barach68b0fb02017-02-28 15:15:56 -0500571{
Florin Coras6792ec02017-03-13 03:49:51 -0700572 vlib_main_t *vm = vlib_get_main ();
Florin Corasd79b41e2017-03-04 05:37:52 -0800573 u8 flags = 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500574
575 tcp_reuse_buffer (vm, b);
Florin Corasd79b41e2017-03-04 05:37:52 -0800576
Florin Corase69f4952017-03-07 10:06:24 -0800577 flags = TCP_FLAG_FIN | TCP_FLAG_ACK;
Florin Corasd79b41e2017-03-04 05:37:52 -0800578 tcp_make_ack_i (tc, b, TCP_STATE_ESTABLISHED, flags);
Dave Barach68b0fb02017-02-28 15:15:56 -0500579
580 /* Reset flags, make sure ack is sent */
Dave Barach68b0fb02017-02-28 15:15:56 -0500581 vnet_buffer (b)->tcp.flags &= ~TCP_BUF_FLAG_DUPACK;
Florin Coras4eeeaaf2017-09-05 14:03:37 -0400582}
Dave Barach68b0fb02017-02-28 15:15:56 -0500583
Florin Coras4eeeaaf2017-09-05 14:03:37 -0400584/**
585 * Convert buffer to SYN
586 */
587void
588tcp_make_syn (tcp_connection_t * tc, vlib_buffer_t * b)
589{
590 u8 tcp_hdr_opts_len, tcp_opts_len;
591 tcp_header_t *th;
592 u16 initial_wnd;
593 tcp_options_t snd_opts;
594
595 initial_wnd = tcp_initial_window_to_advertise (tc);
596
597 /* Make and write options */
Dave Barachb7b92992018-10-17 10:38:51 -0400598 clib_memset (&snd_opts, 0, sizeof (snd_opts));
Florin Coras4eeeaaf2017-09-05 14:03:37 -0400599 tcp_opts_len = tcp_make_syn_options (&snd_opts, tc->rcv_wscale);
600 tcp_hdr_opts_len = tcp_opts_len + sizeof (tcp_header_t);
601
602 th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, tc->iss,
603 tc->rcv_nxt, tcp_hdr_opts_len, TCP_FLAG_SYN,
604 initial_wnd);
605 vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
606 tcp_options_write ((u8 *) (th + 1), &snd_opts);
Dave Barach68b0fb02017-02-28 15:15:56 -0500607}
608
609/**
610 * Convert buffer to SYN-ACK
611 */
612void
613tcp_make_synack (tcp_connection_t * tc, vlib_buffer_t * b)
614{
Florin Coras6792ec02017-03-13 03:49:51 -0700615 vlib_main_t *vm = vlib_get_main ();
Dave Barach68b0fb02017-02-28 15:15:56 -0500616 tcp_options_t _snd_opts, *snd_opts = &_snd_opts;
617 u8 tcp_opts_len, tcp_hdr_opts_len;
618 tcp_header_t *th;
619 u16 initial_wnd;
Dave Barach68b0fb02017-02-28 15:15:56 -0500620
Dave Barachb7b92992018-10-17 10:38:51 -0400621 clib_memset (snd_opts, 0, sizeof (*snd_opts));
Dave Barach68b0fb02017-02-28 15:15:56 -0500622 tcp_reuse_buffer (vm, b);
623
Dave Barach68b0fb02017-02-28 15:15:56 -0500624 initial_wnd = tcp_initial_window_to_advertise (tc);
Dave Barach68b0fb02017-02-28 15:15:56 -0500625 tcp_opts_len = tcp_make_synack_options (tc, snd_opts);
626 tcp_hdr_opts_len = tcp_opts_len + sizeof (tcp_header_t);
627
628 th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, tc->iss,
629 tc->rcv_nxt, tcp_hdr_opts_len,
630 TCP_FLAG_SYN | TCP_FLAG_ACK, initial_wnd);
Dave Barach68b0fb02017-02-28 15:15:56 -0500631 tcp_options_write ((u8 *) (th + 1), snd_opts);
632
633 vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
634 vnet_buffer (b)->tcp.flags = TCP_BUF_FLAG_ACK;
635
Florin Coras4eeeaaf2017-09-05 14:03:37 -0400636 /* Init retransmit timer. Use update instead of set because of
637 * retransmissions */
638 tcp_retransmit_timer_force_update (tc);
Florin Coras6534b7a2017-07-18 05:38:03 -0400639 TCP_EVT_DBG (TCP_EVT_SYNACK_SENT, tc);
Dave Barach68b0fb02017-02-28 15:15:56 -0500640}
641
642always_inline void
Florin Corasbe72ae62018-11-01 11:23:03 -0700643tcp_enqueue_to_ip_lookup_i (tcp_worker_ctx_t * wrk, vlib_buffer_t * b, u32 bi,
Florin Coras56b39f62018-03-27 17:29:32 -0700644 u8 is_ip4, u32 fib_index, u8 flush)
Dave Barach68b0fb02017-02-28 15:15:56 -0500645{
Florin Corasbe72ae62018-11-01 11:23:03 -0700646 vlib_main_t *vm = wrk->vm;
Dave Barach68b0fb02017-02-28 15:15:56 -0500647 u32 *to_next, next_index;
648 vlib_frame_t *f;
649
Damjan Marion213b5aa2017-07-13 21:19:27 +0200650 b->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
Dave Barach68b0fb02017-02-28 15:15:56 -0500651 b->error = 0;
652
Florin Coras56b39f62018-03-27 17:29:32 -0700653 vnet_buffer (b)->sw_if_index[VLIB_TX] = fib_index;
654 vnet_buffer (b)->sw_if_index[VLIB_RX] = 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500655
656 /* Send to IP lookup */
657 next_index = is_ip4 ? ip4_lookup_node.index : ip6_lookup_node.index;
Florin Corasf988e692017-11-27 04:34:14 -0500658 tcp_trajectory_add_start (b, 1);
Dave Barach68b0fb02017-02-28 15:15:56 -0500659
Florin Corasbe72ae62018-11-01 11:23:03 -0700660 f = wrk->ip_lookup_tx_frames[!is_ip4];
Florin Coras9d063042017-09-14 03:08:00 -0400661 if (!f)
662 {
663 f = vlib_get_frame_to_node (vm, next_index);
664 ASSERT (f);
Florin Corasbe72ae62018-11-01 11:23:03 -0700665 wrk->ip_lookup_tx_frames[!is_ip4] = f;
Florin Coras9d063042017-09-14 03:08:00 -0400666 }
667
Dave Barach68b0fb02017-02-28 15:15:56 -0500668 to_next = vlib_frame_vector_args (f);
Florin Coras9d063042017-09-14 03:08:00 -0400669 to_next[f->n_vectors] = bi;
670 f->n_vectors += 1;
671 if (flush || f->n_vectors == VLIB_FRAME_SIZE)
672 {
673 vlib_put_frame_to_node (vm, next_index, f);
Florin Corasbe72ae62018-11-01 11:23:03 -0700674 wrk->ip_lookup_tx_frames[!is_ip4] = 0;
Florin Coras9d063042017-09-14 03:08:00 -0400675 }
676}
677
Florin Coras0dbd5172018-06-25 16:19:34 -0700678static void
Florin Corasbe72ae62018-11-01 11:23:03 -0700679tcp_enqueue_to_ip_lookup_now (tcp_worker_ctx_t * wrk, vlib_buffer_t * b,
680 u32 bi, u8 is_ip4, u32 fib_index)
Florin Coras9d063042017-09-14 03:08:00 -0400681{
Florin Corasbe72ae62018-11-01 11:23:03 -0700682 tcp_enqueue_to_ip_lookup_i (wrk, b, bi, is_ip4, fib_index, 1);
Florin Coras9d063042017-09-14 03:08:00 -0400683}
684
Florin Coras0dbd5172018-06-25 16:19:34 -0700685static void
Florin Corasbe72ae62018-11-01 11:23:03 -0700686tcp_enqueue_to_ip_lookup (tcp_worker_ctx_t * wrk, vlib_buffer_t * b, u32 bi,
Florin Coras56b39f62018-03-27 17:29:32 -0700687 u8 is_ip4, u32 fib_index)
Florin Coras9d063042017-09-14 03:08:00 -0400688{
Florin Corasbe72ae62018-11-01 11:23:03 -0700689 tcp_enqueue_to_ip_lookup_i (wrk, b, bi, is_ip4, fib_index, 0);
690 if (wrk->vm->thread_index == 0 && vlib_num_workers ())
691 session_flush_frames_main_thread (wrk->vm);
Dave Barach68b0fb02017-02-28 15:15:56 -0500692}
693
Florin Coras1f152cd2017-08-18 19:28:03 -0700694always_inline void
Florin Corasbe72ae62018-11-01 11:23:03 -0700695tcp_enqueue_to_output_i (tcp_worker_ctx_t * wrk, vlib_buffer_t * b, u32 bi,
Florin Coras1f152cd2017-08-18 19:28:03 -0700696 u8 is_ip4, u8 flush)
697{
Florin Coras1f152cd2017-08-18 19:28:03 -0700698 u32 *to_next, next_index;
699 vlib_frame_t *f;
700
701 b->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
702 b->error = 0;
703
704 /* Decide where to send the packet */
705 next_index = is_ip4 ? tcp4_output_node.index : tcp6_output_node.index;
Florin Corasf988e692017-11-27 04:34:14 -0500706 tcp_trajectory_add_start (b, 2);
Florin Coras1f152cd2017-08-18 19:28:03 -0700707
708 /* Get frame to v4/6 output node */
Florin Corasbe72ae62018-11-01 11:23:03 -0700709 f = wrk->tx_frames[!is_ip4];
Florin Coras1f152cd2017-08-18 19:28:03 -0700710 if (!f)
711 {
Florin Corasbe72ae62018-11-01 11:23:03 -0700712 f = vlib_get_frame_to_node (wrk->vm, next_index);
Florin Coras1f152cd2017-08-18 19:28:03 -0700713 ASSERT (f);
Florin Corasbe72ae62018-11-01 11:23:03 -0700714 wrk->tx_frames[!is_ip4] = f;
Florin Coras1f152cd2017-08-18 19:28:03 -0700715 }
716 to_next = vlib_frame_vector_args (f);
717 to_next[f->n_vectors] = bi;
718 f->n_vectors += 1;
719 if (flush || f->n_vectors == VLIB_FRAME_SIZE)
720 {
Florin Corasbe72ae62018-11-01 11:23:03 -0700721 vlib_put_frame_to_node (wrk->vm, next_index, f);
722 wrk->tx_frames[!is_ip4] = 0;
Florin Coras1f152cd2017-08-18 19:28:03 -0700723 }
724}
725
Florin Coras0dbd5172018-06-25 16:19:34 -0700726static void
Florin Corasbe72ae62018-11-01 11:23:03 -0700727tcp_enqueue_to_output (tcp_worker_ctx_t * wrk, vlib_buffer_t * b, u32 bi,
728 u8 is_ip4)
Florin Coras1f152cd2017-08-18 19:28:03 -0700729{
Florin Corasbe72ae62018-11-01 11:23:03 -0700730 tcp_enqueue_to_output_i (wrk, b, bi, is_ip4, 0);
Florin Coras1f152cd2017-08-18 19:28:03 -0700731}
732
Florin Coras0dbd5172018-06-25 16:19:34 -0700733static void
Florin Corasbe72ae62018-11-01 11:23:03 -0700734tcp_enqueue_to_output_now (tcp_worker_ctx_t * wrk, vlib_buffer_t * b, u32 bi,
Florin Coras1f152cd2017-08-18 19:28:03 -0700735 u8 is_ip4)
736{
Florin Corasbe72ae62018-11-01 11:23:03 -0700737 tcp_enqueue_to_output_i (wrk, b, bi, is_ip4, 1);
Florin Coras1f152cd2017-08-18 19:28:03 -0700738}
739
Florin Coras0dbd5172018-06-25 16:19:34 -0700740static int
Dave Barach68b0fb02017-02-28 15:15:56 -0500741tcp_make_reset_in_place (vlib_main_t * vm, vlib_buffer_t * b0,
Florin Corasdc629cd2017-05-09 00:52:37 -0700742 tcp_state_t state, u8 thread_index, u8 is_ip4)
Dave Barach68b0fb02017-02-28 15:15:56 -0500743{
Dave Barach68b0fb02017-02-28 15:15:56 -0500744 ip4_header_t *ih4;
745 ip6_header_t *ih6;
746 tcp_header_t *th0;
Florin Corasdc629cd2017-05-09 00:52:37 -0700747 ip4_address_t src_ip40, dst_ip40;
748 ip6_address_t src_ip60, dst_ip60;
749 u16 src_port, dst_port;
Dave Barach68b0fb02017-02-28 15:15:56 -0500750 u32 tmp;
Florin Corasdc629cd2017-05-09 00:52:37 -0700751 u32 seq, ack;
752 u8 flags;
Dave Barach68b0fb02017-02-28 15:15:56 -0500753
754 /* Find IP and TCP headers */
Florin Corasdc629cd2017-05-09 00:52:37 -0700755 th0 = tcp_buffer_hdr (b0);
756
757 /* Save src and dst ip */
Dave Barach68b0fb02017-02-28 15:15:56 -0500758 if (is_ip4)
759 {
760 ih4 = vlib_buffer_get_current (b0);
Florin Corasdc629cd2017-05-09 00:52:37 -0700761 ASSERT ((ih4->ip_version_and_header_length & 0xF0) == 0x40);
762 src_ip40.as_u32 = ih4->src_address.as_u32;
763 dst_ip40.as_u32 = ih4->dst_address.as_u32;
Dave Barach68b0fb02017-02-28 15:15:56 -0500764 }
765 else
766 {
767 ih6 = vlib_buffer_get_current (b0);
Dave Barach68b0fb02017-02-28 15:15:56 -0500768 ASSERT ((ih6->ip_version_traffic_class_and_flow_label & 0xF0) == 0x60);
769 clib_memcpy (&src_ip60, &ih6->src_address, sizeof (ip6_address_t));
Florin Corasdc629cd2017-05-09 00:52:37 -0700770 clib_memcpy (&dst_ip60, &ih6->dst_address, sizeof (ip6_address_t));
Dave Barach68b0fb02017-02-28 15:15:56 -0500771 }
772
Florin Corasdc629cd2017-05-09 00:52:37 -0700773 src_port = th0->src_port;
774 dst_port = th0->dst_port;
775
776 /* Try to determine what/why we're actually resetting */
Dave Barach68b0fb02017-02-28 15:15:56 -0500777 if (state == TCP_STATE_CLOSED)
778 {
779 if (!tcp_syn (th0))
780 return -1;
781
782 tmp = clib_net_to_host_u32 (th0->seq_number);
783
784 /* Got a SYN for no listener. */
Florin Corasdc629cd2017-05-09 00:52:37 -0700785 flags = TCP_FLAG_RST | TCP_FLAG_ACK;
786 ack = clib_host_to_net_u32 (tmp + 1);
787 seq = 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500788 }
Florin Corasdc629cd2017-05-09 00:52:37 -0700789 else
Dave Barach68b0fb02017-02-28 15:15:56 -0500790 {
Florin Corasdc629cd2017-05-09 00:52:37 -0700791 flags = TCP_FLAG_RST;
792 seq = th0->ack_number;
793 ack = 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500794 }
795
Florin Corasdc629cd2017-05-09 00:52:37 -0700796 tcp_reuse_buffer (vm, b0);
Florin Corasf988e692017-11-27 04:34:14 -0500797 tcp_trajectory_add_start (b0, 4);
Florin Corasdc629cd2017-05-09 00:52:37 -0700798 th0 = vlib_buffer_push_tcp_net_order (b0, dst_port, src_port, seq, ack,
799 sizeof (tcp_header_t), flags, 0);
Dave Barach68b0fb02017-02-28 15:15:56 -0500800
Dave Barach68b0fb02017-02-28 15:15:56 -0500801 if (is_ip4)
802 {
Florin Corasdc629cd2017-05-09 00:52:37 -0700803 ih4 = vlib_buffer_push_ip4 (vm, b0, &dst_ip40, &src_ip40,
Florin Corasfdbc3822017-07-27 00:34:12 -0700804 IP_PROTOCOL_TCP, 1);
Dave Barach68b0fb02017-02-28 15:15:56 -0500805 th0->checksum = ip4_tcp_udp_compute_checksum (vm, b0, ih4);
806 }
807 else
808 {
809 int bogus = ~0;
Florin Corasdc629cd2017-05-09 00:52:37 -0700810 ih6 = vlib_buffer_push_ip6 (vm, b0, &dst_ip60, &src_ip60,
811 IP_PROTOCOL_TCP);
Dave Barach68b0fb02017-02-28 15:15:56 -0500812 th0->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b0, ih6, &bogus);
813 ASSERT (!bogus);
814 }
815
816 return 0;
817}
818
819/**
820 * Send reset without reusing existing buffer
Florin Coras1f152cd2017-08-18 19:28:03 -0700821 *
822 * It extracts connection info out of original packet
Dave Barach68b0fb02017-02-28 15:15:56 -0500823 */
824void
Florin Coras1f152cd2017-08-18 19:28:03 -0700825tcp_send_reset_w_pkt (tcp_connection_t * tc, vlib_buffer_t * pkt, u8 is_ip4)
Dave Barach68b0fb02017-02-28 15:15:56 -0500826{
Florin Corasbe72ae62018-11-01 11:23:03 -0700827 tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
828 vlib_main_t *vm = wrk->vm;
Dave Barach68b0fb02017-02-28 15:15:56 -0500829 vlib_buffer_t *b;
Florin Coras56b39f62018-03-27 17:29:32 -0700830 u32 bi, sw_if_index, fib_index;
Dave Barach68b0fb02017-02-28 15:15:56 -0500831 u8 tcp_hdr_len, flags = 0;
832 tcp_header_t *th, *pkt_th;
833 u32 seq, ack;
834 ip4_header_t *ih4, *pkt_ih4;
835 ip6_header_t *ih6, *pkt_ih6;
Florin Coras56b39f62018-03-27 17:29:32 -0700836 fib_protocol_t fib_proto;
Dave Barach68b0fb02017-02-28 15:15:56 -0500837
Florin Corasbe72ae62018-11-01 11:23:03 -0700838 if (PREDICT_FALSE (tcp_get_free_buffer_index (wrk, &bi)))
Florin Coras66b11312017-07-31 17:18:03 -0700839 return;
840
Dave Barach68b0fb02017-02-28 15:15:56 -0500841 b = vlib_get_buffer (vm, bi);
Florin Coras56b39f62018-03-27 17:29:32 -0700842 sw_if_index = vnet_buffer (pkt)->sw_if_index[VLIB_RX];
843 fib_proto = is_ip4 ? FIB_PROTOCOL_IP4 : FIB_PROTOCOL_IP6;
844 fib_index = fib_table_get_index_for_sw_if_index (fib_proto, sw_if_index);
Florin Coras1f152cd2017-08-18 19:28:03 -0700845 tcp_init_buffer (vm, b);
Dave Barach68b0fb02017-02-28 15:15:56 -0500846
847 /* Make and write options */
848 tcp_hdr_len = sizeof (tcp_header_t);
849
850 if (is_ip4)
851 {
852 pkt_ih4 = vlib_buffer_get_current (pkt);
853 pkt_th = ip4_next_header (pkt_ih4);
854 }
855 else
856 {
857 pkt_ih6 = vlib_buffer_get_current (pkt);
858 pkt_th = ip6_next_header (pkt_ih6);
859 }
860
861 if (tcp_ack (pkt_th))
862 {
863 flags = TCP_FLAG_RST;
864 seq = pkt_th->ack_number;
Florin Coras776f3d82018-11-02 08:23:58 -0700865 ack = (tc->state >= TCP_STATE_SYN_RCVD) ? tc->rcv_nxt : 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500866 }
867 else
868 {
869 flags = TCP_FLAG_RST | TCP_FLAG_ACK;
870 seq = 0;
871 ack = clib_host_to_net_u32 (vnet_buffer (pkt)->tcp.seq_end);
872 }
873
874 th = vlib_buffer_push_tcp_net_order (b, pkt_th->dst_port, pkt_th->src_port,
875 seq, ack, tcp_hdr_len, flags, 0);
876
877 /* Swap src and dst ip */
878 if (is_ip4)
879 {
880 ASSERT ((pkt_ih4->ip_version_and_header_length & 0xF0) == 0x40);
881 ih4 = vlib_buffer_push_ip4 (vm, b, &pkt_ih4->dst_address,
Florin Corasfdbc3822017-07-27 00:34:12 -0700882 &pkt_ih4->src_address, IP_PROTOCOL_TCP, 1);
Dave Barach68b0fb02017-02-28 15:15:56 -0500883 th->checksum = ip4_tcp_udp_compute_checksum (vm, b, ih4);
884 }
885 else
886 {
887 int bogus = ~0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500888 ASSERT ((pkt_ih6->ip_version_traffic_class_and_flow_label & 0xF0) ==
889 0x60);
rootc9d1c5b2017-08-15 12:58:31 -0400890 ih6 = vlib_buffer_push_ip6 (vm, b, &pkt_ih6->dst_address,
891 &pkt_ih6->src_address, IP_PROTOCOL_TCP);
Dave Barach68b0fb02017-02-28 15:15:56 -0500892 th->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ih6, &bogus);
893 ASSERT (!bogus);
894 }
895
Florin Corasbe72ae62018-11-01 11:23:03 -0700896 tcp_enqueue_to_ip_lookup_now (wrk, b, bi, is_ip4, fib_index);
Florin Coras6534b7a2017-07-18 05:38:03 -0400897 TCP_EVT_DBG (TCP_EVT_RST_SENT, tc);
Dave Barach68b0fb02017-02-28 15:15:56 -0500898}
899
Florin Coras1f152cd2017-08-18 19:28:03 -0700900/**
901 * Build and set reset packet for connection
902 */
903void
904tcp_send_reset (tcp_connection_t * tc)
905{
Florin Corasbe72ae62018-11-01 11:23:03 -0700906 tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
907 vlib_main_t *vm = wrk->vm;
Florin Coras1f152cd2017-08-18 19:28:03 -0700908 vlib_buffer_t *b;
909 u32 bi;
910 tcp_header_t *th;
911 u16 tcp_hdr_opts_len, advertise_wnd, opts_write_len;
912 u8 flags;
913
Florin Corasbe72ae62018-11-01 11:23:03 -0700914 if (PREDICT_FALSE (tcp_get_free_buffer_index (wrk, &bi)))
Florin Coras1f152cd2017-08-18 19:28:03 -0700915 return;
916 b = vlib_get_buffer (vm, bi);
917 tcp_init_buffer (vm, b);
918
919 tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts, tc->state);
920 tcp_hdr_opts_len = tc->snd_opts_len + sizeof (tcp_header_t);
921 advertise_wnd = tcp_window_to_advertise (tc, TCP_STATE_ESTABLISHED);
922 flags = TCP_FLAG_RST;
923 th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, tc->snd_nxt,
924 tc->rcv_nxt, tcp_hdr_opts_len, flags,
925 advertise_wnd);
926 opts_write_len = tcp_options_write ((u8 *) (th + 1), &tc->snd_opts);
927 ASSERT (opts_write_len == tc->snd_opts_len);
928 vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
Florin Corasf1762d62017-09-24 19:43:08 -0400929 if (tc->c_is_ip4)
930 {
931 ip4_header_t *ih4;
932 ih4 = vlib_buffer_push_ip4 (vm, b, &tc->c_lcl_ip.ip4,
933 &tc->c_rmt_ip.ip4, IP_PROTOCOL_TCP, 0);
934 th->checksum = ip4_tcp_udp_compute_checksum (vm, b, ih4);
935 }
936 else
937 {
938 int bogus = ~0;
939 ip6_header_t *ih6;
940 ih6 = vlib_buffer_push_ip6 (vm, b, &tc->c_lcl_ip.ip6,
941 &tc->c_rmt_ip.ip6, IP_PROTOCOL_TCP);
942 th->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ih6, &bogus);
943 ASSERT (!bogus);
944 }
Florin Corasbe72ae62018-11-01 11:23:03 -0700945 tcp_enqueue_to_ip_lookup_now (wrk, b, bi, tc->c_is_ip4, tc->c_fib_index);
Florin Corasf1762d62017-09-24 19:43:08 -0400946 TCP_EVT_DBG (TCP_EVT_RST_SENT, tc);
Florin Coras1f152cd2017-08-18 19:28:03 -0700947}
948
Florin Coras0dbd5172018-06-25 16:19:34 -0700949static void
Florin Corasbe72ae62018-11-01 11:23:03 -0700950tcp_push_ip_hdr (tcp_worker_ctx_t * wrk, tcp_connection_t * tc,
951 vlib_buffer_t * b)
Dave Barach68b0fb02017-02-28 15:15:56 -0500952{
953 tcp_header_t *th = vlib_buffer_get_current (b);
Florin Corasbe72ae62018-11-01 11:23:03 -0700954 vlib_main_t *vm = wrk->vm;
Dave Barach68b0fb02017-02-28 15:15:56 -0500955 if (tc->c_is_ip4)
956 {
957 ip4_header_t *ih;
Dave Barach2c25a622017-06-26 11:35:07 -0400958 ih = vlib_buffer_push_ip4 (vm, b, &tc->c_lcl_ip4,
Florin Corasfdbc3822017-07-27 00:34:12 -0700959 &tc->c_rmt_ip4, IP_PROTOCOL_TCP, 1);
Dave Barach2c25a622017-06-26 11:35:07 -0400960 th->checksum = ip4_tcp_udp_compute_checksum (vm, b, ih);
Dave Barach68b0fb02017-02-28 15:15:56 -0500961 }
962 else
963 {
964 ip6_header_t *ih;
965 int bogus = ~0;
966
Dave Barach2c25a622017-06-26 11:35:07 -0400967 ih = vlib_buffer_push_ip6 (vm, b, &tc->c_lcl_ip6,
Dave Barach68b0fb02017-02-28 15:15:56 -0500968 &tc->c_rmt_ip6, IP_PROTOCOL_TCP);
Dave Barach2c25a622017-06-26 11:35:07 -0400969 th->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ih, &bogus);
Dave Barach68b0fb02017-02-28 15:15:56 -0500970 ASSERT (!bogus);
971 }
972}
973
974/**
975 * Send SYN
976 *
977 * Builds a SYN packet for a half-open connection and sends it to ipx_lookup.
978 * The packet is not forwarded through tcpx_output to avoid doing lookups
979 * in the half_open pool.
980 */
981void
982tcp_send_syn (tcp_connection_t * tc)
983{
Florin Corasbe72ae62018-11-01 11:23:03 -0700984 tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
985 vlib_main_t *vm = wrk->vm;
Dave Barach68b0fb02017-02-28 15:15:56 -0500986 vlib_buffer_t *b;
987 u32 bi;
Dave Barach68b0fb02017-02-28 15:15:56 -0500988
Florin Corasf988e692017-11-27 04:34:14 -0500989 /*
990 * Setup retransmit and establish timers before requesting buffer
991 * such that we can return if we've ran out.
992 */
993 tcp_timer_set (tc, TCP_TIMER_ESTABLISH, TCP_ESTABLISH_TIME);
994 tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN,
995 tc->rto * TCP_TO_TIMER_TICK);
996
Florin Corasbe72ae62018-11-01 11:23:03 -0700997 if (PREDICT_FALSE (tcp_get_free_buffer_index (wrk, &bi)))
Florin Coras66b11312017-07-31 17:18:03 -0700998 return;
999
Dave Barach68b0fb02017-02-28 15:15:56 -05001000 b = vlib_get_buffer (vm, bi);
Florin Coras1f152cd2017-08-18 19:28:03 -07001001 tcp_init_buffer (vm, b);
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001002 tcp_make_syn (tc, b);
Dave Barach68b0fb02017-02-28 15:15:56 -05001003
1004 /* Measure RTT with this */
1005 tc->rtt_ts = tcp_time_now ();
1006 tc->rtt_seq = tc->snd_nxt;
Dave Barach68b0fb02017-02-28 15:15:56 -05001007 tc->rto_boff = 0;
1008
Florin Corasbe72ae62018-11-01 11:23:03 -07001009 tcp_push_ip_hdr (wrk, tc, b);
1010 tcp_enqueue_to_ip_lookup (wrk, b, bi, tc->c_is_ip4, tc->c_fib_index);
Florin Coras6534b7a2017-07-18 05:38:03 -04001011 TCP_EVT_DBG (TCP_EVT_SYN_SENT, tc);
Dave Barach68b0fb02017-02-28 15:15:56 -05001012}
1013
Florin Coras66b11312017-07-31 17:18:03 -07001014/**
1015 * Flush tx frame populated by retransmits and timer pops
1016 */
1017void
Florin Corasbe72ae62018-11-01 11:23:03 -07001018tcp_flush_frame_to_output (tcp_worker_ctx_t * wrk, u8 is_ip4)
Florin Coras66b11312017-07-31 17:18:03 -07001019{
Florin Corasbe72ae62018-11-01 11:23:03 -07001020 if (wrk->tx_frames[!is_ip4])
Florin Coras66b11312017-07-31 17:18:03 -07001021 {
1022 u32 next_index;
1023 next_index = is_ip4 ? tcp4_output_node.index : tcp6_output_node.index;
Florin Corasbe72ae62018-11-01 11:23:03 -07001024 vlib_put_frame_to_node (wrk->vm, next_index, wrk->tx_frames[!is_ip4]);
1025 wrk->tx_frames[!is_ip4] = 0;
Florin Coras66b11312017-07-31 17:18:03 -07001026 }
1027}
1028
1029/**
Florin Coras9d063042017-09-14 03:08:00 -04001030 * Flush ip lookup tx frames populated by timer pops
1031 */
Florin Coras0dbd5172018-06-25 16:19:34 -07001032static void
Florin Corasbe72ae62018-11-01 11:23:03 -07001033tcp_flush_frame_to_ip_lookup (tcp_worker_ctx_t * wrk, u8 is_ip4)
Florin Coras9d063042017-09-14 03:08:00 -04001034{
Florin Corasbe72ae62018-11-01 11:23:03 -07001035 if (wrk->ip_lookup_tx_frames[!is_ip4])
Florin Coras9d063042017-09-14 03:08:00 -04001036 {
1037 u32 next_index;
1038 next_index = is_ip4 ? ip4_lookup_node.index : ip6_lookup_node.index;
Florin Corasbe72ae62018-11-01 11:23:03 -07001039 vlib_put_frame_to_node (wrk->vm, next_index,
1040 wrk->ip_lookup_tx_frames[!is_ip4]);
1041 wrk->ip_lookup_tx_frames[!is_ip4] = 0;
Florin Coras9d063042017-09-14 03:08:00 -04001042 }
1043}
1044
1045/**
1046 * Flush v4 and v6 tcp and ip-lookup tx frames for thread index
Florin Coras66b11312017-07-31 17:18:03 -07001047 */
1048void
Florin Corasbe72ae62018-11-01 11:23:03 -07001049tcp_flush_frames_to_output (tcp_worker_ctx_t * wrk)
Florin Coras66b11312017-07-31 17:18:03 -07001050{
Florin Corasbe72ae62018-11-01 11:23:03 -07001051 tcp_flush_frame_to_output (wrk, 1);
1052 tcp_flush_frame_to_output (wrk, 0);
1053 tcp_flush_frame_to_ip_lookup (wrk, 1);
1054 tcp_flush_frame_to_ip_lookup (wrk, 0);
Dave Barach68b0fb02017-02-28 15:15:56 -05001055}
1056
1057/**
1058 * Send FIN
1059 */
1060void
1061tcp_send_fin (tcp_connection_t * tc)
1062{
Florin Corasbe72ae62018-11-01 11:23:03 -07001063 tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
1064 vlib_main_t *vm = wrk->vm;
Florin Coras9d063042017-09-14 03:08:00 -04001065 vlib_buffer_t *b;
1066 u32 bi;
1067 u8 fin_snt = 0;
1068
Florin Corasc977e7c2018-10-16 20:30:31 -07001069 fin_snt = tc->flags & TCP_CONN_FINSNT;
1070 if (fin_snt)
1071 tc->snd_nxt = tc->snd_una;
1072
Florin Corasbe72ae62018-11-01 11:23:03 -07001073 if (PREDICT_FALSE (tcp_get_free_buffer_index (wrk, &bi)))
Florin Coraseb97e5f2018-10-15 21:35:42 -07001074 {
1075 /* Out of buffers so program fin retransmit ASAP */
1076 tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1);
Florin Corasc977e7c2018-10-16 20:30:31 -07001077 goto post_enqueue;
Florin Coraseb97e5f2018-10-15 21:35:42 -07001078 }
1079
Florin Corasc977e7c2018-10-16 20:30:31 -07001080 tcp_retransmit_timer_force_update (tc);
Dave Barach68b0fb02017-02-28 15:15:56 -05001081 b = vlib_get_buffer (vm, bi);
Florin Coras24793e32018-05-09 11:34:25 -07001082 tcp_init_buffer (vm, b);
Florin Corasd79b41e2017-03-04 05:37:52 -08001083 tcp_make_fin (tc, b);
Florin Corasbe72ae62018-11-01 11:23:03 -07001084 tcp_enqueue_to_output_now (wrk, b, bi, tc->c_is_ip4);
Florin Corasc977e7c2018-10-16 20:30:31 -07001085 TCP_EVT_DBG (TCP_EVT_FIN_SENT, tc);
1086
1087post_enqueue:
Florin Coras9d063042017-09-14 03:08:00 -04001088 if (!fin_snt)
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001089 {
1090 tc->flags |= TCP_CONN_FINSNT;
1091 tc->flags &= ~TCP_CONN_FINPNDG;
Florin Coras9d063042017-09-14 03:08:00 -04001092 /* Account for the FIN */
1093 tc->snd_una_max += 1;
1094 tc->snd_nxt = tc->snd_una_max;
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001095 }
Florin Corasa096f2d2017-09-28 23:49:42 -04001096 else
1097 {
1098 tc->snd_nxt = tc->snd_una_max;
1099 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001100}
1101
1102always_inline u8
Florin Corasb2215d62017-08-01 16:56:58 -07001103tcp_make_state_flags (tcp_connection_t * tc, tcp_state_t next_state)
Dave Barach68b0fb02017-02-28 15:15:56 -05001104{
1105 switch (next_state)
1106 {
1107 case TCP_STATE_ESTABLISHED:
Florin Coras00dfe542018-06-11 13:15:11 -07001108 case TCP_STATE_CLOSE_WAIT:
Dave Barach68b0fb02017-02-28 15:15:56 -05001109 return TCP_FLAG_ACK;
1110 case TCP_STATE_SYN_RCVD:
1111 return TCP_FLAG_SYN | TCP_FLAG_ACK;
1112 case TCP_STATE_SYN_SENT:
1113 return TCP_FLAG_SYN;
1114 case TCP_STATE_LAST_ACK:
1115 case TCP_STATE_FIN_WAIT_1:
Florin Corasb2215d62017-08-01 16:56:58 -07001116 if (tc->snd_nxt + 1 < tc->snd_una_max)
1117 return TCP_FLAG_ACK;
1118 else
1119 return TCP_FLAG_FIN;
Dave Barach68b0fb02017-02-28 15:15:56 -05001120 default:
1121 clib_warning ("Shouldn't be here!");
1122 }
1123 return 0;
1124}
1125
1126/**
1127 * Push TCP header and update connection variables
1128 */
Florin Coras0dbd5172018-06-25 16:19:34 -07001129always_inline void
Dave Barach68b0fb02017-02-28 15:15:56 -05001130tcp_push_hdr_i (tcp_connection_t * tc, vlib_buffer_t * b,
Florin Corasb26743d2018-06-26 09:31:04 -07001131 tcp_state_t next_state, u8 compute_opts, u8 maybe_burst)
Dave Barach68b0fb02017-02-28 15:15:56 -05001132{
1133 u32 advertise_wnd, data_len;
Florin Corasb26743d2018-06-26 09:31:04 -07001134 u8 tcp_hdr_opts_len, flags;
1135 tcp_main_t *tm = &tcp_main;
Dave Barach68b0fb02017-02-28 15:15:56 -05001136 tcp_header_t *th;
1137
Florin Corasb26743d2018-06-26 09:31:04 -07001138 data_len = b->current_length;
1139 if (PREDICT_FALSE (b->flags & VLIB_BUFFER_NEXT_PRESENT))
1140 data_len += b->total_length_not_including_first_buffer;
1141
Dave Barach68b0fb02017-02-28 15:15:56 -05001142 vnet_buffer (b)->tcp.flags = 0;
Florin Corasb26743d2018-06-26 09:31:04 -07001143 vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
Dave Barach68b0fb02017-02-28 15:15:56 -05001144
Florin Corasc8343412017-05-04 14:25:50 -07001145 if (compute_opts)
1146 tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts, tc->state);
1147
Florin Corasc8343412017-05-04 14:25:50 -07001148 tcp_hdr_opts_len = tc->snd_opts_len + sizeof (tcp_header_t);
Florin Corasb26743d2018-06-26 09:31:04 -07001149
1150 if (maybe_burst)
1151 advertise_wnd = tc->rcv_wnd >> tc->rcv_wscale;
1152 else
1153 advertise_wnd = tcp_window_to_advertise (tc, next_state);
1154
Florin Corasb2215d62017-08-01 16:56:58 -07001155 flags = tcp_make_state_flags (tc, next_state);
Dave Barach68b0fb02017-02-28 15:15:56 -05001156
Dave Barach68b0fb02017-02-28 15:15:56 -05001157 th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, tc->snd_nxt,
1158 tc->rcv_nxt, tcp_hdr_opts_len, flags,
1159 advertise_wnd);
Dave Barach68b0fb02017-02-28 15:15:56 -05001160
Florin Corasb26743d2018-06-26 09:31:04 -07001161 if (maybe_burst)
1162 {
1163 clib_memcpy ((u8 *) (th + 1),
1164 tm->wrk_ctx[tc->c_thread_index].cached_opts,
1165 tc->snd_opts_len);
1166 }
1167 else
1168 {
1169 u8 len = tcp_options_write ((u8 *) (th + 1), &tc->snd_opts);
1170 ASSERT (len == tc->snd_opts_len);
1171 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001172
Florin Coras93992a92017-05-24 18:03:56 -07001173 /*
1174 * Update connection variables
1175 */
1176
Dave Barach68b0fb02017-02-28 15:15:56 -05001177 tc->snd_nxt += data_len;
Florin Corasc28764f2017-04-26 00:08:42 -07001178 tc->rcv_las = tc->rcv_nxt;
Florin Coras3e350af2017-03-30 02:54:28 -07001179
Florin Corase69f4952017-03-07 10:06:24 -08001180 TCP_EVT_DBG (TCP_EVT_PKTIZE, tc);
Dave Barach68b0fb02017-02-28 15:15:56 -05001181}
1182
Florin Coras0dbd5172018-06-25 16:19:34 -07001183u32
1184tcp_push_header (tcp_connection_t * tc, vlib_buffer_t * b)
1185{
Florin Corasb26743d2018-06-26 09:31:04 -07001186 tcp_push_hdr_i (tc, b, TCP_STATE_ESTABLISHED, /* compute opts */ 0,
1187 /* burst */ 1);
Florin Coras0dbd5172018-06-25 16:19:34 -07001188 tc->snd_una_max = tc->snd_nxt;
Florin Coras3ec66b02018-08-23 16:27:05 -07001189 ASSERT (seq_leq (tc->snd_una_max, tc->snd_una + tc->snd_wnd
1190 + tcp_fastrecovery_sent_1_smss (tc) * tc->snd_mss));
Florin Coras0dbd5172018-06-25 16:19:34 -07001191 tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una);
1192 /* If not tracking an ACK, start tracking */
1193 if (tc->rtt_ts == 0 && !tcp_in_cong_recovery (tc))
1194 {
Florin Corasd67f1122018-05-21 17:47:40 -07001195 tc->rtt_ts = tcp_time_now_us (tc->c_thread_index);
Florin Coras0dbd5172018-06-25 16:19:34 -07001196 tc->rtt_seq = tc->snd_nxt;
1197 }
1198 if (PREDICT_FALSE (!tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT)))
1199 {
1200 tcp_retransmit_timer_set (tc);
1201 tc->rto_boff = 0;
1202 }
1203 tcp_trajectory_add_start (b, 3);
1204 return 0;
1205}
1206
Dave Barach68b0fb02017-02-28 15:15:56 -05001207void
Florin Coras6792ec02017-03-13 03:49:51 -07001208tcp_send_ack (tcp_connection_t * tc)
Dave Barach68b0fb02017-02-28 15:15:56 -05001209{
Florin Corasbe72ae62018-11-01 11:23:03 -07001210 tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
1211 vlib_main_t *vm = wrk->vm;
Dave Barach68b0fb02017-02-28 15:15:56 -05001212 vlib_buffer_t *b;
1213 u32 bi;
1214
Dave Barach68b0fb02017-02-28 15:15:56 -05001215 /* Get buffer */
Florin Corasbe72ae62018-11-01 11:23:03 -07001216 if (PREDICT_FALSE (tcp_get_free_buffer_index (wrk, &bi)))
Florin Coras66b11312017-07-31 17:18:03 -07001217 return;
Dave Barach68b0fb02017-02-28 15:15:56 -05001218 b = vlib_get_buffer (vm, bi);
Florin Coras24793e32018-05-09 11:34:25 -07001219 tcp_init_buffer (vm, b);
Dave Barach68b0fb02017-02-28 15:15:56 -05001220
1221 /* Fill in the ACK */
1222 tcp_make_ack (tc, b);
Florin Corasbe72ae62018-11-01 11:23:03 -07001223 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Dave Barach68b0fb02017-02-28 15:15:56 -05001224}
1225
Florin Corasb2215d62017-08-01 16:56:58 -07001226/**
1227 * Delayed ack timer handler
1228 *
1229 * Sends delayed ACK when timer expires
1230 */
Florin Coras6792ec02017-03-13 03:49:51 -07001231void
1232tcp_timer_delack_handler (u32 index)
1233{
Damjan Marion586afd72017-04-05 19:18:20 +02001234 u32 thread_index = vlib_get_thread_index ();
Florin Coras6792ec02017-03-13 03:49:51 -07001235 tcp_connection_t *tc;
1236
1237 tc = tcp_connection_get (index, thread_index);
1238 tc->timers[TCP_TIMER_DELACK] = TCP_TIMER_HANDLE_INVALID;
Florin Coras6792ec02017-03-13 03:49:51 -07001239 tcp_send_ack (tc);
1240}
1241
Florin Corasb2215d62017-08-01 16:56:58 -07001242/**
Florin Coras36ee9f12018-11-02 12:52:10 -07001243 * Allocate a new buffer and build a new tcp segment
Dave Barach68b0fb02017-02-28 15:15:56 -05001244 *
Florin Coras36ee9f12018-11-02 12:52:10 -07001245 * @param wrk tcp worker
1246 * @param tc connection for which the segment will be allocated
1247 * @param offset offset of the first byte in the tx fifo
1248 * @param max_deq_byte segment size
1249 * @param[out] b pointer to buffer allocated
1250 *
1251 * @return the number of bytes in the segment or 0 if buffer cannot be
1252 * allocated or no data available
Florin Coras93992a92017-05-24 18:03:56 -07001253 */
Florin Coras36ee9f12018-11-02 12:52:10 -07001254static int
1255tcp_prepare_segment (tcp_worker_ctx_t * wrk, tcp_connection_t * tc,
1256 u32 offset, u32 max_deq_bytes, vlib_buffer_t ** b)
Dave Barach68b0fb02017-02-28 15:15:56 -05001257{
Florin Corasbe72ae62018-11-01 11:23:03 -07001258 u32 bytes_per_buffer = vnet_get_tcp_main ()->bytes_per_buffer;
Florin Coras36ee9f12018-11-02 12:52:10 -07001259 u32 bi, seg_size;
Florin Corasbe72ae62018-11-01 11:23:03 -07001260 vlib_main_t *vm = wrk->vm;
Florin Coras93992a92017-05-24 18:03:56 -07001261 int n_bytes = 0;
Florin Corase87216f2017-08-17 16:59:22 -07001262 u8 *data;
Dave Barach68b0fb02017-02-28 15:15:56 -05001263
Florin Coras1f152cd2017-08-18 19:28:03 -07001264 seg_size = max_deq_bytes + MAX_HDRS_LEN;
1265
Florin Corasb2215d62017-08-01 16:56:58 -07001266 /*
1267 * Prepare options
1268 */
Florin Corasc8343412017-05-04 14:25:50 -07001269 tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts, tc->state);
1270
Florin Corasb2215d62017-08-01 16:56:58 -07001271 /*
1272 * Allocate and fill in buffer(s)
1273 */
Dave Barach68b0fb02017-02-28 15:15:56 -05001274
Florin Corasb2215d62017-08-01 16:56:58 -07001275 /* Easy case, buffer size greater than mss */
Florin Corasbe72ae62018-11-01 11:23:03 -07001276 if (PREDICT_TRUE (seg_size <= bytes_per_buffer))
Florin Corasb2215d62017-08-01 16:56:58 -07001277 {
Florin Corasbe72ae62018-11-01 11:23:03 -07001278 if (PREDICT_FALSE (tcp_get_free_buffer_index (wrk, &bi)))
Florin Coras24793e32018-05-09 11:34:25 -07001279 return 0;
1280 *b = vlib_get_buffer (vm, bi);
1281 data = tcp_init_buffer (vm, *b);
Florin Corase87216f2017-08-17 16:59:22 -07001282 n_bytes = stream_session_peek_bytes (&tc->connection, data, offset,
1283 max_deq_bytes);
Florin Corasb2215d62017-08-01 16:56:58 -07001284 ASSERT (n_bytes == max_deq_bytes);
1285 b[0]->current_length = n_bytes;
Florin Corasb26743d2018-06-26 09:31:04 -07001286 tcp_push_hdr_i (tc, *b, tc->state, /* compute opts */ 0, /* burst */ 0);
Florin Coras208daa12018-07-02 01:30:51 -07001287 if (seq_gt (tc->snd_nxt, tc->snd_una_max))
1288 tc->snd_una_max = tc->snd_nxt;
Florin Corasb2215d62017-08-01 16:56:58 -07001289 }
1290 /* Split mss into multiple buffers */
1291 else
1292 {
1293 u32 chain_bi = ~0, n_bufs_per_seg;
Florin Corasb2215d62017-08-01 16:56:58 -07001294 u16 n_peeked, len_to_deq, available_bufs;
1295 vlib_buffer_t *chain_b, *prev_b;
Florin Corasb2215d62017-08-01 16:56:58 -07001296 int i;
1297
Florin Corasb2215d62017-08-01 16:56:58 -07001298 /* Make sure we have enough buffers */
Florin Corasbe72ae62018-11-01 11:23:03 -07001299 n_bufs_per_seg = ceil ((double) seg_size / bytes_per_buffer);
1300 available_bufs = vec_len (wrk->tx_buffers);
Florin Corasb2215d62017-08-01 16:56:58 -07001301 if (n_bufs_per_seg > available_bufs)
1302 {
Florin Corasbe72ae62018-11-01 11:23:03 -07001303 tcp_alloc_tx_buffers (wrk, &available_bufs, VLIB_FRAME_SIZE);
Florin Coras24793e32018-05-09 11:34:25 -07001304 if (n_bufs_per_seg > available_bufs)
Florin Corasb2215d62017-08-01 16:56:58 -07001305 {
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001306 *b = 0;
Florin Corasb2215d62017-08-01 16:56:58 -07001307 return 0;
1308 }
1309 }
1310
Florin Coras9ece3c02018-11-05 11:06:53 -08001311 (void) tcp_get_free_buffer_index (wrk, &bi);
Florin Coras24793e32018-05-09 11:34:25 -07001312 ASSERT (bi != (u32) ~ 0);
1313 *b = vlib_get_buffer (vm, bi);
1314 data = tcp_init_buffer (vm, *b);
Florin Corase87216f2017-08-17 16:59:22 -07001315 n_bytes = stream_session_peek_bytes (&tc->connection, data, offset,
Florin Corasbe72ae62018-11-01 11:23:03 -07001316 bytes_per_buffer - MAX_HDRS_LEN);
Florin Corasb2215d62017-08-01 16:56:58 -07001317 b[0]->current_length = n_bytes;
1318 b[0]->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
1319 b[0]->total_length_not_including_first_buffer = 0;
Florin Corasb2215d62017-08-01 16:56:58 -07001320 max_deq_bytes -= n_bytes;
1321
1322 chain_b = *b;
1323 for (i = 1; i < n_bufs_per_seg; i++)
1324 {
1325 prev_b = chain_b;
Florin Corasbe72ae62018-11-01 11:23:03 -07001326 len_to_deq = clib_min (max_deq_bytes, bytes_per_buffer);
1327 tcp_get_free_buffer_index (wrk, &chain_bi);
Florin Corasb2215d62017-08-01 16:56:58 -07001328 ASSERT (chain_bi != (u32) ~ 0);
1329 chain_b = vlib_get_buffer (vm, chain_bi);
1330 chain_b->current_data = 0;
Florin Corase87216f2017-08-17 16:59:22 -07001331 data = vlib_buffer_get_current (chain_b);
1332 n_peeked = stream_session_peek_bytes (&tc->connection, data,
Florin Coras1f152cd2017-08-18 19:28:03 -07001333 offset + n_bytes, len_to_deq);
Florin Corasb2215d62017-08-01 16:56:58 -07001334 ASSERT (n_peeked == len_to_deq);
Florin Coras1f152cd2017-08-18 19:28:03 -07001335 n_bytes += n_peeked;
Florin Corasb2215d62017-08-01 16:56:58 -07001336 chain_b->current_length = n_peeked;
Florin Coras1f152cd2017-08-18 19:28:03 -07001337 chain_b->next_buffer = 0;
Florin Corasb2215d62017-08-01 16:56:58 -07001338
1339 /* update previous buffer */
1340 prev_b->next_buffer = chain_bi;
1341 prev_b->flags |= VLIB_BUFFER_NEXT_PRESENT;
1342
Florin Corasb2215d62017-08-01 16:56:58 -07001343 max_deq_bytes -= n_peeked;
Florin Coras1f152cd2017-08-18 19:28:03 -07001344 b[0]->total_length_not_including_first_buffer += n_peeked;
Florin Corasb2215d62017-08-01 16:56:58 -07001345 }
Florin Coras1f152cd2017-08-18 19:28:03 -07001346
Florin Corasb26743d2018-06-26 09:31:04 -07001347 tcp_push_hdr_i (tc, *b, tc->state, /* compute opts */ 0, /* burst */ 0);
Florin Coras208daa12018-07-02 01:30:51 -07001348 if (seq_gt (tc->snd_nxt, tc->snd_una_max))
1349 tc->snd_una_max = tc->snd_nxt;
Florin Corasb2215d62017-08-01 16:56:58 -07001350 }
1351
Florin Coras93992a92017-05-24 18:03:56 -07001352 ASSERT (n_bytes > 0);
Florin Corasbe72ae62018-11-01 11:23:03 -07001353 ASSERT (((*b)->current_data + (*b)->current_length) <= bytes_per_buffer);
Florin Corasbb292f42017-05-19 09:49:19 -07001354
Florin Coras36ee9f12018-11-02 12:52:10 -07001355 return n_bytes;
1356}
1357
1358/**
1359 * Build a retransmit segment
1360 *
1361 * @return the number of bytes in the segment or 0 if there's nothing to
1362 * retransmit
1363 */
1364static u32
1365tcp_prepare_retransmit_segment (tcp_worker_ctx_t * wrk,
1366 tcp_connection_t * tc, u32 offset,
1367 u32 max_deq_bytes, vlib_buffer_t ** b)
1368{
1369 u32 start, available_bytes;
1370 int n_bytes = 0;
1371
1372 ASSERT (tc->state >= TCP_STATE_ESTABLISHED);
1373 ASSERT (max_deq_bytes != 0);
1374
1375 /*
1376 * Make sure we can retransmit something
1377 */
1378 available_bytes = session_tx_fifo_max_dequeue (&tc->connection);
1379 ASSERT (available_bytes >= offset);
1380 available_bytes -= offset;
1381 if (!available_bytes)
1382 return 0;
1383
1384 max_deq_bytes = clib_min (tc->snd_mss, max_deq_bytes);
1385 max_deq_bytes = clib_min (available_bytes, max_deq_bytes);
1386
1387 /* Start is beyond snd_congestion */
1388 start = tc->snd_una + offset;
1389 if (seq_geq (start, tc->snd_congestion))
1390 goto done;
1391
1392 /* Don't overshoot snd_congestion */
1393 if (seq_gt (start + max_deq_bytes, tc->snd_congestion))
1394 {
1395 max_deq_bytes = tc->snd_congestion - start;
1396 if (max_deq_bytes == 0)
1397 goto done;
1398 }
1399
1400 n_bytes = tcp_prepare_segment (wrk, tc, offset, max_deq_bytes, b);
1401 if (!n_bytes)
1402 return 0;
1403
Florin Coras93992a92017-05-24 18:03:56 -07001404 if (tcp_in_fastrecovery (tc))
1405 tc->snd_rxt_bytes += n_bytes;
Dave Barach68b0fb02017-02-28 15:15:56 -05001406
Florin Coras6792ec02017-03-13 03:49:51 -07001407done:
1408 TCP_EVT_DBG (TCP_EVT_CC_RTX, tc, offset, n_bytes);
Dave Barach68b0fb02017-02-28 15:15:56 -05001409 return n_bytes;
1410}
1411
Florin Coras6792ec02017-03-13 03:49:51 -07001412/**
1413 * Reset congestion control, switch cwnd to loss window and try again.
1414 */
1415static void
Florin Corasca1c8f32018-05-23 21:01:30 -07001416tcp_rxt_timeout_cc (tcp_connection_t * tc)
Florin Coras6792ec02017-03-13 03:49:51 -07001417{
Florin Corasca1c8f32018-05-23 21:01:30 -07001418 TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 6);
Florin Coras93992a92017-05-24 18:03:56 -07001419 tc->prev_ssthresh = tc->ssthresh;
1420 tc->prev_cwnd = tc->cwnd;
1421
Florin Coras6792ec02017-03-13 03:49:51 -07001422 /* Cleanly recover cc (also clears up fast retransmit) */
1423 if (tcp_in_fastrecovery (tc))
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001424 {
1425 /* TODO be less aggressive about this */
1426 scoreboard_clear (&tc->sack_sb);
1427 tcp_cc_fastrecovery_exit (tc);
1428 }
Florin Coras6792ec02017-03-13 03:49:51 -07001429
1430 /* Start again from the beginning */
Florin Corasd2aab832018-05-22 11:39:59 -07001431 tc->cc_algo->congestion (tc);
Florin Coras6792ec02017-03-13 03:49:51 -07001432 tc->cwnd = tcp_loss_wnd (tc);
1433 tc->snd_congestion = tc->snd_una_max;
Florin Corasf1762d62017-09-24 19:43:08 -04001434 tc->rtt_ts = 0;
Florin Corasd2aab832018-05-22 11:39:59 -07001435 tc->cwnd_acc_bytes = 0;
Florin Corasc44a5582018-11-01 16:30:54 -07001436 tcp_connection_tx_pacer_reset (tc, tc->cwnd, 2 * tc->snd_mss);
Florin Coras3af90fc2017-05-03 21:09:42 -07001437 tcp_recovery_on (tc);
Florin Coras6792ec02017-03-13 03:49:51 -07001438}
1439
Florin Coras0dbd5172018-06-25 16:19:34 -07001440static inline void
Dave Barach68b0fb02017-02-28 15:15:56 -05001441tcp_timer_retransmit_handler_i (u32 index, u8 is_syn)
1442{
Damjan Marion586afd72017-04-05 19:18:20 +02001443 u32 thread_index = vlib_get_thread_index ();
Florin Corasbe72ae62018-11-01 11:23:03 -07001444 tcp_worker_ctx_t *wrk = tcp_get_worker (thread_index);
1445 vlib_main_t *vm = wrk->vm;
Dave Barach68b0fb02017-02-28 15:15:56 -05001446 tcp_connection_t *tc;
Florin Corasb2215d62017-08-01 16:56:58 -07001447 vlib_buffer_t *b = 0;
Florin Coras3af90fc2017-05-03 21:09:42 -07001448 u32 bi, n_bytes;
Dave Barach68b0fb02017-02-28 15:15:56 -05001449
1450 if (is_syn)
1451 {
1452 tc = tcp_half_open_connection_get (index);
Dave Barachb7f1faa2017-08-29 11:43:37 -04001453 /* Note: the connection may have transitioned to ESTABLISHED... */
1454 if (PREDICT_FALSE (tc == 0))
1455 return;
Florin Coras68810622017-07-24 17:40:28 -07001456 tc->timers[TCP_TIMER_RETRANSMIT_SYN] = TCP_TIMER_HANDLE_INVALID;
Dave Barach68b0fb02017-02-28 15:15:56 -05001457 }
1458 else
1459 {
1460 tc = tcp_connection_get (index, thread_index);
Dave Barachb7f1faa2017-08-29 11:43:37 -04001461 /* Note: the connection may have been closed and pool_put */
1462 if (PREDICT_FALSE (tc == 0))
1463 return;
Florin Coras68810622017-07-24 17:40:28 -07001464 tc->timers[TCP_TIMER_RETRANSMIT] = TCP_TIMER_HANDLE_INVALID;
Dave Barach68b0fb02017-02-28 15:15:56 -05001465 }
1466
Florin Corase04c2992017-03-01 08:17:34 -08001467 if (tc->state >= TCP_STATE_ESTABLISHED)
Dave Barach68b0fb02017-02-28 15:15:56 -05001468 {
Florin Corase55a6d72018-10-31 23:09:22 -07001469 TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 2);
1470
Florin Coras93992a92017-05-24 18:03:56 -07001471 /* Lost FIN, retransmit and return */
Florin Coras93e65802017-11-29 00:07:11 -05001472 if (tcp_is_lost_fin (tc))
Florin Coras93992a92017-05-24 18:03:56 -07001473 {
1474 tcp_send_fin (tc);
Florin Corasf988e692017-11-27 04:34:14 -05001475 tc->rto_boff += 1;
1476 tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
Florin Coras93992a92017-05-24 18:03:56 -07001477 return;
1478 }
1479
Florin Coras3ec66b02018-08-23 16:27:05 -07001480 /* Shouldn't be here. This condition is tricky because it has to take
1481 * into account boff > 0 due to persist timeout. */
Florin Coras887b7ba2018-06-09 06:49:59 -07001482 if ((tc->rto_boff == 0 && tc->snd_una == tc->snd_una_max)
Florin Coras3ec66b02018-08-23 16:27:05 -07001483 || (tc->rto_boff > 0 && seq_geq (tc->snd_una, tc->snd_congestion)
1484 && !tcp_flight_size (tc)))
Florin Corasa096f2d2017-09-28 23:49:42 -04001485 {
Florin Coras3ec66b02018-08-23 16:27:05 -07001486 ASSERT (!tcp_in_recovery (tc));
1487 tc->rto_boff = 0;
Florin Corasa096f2d2017-09-28 23:49:42 -04001488 return;
1489 }
1490
Florin Coras3ec66b02018-08-23 16:27:05 -07001491 /* We're not in recovery so make sure rto_boff is 0. Can be non 0 due
1492 * to persist timer timeout */
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001493 if (!tcp_in_recovery (tc) && tc->rto_boff > 0)
1494 {
1495 tc->rto_boff = 0;
1496 tcp_update_rto (tc);
1497 }
1498
1499 /* Increment RTO backoff (also equal to number of retries) and go back
1500 * to first un-acked byte */
1501 tc->rto_boff += 1;
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001502
Florin Coras6792ec02017-03-13 03:49:51 -07001503 /* First retransmit timeout */
1504 if (tc->rto_boff == 1)
Florin Corasca1c8f32018-05-23 21:01:30 -07001505 tcp_rxt_timeout_cc (tc);
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001506 else
1507 scoreboard_clear (&tc->sack_sb);
Dave Barach68b0fb02017-02-28 15:15:56 -05001508
Florin Coras3ec66b02018-08-23 16:27:05 -07001509 /* If we've sent beyond snd_congestion, update it */
1510 if (seq_gt (tc->snd_una_max, tc->snd_congestion))
1511 tc->snd_congestion = tc->snd_una_max;
1512
Florin Corasd2aab832018-05-22 11:39:59 -07001513 tc->snd_una_max = tc->snd_nxt = tc->snd_una;
Dave Barach68b0fb02017-02-28 15:15:56 -05001514 tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
1515
Florin Coras3ec66b02018-08-23 16:27:05 -07001516 /* Send one segment. Note that n_bytes may be zero due to buffer
1517 * shortfall */
Florin Corasbe72ae62018-11-01 11:23:03 -07001518 n_bytes = tcp_prepare_retransmit_segment (wrk, tc, 0, tc->snd_mss, &b);
Dave Barachb7f1faa2017-08-29 11:43:37 -04001519
Florin Coras3af90fc2017-05-03 21:09:42 -07001520 if (n_bytes == 0)
Florin Corase04c2992017-03-01 08:17:34 -08001521 {
Florin Coras3ec66b02018-08-23 16:27:05 -07001522 tcp_retransmit_timer_force_update (tc);
Florin Corase04c2992017-03-01 08:17:34 -08001523 return;
1524 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001525
Dave Barachb7f1faa2017-08-29 11:43:37 -04001526 bi = vlib_get_buffer_index (vm, b);
1527
Florin Coras93992a92017-05-24 18:03:56 -07001528 /* For first retransmit, record timestamp (Eifel detection RFC3522) */
1529 if (tc->rto_boff == 1)
Florin Corasbe72ae62018-11-01 11:23:03 -07001530 tc->snd_rxt_ts = tcp_time_now_w_thread (tc->c_thread_index);
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001531
Florin Corasbe72ae62018-11-01 11:23:03 -07001532 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Florin Coras3ec66b02018-08-23 16:27:05 -07001533 tcp_retransmit_timer_force_update (tc);
Florin Coras93992a92017-05-24 18:03:56 -07001534 }
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001535 /* Retransmit for SYN */
1536 else if (tc->state == TCP_STATE_SYN_SENT)
Florin Coras93992a92017-05-24 18:03:56 -07001537 {
Florin Coras68810622017-07-24 17:40:28 -07001538 /* Half-open connection actually moved to established but we were
1539 * waiting for syn retransmit to pop to call cleanup from the right
1540 * thread. */
1541 if (tc->flags & TCP_CONN_HALF_OPEN_DONE)
1542 {
Florin Coras68810622017-07-24 17:40:28 -07001543 if (tcp_half_open_connection_cleanup (tc))
1544 {
1545 clib_warning ("could not remove half-open connection");
1546 ASSERT (0);
1547 }
1548 return;
1549 }
1550
Florin Corase55a6d72018-10-31 23:09:22 -07001551 TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 2);
1552
Dave Barach68b0fb02017-02-28 15:15:56 -05001553 /* Try without increasing RTO a number of times. If this fails,
1554 * start growing RTO exponentially */
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001555 tc->rto_boff += 1;
Dave Barach68b0fb02017-02-28 15:15:56 -05001556 if (tc->rto_boff > TCP_RTO_SYN_RETRIES)
1557 tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
1558
Florin Corasf988e692017-11-27 04:34:14 -05001559 tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN,
1560 tc->rto * TCP_TO_TIMER_TICK);
1561
Florin Corasbe72ae62018-11-01 11:23:03 -07001562 if (PREDICT_FALSE (tcp_get_free_buffer_index (wrk, &bi)))
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001563 return;
1564
Florin Corasb2215d62017-08-01 16:56:58 -07001565 b = vlib_get_buffer (vm, bi);
Florin Coras1f152cd2017-08-18 19:28:03 -07001566 tcp_init_buffer (vm, b);
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001567 tcp_make_syn (tc, b);
Florin Corase04c2992017-03-01 08:17:34 -08001568
Dave Barach2c25a622017-06-26 11:35:07 -04001569 tc->rtt_ts = 0;
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001570 TCP_EVT_DBG (TCP_EVT_SYN_RXT, tc, 0);
1571
1572 /* This goes straight to ipx_lookup. Retransmit timer set already */
Florin Corasbe72ae62018-11-01 11:23:03 -07001573 tcp_push_ip_hdr (wrk, tc, b);
1574 tcp_enqueue_to_ip_lookup (wrk, b, bi, tc->c_is_ip4, tc->c_fib_index);
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001575 }
1576 /* Retransmit SYN-ACK */
1577 else if (tc->state == TCP_STATE_SYN_RCVD)
1578 {
Florin Corase55a6d72018-10-31 23:09:22 -07001579 TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 2);
1580
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001581 tc->rto_boff += 1;
Florin Coras9d063042017-09-14 03:08:00 -04001582 if (tc->rto_boff > TCP_RTO_SYN_RETRIES)
1583 tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001584 tc->rtt_ts = 0;
1585
Florin Corasbe72ae62018-11-01 11:23:03 -07001586 if (PREDICT_FALSE (tcp_get_free_buffer_index (wrk, &bi)))
Florin Corasf988e692017-11-27 04:34:14 -05001587 {
1588 tcp_retransmit_timer_force_update (tc);
1589 return;
1590 }
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001591
1592 b = vlib_get_buffer (vm, bi);
Florin Coras24793e32018-05-09 11:34:25 -07001593 tcp_init_buffer (vm, b);
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001594 tcp_make_synack (tc, b);
1595 TCP_EVT_DBG (TCP_EVT_SYN_RXT, tc, 1);
1596
1597 /* Retransmit timer already updated, just enqueue to output */
Florin Corasbe72ae62018-11-01 11:23:03 -07001598 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Dave Barach68b0fb02017-02-28 15:15:56 -05001599 }
Florin Coras93992a92017-05-24 18:03:56 -07001600 else
1601 {
1602 ASSERT (tc->state == TCP_STATE_CLOSED);
Florin Coras93992a92017-05-24 18:03:56 -07001603 return;
1604 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001605}
1606
1607void
1608tcp_timer_retransmit_handler (u32 index)
1609{
1610 tcp_timer_retransmit_handler_i (index, 0);
1611}
1612
1613void
1614tcp_timer_retransmit_syn_handler (u32 index)
1615{
1616 tcp_timer_retransmit_handler_i (index, 1);
1617}
1618
1619/**
Florin Coras3e350af2017-03-30 02:54:28 -07001620 * Got 0 snd_wnd from peer, try to do something about it.
1621 *
1622 */
1623void
1624tcp_timer_persist_handler (u32 index)
1625{
Damjan Marion586afd72017-04-05 19:18:20 +02001626 u32 thread_index = vlib_get_thread_index ();
Florin Corasbe72ae62018-11-01 11:23:03 -07001627 tcp_worker_ctx_t *wrk = tcp_get_worker (thread_index);
1628 u32 bi, max_snd_bytes, available_bytes, offset;
1629 tcp_main_t *tm = vnet_get_tcp_main ();
1630 vlib_main_t *vm = wrk->vm;
Florin Coras3e350af2017-03-30 02:54:28 -07001631 tcp_connection_t *tc;
1632 vlib_buffer_t *b;
Florin Coras93992a92017-05-24 18:03:56 -07001633 int n_bytes = 0;
Florin Coras1f152cd2017-08-18 19:28:03 -07001634 u8 *data;
Florin Coras3e350af2017-03-30 02:54:28 -07001635
Florin Corasdb84e572017-05-09 18:54:52 -07001636 tc = tcp_connection_get_if_valid (index, thread_index);
1637
1638 if (!tc)
1639 return;
Florin Coras3e350af2017-03-30 02:54:28 -07001640
1641 /* Make sure timer handle is set to invalid */
1642 tc->timers[TCP_TIMER_PERSIST] = TCP_TIMER_HANDLE_INVALID;
1643
1644 /* Problem already solved or worse */
Florin Corasf03a59a2017-06-09 21:07:32 -07001645 if (tc->state == TCP_STATE_CLOSED || tc->state > TCP_STATE_ESTABLISHED
Florin Coras50958952017-08-29 14:50:13 -07001646 || tc->snd_wnd > tc->snd_mss || tcp_in_recovery (tc))
Florin Coras3e350af2017-03-30 02:54:28 -07001647 return;
1648
Florin Coras25579b42018-06-06 17:55:02 -07001649 available_bytes = session_tx_fifo_max_dequeue (&tc->connection);
Florin Coras50958952017-08-29 14:50:13 -07001650 offset = tc->snd_una_max - tc->snd_una;
1651
1652 /* Reprogram persist if no new bytes available to send. We may have data
1653 * next time */
1654 if (!available_bytes)
1655 {
1656 tcp_persist_timer_set (tc);
1657 return;
1658 }
1659
1660 if (available_bytes <= offset)
1661 {
1662 ASSERT (tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT));
1663 return;
1664 }
1665
Florin Coras3e350af2017-03-30 02:54:28 -07001666 /* Increment RTO backoff */
1667 tc->rto_boff += 1;
1668 tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
1669
Florin Corasb2215d62017-08-01 16:56:58 -07001670 /*
1671 * Try to force the first unsent segment (or buffer)
1672 */
Florin Corasbe72ae62018-11-01 11:23:03 -07001673 if (PREDICT_FALSE (tcp_get_free_buffer_index (wrk, &bi)))
Florin Coras3ec66b02018-08-23 16:27:05 -07001674 {
1675 tcp_persist_timer_set (tc);
1676 return;
1677 }
Florin Coras3e350af2017-03-30 02:54:28 -07001678 b = vlib_get_buffer (vm, bi);
Florin Coras1f152cd2017-08-18 19:28:03 -07001679 data = tcp_init_buffer (vm, b);
Florin Coras93992a92017-05-24 18:03:56 -07001680
Florin Coras1f152cd2017-08-18 19:28:03 -07001681 tcp_validate_txf_size (tc, offset);
Florin Corasc8343412017-05-04 14:25:50 -07001682 tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts, tc->state);
Florin Coras1f152cd2017-08-18 19:28:03 -07001683 max_snd_bytes = clib_min (tc->snd_mss, tm->bytes_per_buffer - MAX_HDRS_LEN);
1684 n_bytes = stream_session_peek_bytes (&tc->connection, data, offset,
1685 max_snd_bytes);
Florin Coras3e350af2017-03-30 02:54:28 -07001686 b->current_length = n_bytes;
Florin Coras84275e92017-09-26 12:30:40 -04001687 ASSERT (n_bytes != 0 && (tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT)
1688 || tc->snd_nxt == tc->snd_una_max
1689 || tc->rto_boff > 1));
Florin Coras93992a92017-05-24 18:03:56 -07001690
Florin Corasb26743d2018-06-26 09:31:04 -07001691 tcp_push_hdr_i (tc, b, tc->state, /* compute opts */ 0, /* burst */ 0);
Florin Coras8b20bf52018-06-14 14:55:50 -07001692 tc->snd_una_max = tc->snd_nxt;
1693 tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una);
Florin Corasbe72ae62018-11-01 11:23:03 -07001694 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Florin Coras3e350af2017-03-30 02:54:28 -07001695
Florin Coras1f152cd2017-08-18 19:28:03 -07001696 /* Just sent new data, enable retransmit */
1697 tcp_retransmit_timer_update (tc);
Florin Coras3e350af2017-03-30 02:54:28 -07001698}
1699
1700/**
Florin Coras6792ec02017-03-13 03:49:51 -07001701 * Retransmit first unacked segment
1702 */
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001703int
Florin Corasbe72ae62018-11-01 11:23:03 -07001704tcp_retransmit_first_unacked (tcp_worker_ctx_t * wrk, tcp_connection_t * tc)
Dave Barach68b0fb02017-02-28 15:15:56 -05001705{
Florin Corasb2215d62017-08-01 16:56:58 -07001706 u32 bi, old_snd_nxt, n_bytes;
Florin Corasbe72ae62018-11-01 11:23:03 -07001707 vlib_main_t *vm = wrk->vm;
1708 vlib_buffer_t *b;
Dave Barach68b0fb02017-02-28 15:15:56 -05001709
Florin Coras93992a92017-05-24 18:03:56 -07001710 old_snd_nxt = tc->snd_nxt;
Dave Barach68b0fb02017-02-28 15:15:56 -05001711 tc->snd_nxt = tc->snd_una;
1712
Florin Corase55a6d72018-10-31 23:09:22 -07001713 TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 1);
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001714
Florin Corasbe72ae62018-11-01 11:23:03 -07001715 n_bytes = tcp_prepare_retransmit_segment (wrk, tc, 0, tc->snd_mss, &b);
Florin Corasb2215d62017-08-01 16:56:58 -07001716 if (!n_bytes)
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001717 return -1;
1718
Florin Corasb2215d62017-08-01 16:56:58 -07001719 bi = vlib_get_buffer_index (vm, b);
Florin Corasbe72ae62018-11-01 11:23:03 -07001720 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Florin Coras93992a92017-05-24 18:03:56 -07001721 tc->snd_nxt = old_snd_nxt;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001722
1723 return 0;
Florin Coras6792ec02017-03-13 03:49:51 -07001724}
1725
Florin Coras36ee9f12018-11-02 12:52:10 -07001726static int
1727tcp_fast_retransmit_unsent (tcp_worker_ctx_t * wrk, tcp_connection_t * tc,
1728 u32 burst_size)
1729{
1730 u32 offset, n_segs = 0, n_written, bi;
1731 vlib_main_t *vm = wrk->vm;
1732 vlib_buffer_t *b = 0;
1733
1734 tc->snd_nxt = tc->snd_una_max;
1735 offset = tc->snd_una_max - tc->snd_una;
1736 while (n_segs < burst_size)
1737 {
1738 n_written = tcp_prepare_segment (wrk, tc, offset, tc->snd_mss, &b);
1739 if (!n_written)
1740 goto done;
1741
1742 bi = vlib_get_buffer_index (vm, b);
1743 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
1744 offset += n_written;
1745 n_segs += 1;
1746 }
1747
1748done:
1749 return n_segs;
1750}
1751
1752#define scoreboard_rescue_rxt_valid(_sb, _tc) \
1753 (seq_geq (_sb->rescue_rxt, _tc->snd_una) \
1754 && seq_leq (_sb->rescue_rxt, _tc->snd_congestion))
1755
Florin Coras6792ec02017-03-13 03:49:51 -07001756/**
Florin Coras93992a92017-05-24 18:03:56 -07001757 * Do fast retransmit with SACKs
Florin Coras6792ec02017-03-13 03:49:51 -07001758 */
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001759int
Florin Corasbe72ae62018-11-01 11:23:03 -07001760tcp_fast_retransmit_sack (tcp_worker_ctx_t * wrk, tcp_connection_t * tc,
1761 u32 burst_size)
Dave Barach68b0fb02017-02-28 15:15:56 -05001762{
Florin Coras36ee9f12018-11-02 12:52:10 -07001763 u32 n_written = 0, offset, max_bytes, n_segs = 0, n_segs_now;
Florin Coras93992a92017-05-24 18:03:56 -07001764 sack_scoreboard_hole_t *hole;
Florin Coras36ee9f12018-11-02 12:52:10 -07001765 vlib_main_t *vm = wrk->vm;
1766 vlib_buffer_t *b = 0;
Florin Coras93992a92017-05-24 18:03:56 -07001767 sack_scoreboard_t *sb;
1768 u32 bi, old_snd_nxt;
1769 int snd_space;
Florin Coras36ee9f12018-11-02 12:52:10 -07001770 u32 max_deq;
Florin Coras93992a92017-05-24 18:03:56 -07001771 u8 snd_limited = 0, can_rescue = 0;
Dave Barach68b0fb02017-02-28 15:15:56 -05001772
1773 ASSERT (tcp_in_fastrecovery (tc));
Dave Barach68b0fb02017-02-28 15:15:56 -05001774
Florin Corasca1c8f32018-05-23 21:01:30 -07001775 snd_space = tcp_available_cc_snd_space (tc);
Florin Corasca1c8f32018-05-23 21:01:30 -07001776 if (snd_space < tc->snd_mss)
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001777 {
Florin Corasbe72ae62018-11-01 11:23:03 -07001778 tcp_program_fastretransmit (wrk, tc);
Florin Coras36ee9f12018-11-02 12:52:10 -07001779 return 0;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001780 }
Florin Corasca1c8f32018-05-23 21:01:30 -07001781
1782 TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 0);
Florin Coras36ee9f12018-11-02 12:52:10 -07001783 old_snd_nxt = tc->snd_nxt;
1784 sb = &tc->sack_sb;
1785 hole = scoreboard_get_hole (sb, sb->cur_rxt_hole);
1786
1787 max_deq = session_tx_fifo_max_dequeue (&tc->connection);
1788 max_deq -= tc->snd_una_max - tc->snd_una;
1789
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001790 while (snd_space > 0 && n_segs < burst_size)
Florin Coras93992a92017-05-24 18:03:56 -07001791 {
Florin Coras36ee9f12018-11-02 12:52:10 -07001792 hole = scoreboard_next_rxt_hole (sb, hole, max_deq, &can_rescue,
1793 &snd_limited);
Florin Coras93992a92017-05-24 18:03:56 -07001794 if (!hole)
1795 {
Florin Coras36ee9f12018-11-02 12:52:10 -07001796 if (max_deq)
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001797 {
Florin Coras36ee9f12018-11-02 12:52:10 -07001798 snd_space = clib_min (max_deq, snd_space);
1799 burst_size = clib_min (burst_size - n_segs,
1800 snd_space / tc->snd_mss);
1801 n_segs_now = tcp_fast_retransmit_unsent (wrk, tc, burst_size);
1802 if (max_deq > n_segs_now * tc->snd_mss)
1803 tcp_program_fastretransmit (wrk, tc);
1804 n_segs += n_segs_now;
1805 goto done;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001806 }
Florin Coras93992a92017-05-24 18:03:56 -07001807
Florin Coras36ee9f12018-11-02 12:52:10 -07001808 if (!can_rescue || scoreboard_rescue_rxt_valid (sb, tc))
1809 break;
1810
Florin Coras93992a92017-05-24 18:03:56 -07001811 /* If rescue rxt undefined or less than snd_una then one segment of
1812 * up to SMSS octets that MUST include the highest outstanding
1813 * unSACKed sequence number SHOULD be returned, and RescueRxt set to
1814 * RecoveryPoint. HighRxt MUST NOT be updated.
1815 */
Florin Coras1f152cd2017-08-18 19:28:03 -07001816 max_bytes = clib_min (tc->snd_mss,
1817 tc->snd_congestion - tc->snd_una);
1818 max_bytes = clib_min (max_bytes, snd_space);
Florin Coras93992a92017-05-24 18:03:56 -07001819 offset = tc->snd_congestion - tc->snd_una - max_bytes;
1820 sb->rescue_rxt = tc->snd_congestion;
1821 tc->snd_nxt = tc->snd_una + offset;
Florin Corasbe72ae62018-11-01 11:23:03 -07001822 n_written = tcp_prepare_retransmit_segment (wrk, tc, offset,
1823 max_bytes, &b);
Florin Coras24793e32018-05-09 11:34:25 -07001824 if (!n_written)
1825 goto done;
1826
Florin Corasb2215d62017-08-01 16:56:58 -07001827 bi = vlib_get_buffer_index (vm, b);
Florin Corasbe72ae62018-11-01 11:23:03 -07001828 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001829 n_segs += 1;
Florin Coras93992a92017-05-24 18:03:56 -07001830 break;
1831 }
1832
Florin Coras1f152cd2017-08-18 19:28:03 -07001833 max_bytes = clib_min (hole->end - sb->high_rxt, snd_space);
1834 max_bytes = snd_limited ? clib_min (max_bytes, tc->snd_mss) : max_bytes;
1835 if (max_bytes == 0)
1836 break;
Florin Coras36ee9f12018-11-02 12:52:10 -07001837
Florin Coras93992a92017-05-24 18:03:56 -07001838 offset = sb->high_rxt - tc->snd_una;
Florin Coras1f152cd2017-08-18 19:28:03 -07001839 tc->snd_nxt = sb->high_rxt;
Florin Corasbe72ae62018-11-01 11:23:03 -07001840 n_written = tcp_prepare_retransmit_segment (wrk, tc, offset, max_bytes,
1841 &b);
Florin Coras36ee9f12018-11-02 12:52:10 -07001842 ASSERT (n_written <= snd_space);
Florin Coras93992a92017-05-24 18:03:56 -07001843
1844 /* Nothing left to retransmit */
1845 if (n_written == 0)
Florin Corasb2215d62017-08-01 16:56:58 -07001846 break;
Florin Coras93992a92017-05-24 18:03:56 -07001847
Florin Corasb2215d62017-08-01 16:56:58 -07001848 bi = vlib_get_buffer_index (vm, b);
Florin Corasbe72ae62018-11-01 11:23:03 -07001849 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Florin Coras36ee9f12018-11-02 12:52:10 -07001850
1851 sb->high_rxt += n_written;
Florin Coras93992a92017-05-24 18:03:56 -07001852 snd_space -= n_written;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001853 n_segs += 1;
Florin Coras93992a92017-05-24 18:03:56 -07001854 }
1855
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001856 if (hole)
Florin Corasbe72ae62018-11-01 11:23:03 -07001857 tcp_program_fastretransmit (wrk, tc);
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001858
Florin Coras24793e32018-05-09 11:34:25 -07001859done:
Florin Coras93992a92017-05-24 18:03:56 -07001860 /* If window allows, send 1 SMSS of new data */
1861 tc->snd_nxt = old_snd_nxt;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001862 return n_segs;
Florin Coras93992a92017-05-24 18:03:56 -07001863}
1864
1865/**
1866 * Fast retransmit without SACK info
1867 */
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001868int
Florin Corasbe72ae62018-11-01 11:23:03 -07001869tcp_fast_retransmit_no_sack (tcp_worker_ctx_t * wrk, tcp_connection_t * tc,
1870 u32 burst_size)
Florin Coras93992a92017-05-24 18:03:56 -07001871{
Florin Coras36ee9f12018-11-02 12:52:10 -07001872 u32 n_written = 0, offset = 0, bi, old_snd_nxt, max_deq, n_segs_now;
Florin Corasbe72ae62018-11-01 11:23:03 -07001873 vlib_main_t *vm = wrk->vm;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001874 int snd_space, n_segs = 0;
Florin Coras93992a92017-05-24 18:03:56 -07001875 vlib_buffer_t *b;
1876
1877 ASSERT (tcp_in_fastrecovery (tc));
1878 TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 0);
Florin Coras93992a92017-05-24 18:03:56 -07001879 old_snd_nxt = tc->snd_nxt;
Florin Coras6792ec02017-03-13 03:49:51 -07001880
Florin Coras36ee9f12018-11-02 12:52:10 -07001881 if (!tcp_fastrecovery_first (tc))
1882 goto send_unsent;
1883
1884 /* RFC 6582: [If a partial ack], retransmit the first unacknowledged
1885 * segment. */
1886 snd_space = tc->sack_sb.last_bytes_delivered;
1887 tc->snd_nxt = tc->snd_una;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001888 while (snd_space > 0 && n_segs < burst_size)
Dave Barach68b0fb02017-02-28 15:15:56 -05001889 {
Florin Coras36ee9f12018-11-02 12:52:10 -07001890 n_written = tcp_prepare_retransmit_segment (wrk, tc, offset,
1891 tc->snd_mss, &b);
Dave Barach68b0fb02017-02-28 15:15:56 -05001892
1893 /* Nothing left to retransmit */
Florin Coras6792ec02017-03-13 03:49:51 -07001894 if (n_written == 0)
Florin Corasb2215d62017-08-01 16:56:58 -07001895 break;
Dave Barach68b0fb02017-02-28 15:15:56 -05001896
Florin Corasb2215d62017-08-01 16:56:58 -07001897 bi = vlib_get_buffer_index (vm, b);
Florin Corasbe72ae62018-11-01 11:23:03 -07001898 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Florin Coras6792ec02017-03-13 03:49:51 -07001899 snd_space -= n_written;
Florin Coras36ee9f12018-11-02 12:52:10 -07001900 offset += n_written;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001901 n_segs += 1;
Dave Barach68b0fb02017-02-28 15:15:56 -05001902 }
1903
Florin Coras36ee9f12018-11-02 12:52:10 -07001904 if (n_segs == burst_size)
1905 goto done;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001906
Florin Coras36ee9f12018-11-02 12:52:10 -07001907send_unsent:
1908
1909 /* RFC 6582: Send a new segment if permitted by the new value of cwnd. */
1910 snd_space = tcp_available_cc_snd_space (tc);
Florin Coras9ece3c02018-11-05 11:06:53 -08001911 if (snd_space < tc->snd_mss || tc->snd_mss == 0)
Florin Coras36ee9f12018-11-02 12:52:10 -07001912 goto done;
1913
1914 max_deq = session_tx_fifo_max_dequeue (&tc->connection);
1915 max_deq -= tc->snd_una_max - tc->snd_una;
1916 if (max_deq)
1917 {
1918 snd_space = clib_min (max_deq, snd_space);
1919 burst_size = clib_min (burst_size - n_segs, snd_space / tc->snd_mss);
1920 n_segs_now = tcp_fast_retransmit_unsent (wrk, tc, burst_size);
1921 if (max_deq > n_segs_now * tc->snd_mss)
1922 tcp_program_fastretransmit (wrk, tc);
1923 n_segs += n_segs_now;
1924 }
1925
1926 /* Restore snd_nxt */
Florin Coras93992a92017-05-24 18:03:56 -07001927 tc->snd_nxt = old_snd_nxt;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001928
Florin Coras36ee9f12018-11-02 12:52:10 -07001929done:
1930 tcp_fastrecovery_first_off (tc);
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001931 return n_segs;
Florin Coras93992a92017-05-24 18:03:56 -07001932}
1933
1934/**
1935 * Do fast retransmit
1936 */
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001937int
Florin Corasbe72ae62018-11-01 11:23:03 -07001938tcp_fast_retransmit (tcp_worker_ctx_t * wrk, tcp_connection_t * tc,
1939 u32 burst_size)
Florin Coras93992a92017-05-24 18:03:56 -07001940{
Florin Corasca1c8f32018-05-23 21:01:30 -07001941 if (tcp_opts_sack_permitted (&tc->rcv_opts))
Florin Corasbe72ae62018-11-01 11:23:03 -07001942 return tcp_fast_retransmit_sack (wrk, tc, burst_size);
Florin Coras93992a92017-05-24 18:03:56 -07001943 else
Florin Corasbe72ae62018-11-01 11:23:03 -07001944 return tcp_fast_retransmit_no_sack (wrk, tc, burst_size);
Dave Barach68b0fb02017-02-28 15:15:56 -05001945}
1946
Florin Coras0dbd5172018-06-25 16:19:34 -07001947static u32
Dave Barach68b0fb02017-02-28 15:15:56 -05001948tcp_session_has_ooo_data (tcp_connection_t * tc)
1949{
Florin Corascea194d2017-10-02 00:18:51 -07001950 stream_session_t *s = session_get (tc->c_s_index, tc->c_thread_index);
Dave Barach68b0fb02017-02-28 15:15:56 -05001951 return svm_fifo_has_ooo_data (s->server_rx_fifo);
1952}
1953
Florin Corasf9d05682018-04-26 08:26:52 -07001954static void
1955tcp_output_handle_link_local (tcp_connection_t * tc0, vlib_buffer_t * b0,
Florin Coras8b20bf52018-06-14 14:55:50 -07001956 u16 * next0, u32 * error0)
Florin Corasf9d05682018-04-26 08:26:52 -07001957{
1958 ip_adjacency_t *adj;
1959 adj_index_t ai;
1960
Florin Coras1c8ff632018-05-17 13:28:34 -07001961 /* Not thread safe but as long as the connection exists the adj should
1962 * not be removed */
Florin Corasf9d05682018-04-26 08:26:52 -07001963 ai = adj_nbr_find (FIB_PROTOCOL_IP6, VNET_LINK_IP6, &tc0->c_rmt_ip,
1964 tc0->sw_if_index);
1965 if (ai == ADJ_INDEX_INVALID)
1966 {
Florin Corasf9d05682018-04-26 08:26:52 -07001967 vnet_buffer (b0)->sw_if_index[VLIB_TX] = ~0;
1968 *next0 = TCP_OUTPUT_NEXT_DROP;
1969 *error0 = TCP_ERROR_LINK_LOCAL_RW;
1970 return;
1971 }
1972
1973 adj = adj_get (ai);
Florin Coras1c8ff632018-05-17 13:28:34 -07001974 if (PREDICT_TRUE (adj->lookup_next_index == IP_LOOKUP_NEXT_REWRITE))
Florin Corasf9d05682018-04-26 08:26:52 -07001975 *next0 = TCP_OUTPUT_NEXT_IP_REWRITE;
1976 else if (adj->lookup_next_index == IP_LOOKUP_NEXT_ARP)
1977 *next0 = TCP_OUTPUT_NEXT_IP_ARP;
Florin Coras1c8ff632018-05-17 13:28:34 -07001978 else
1979 {
1980 *next0 = TCP_OUTPUT_NEXT_DROP;
1981 *error0 = TCP_ERROR_LINK_LOCAL_RW;
1982 }
1983 vnet_buffer (b0)->ip.adj_index[VLIB_TX] = ai;
Florin Corasf9d05682018-04-26 08:26:52 -07001984}
1985
Florin Coras8b20bf52018-06-14 14:55:50 -07001986static void
1987tcp46_output_trace_frame (vlib_main_t * vm, vlib_node_runtime_t * node,
1988 u32 * to_next, u32 n_bufs)
Dave Barach68b0fb02017-02-28 15:15:56 -05001989{
Florin Coras8b20bf52018-06-14 14:55:50 -07001990 u32 n_trace = vlib_get_trace_count (vm, node);
1991 tcp_connection_t *tc;
1992 tcp_tx_trace_t *t;
1993 vlib_buffer_t *b;
1994 tcp_header_t *th;
1995 int i;
Dave Barach68b0fb02017-02-28 15:15:56 -05001996
Florin Coras8b20bf52018-06-14 14:55:50 -07001997 for (i = 0; i < clib_min (n_trace, n_bufs); i++)
Dave Barach68b0fb02017-02-28 15:15:56 -05001998 {
Florin Coras8b20bf52018-06-14 14:55:50 -07001999 b = vlib_get_buffer (vm, to_next[i]);
2000 th = vlib_buffer_get_current (b);
2001 tc = tcp_connection_get (vnet_buffer (b)->tcp.connection_index,
2002 vm->thread_index);
2003 t = vlib_add_trace (vm, node, b, sizeof (*t));
2004 clib_memcpy (&t->tcp_header, th, sizeof (t->tcp_header));
2005 clib_memcpy (&t->tcp_connection, tc, sizeof (t->tcp_connection));
2006 }
2007}
Dave Barach68b0fb02017-02-28 15:15:56 -05002008
Florin Coras0dbd5172018-06-25 16:19:34 -07002009always_inline void
Florin Coras8b20bf52018-06-14 14:55:50 -07002010tcp_output_push_ip (vlib_main_t * vm, vlib_buffer_t * b0,
2011 tcp_connection_t * tc0, u8 is_ip4)
2012{
2013 tcp_header_t *th0 = 0;
Dave Barach68b0fb02017-02-28 15:15:56 -05002014
Florin Coras8b20bf52018-06-14 14:55:50 -07002015 th0 = vlib_buffer_get_current (b0);
2016 TCP_EVT_DBG (TCP_EVT_OUTPUT, tc0, th0->flags, b0->current_length);
2017 if (is_ip4)
2018 {
2019 vlib_buffer_push_ip4 (vm, b0, &tc0->c_lcl_ip4, &tc0->c_rmt_ip4,
2020 IP_PROTOCOL_TCP, 1);
2021 b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
2022 vnet_buffer (b0)->l4_hdr_offset = (u8 *) th0 - b0->data;
2023 th0->checksum = 0;
2024 }
2025 else
2026 {
2027 ip6_header_t *ih0;
2028 ih0 = vlib_buffer_push_ip6 (vm, b0, &tc0->c_lcl_ip6,
2029 &tc0->c_rmt_ip6, IP_PROTOCOL_TCP);
2030 b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
2031 vnet_buffer (b0)->l3_hdr_offset = (u8 *) ih0 - b0->data;
2032 vnet_buffer (b0)->l4_hdr_offset = (u8 *) th0 - b0->data;
2033 th0->checksum = 0;
2034 }
2035}
Dave Barach68b0fb02017-02-28 15:15:56 -05002036
Florin Coras0dbd5172018-06-25 16:19:34 -07002037always_inline void
Florin Coras8b20bf52018-06-14 14:55:50 -07002038tcp_output_handle_packet (tcp_connection_t * tc0, vlib_buffer_t * b0,
2039 u32 * error0, u16 * next0, u8 is_ip4)
2040{
Florin Coras81a13db2018-03-16 08:48:31 -07002041
Florin Coras8b20bf52018-06-14 14:55:50 -07002042 if (PREDICT_FALSE (tc0->state == TCP_STATE_CLOSED))
2043 {
2044 *error0 = TCP_ERROR_INVALID_CONNECTION;
2045 *next0 = TCP_OUTPUT_NEXT_DROP;
2046 return;
Dave Barach68b0fb02017-02-28 15:15:56 -05002047 }
2048
Florin Coras8b20bf52018-06-14 14:55:50 -07002049 vnet_buffer (b0)->sw_if_index[VLIB_TX] = tc0->c_fib_index;
2050 vnet_buffer (b0)->sw_if_index[VLIB_RX] = 0;
2051
2052 if (!is_ip4)
2053 {
2054 if (PREDICT_FALSE (ip6_address_is_link_local_unicast (&tc0->c_rmt_ip6)))
2055 tcp_output_handle_link_local (tc0, b0, next0, error0);
2056 }
2057
2058 /* Filter out DUPACKs if there are no OOO segments left */
2059 if (PREDICT_FALSE (vnet_buffer (b0)->tcp.flags & TCP_BUF_FLAG_DUPACK))
2060 {
2061 /* N.B. Should not filter burst of dupacks. Two issues:
2062 * 1) dupacks open cwnd on remote peer when congested
2063 * 2) acks leaving should have the latest rcv_wnd since the
2064 * burst may have eaten up all of it, so only the old ones
2065 * could be filtered.
2066 */
2067 if (!tcp_session_has_ooo_data (tc0))
2068 {
2069 *error0 = TCP_ERROR_FILTERED_DUPACKS;
2070 *next0 = TCP_OUTPUT_NEXT_DROP;
2071 return;
2072 }
2073 }
2074
2075 /* Stop DELACK timer and fix flags */
2076 tc0->flags &= ~(TCP_CONN_SNDACK);
2077 if (!TCP_ALWAYS_ACK)
2078 tcp_timer_reset (tc0, TCP_TIMER_DELACK);
2079}
2080
2081always_inline uword
2082tcp46_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
2083 vlib_frame_t * frame, int is_ip4)
2084{
2085 u32 n_left_from, *from, thread_index = vm->thread_index;
2086 vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
2087 u16 nexts[VLIB_FRAME_SIZE], *next;
2088
2089 from = vlib_frame_vector_args (frame);
2090 n_left_from = frame->n_vectors;
Florin Corasbe72ae62018-11-01 11:23:03 -07002091 tcp_set_time_now (tcp_get_worker (thread_index));
Florin Coras8b20bf52018-06-14 14:55:50 -07002092
2093 if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
2094 tcp46_output_trace_frame (vm, node, from, n_left_from);
2095
2096 vlib_get_buffers (vm, from, bufs, n_left_from);
2097 b = bufs;
2098 next = nexts;
2099
2100 while (n_left_from >= 4)
2101 {
2102 u32 error0 = TCP_ERROR_PKTS_SENT, error1 = TCP_ERROR_PKTS_SENT;
2103 tcp_connection_t *tc0, *tc1;
2104
2105 {
2106 vlib_prefetch_buffer_header (b[2], STORE);
2107 CLIB_PREFETCH (b[2]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE);
2108
2109 vlib_prefetch_buffer_header (b[3], STORE);
2110 CLIB_PREFETCH (b[3]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE);
2111 }
2112
2113 next[0] = next[1] = TCP_OUTPUT_NEXT_IP_LOOKUP;
2114
2115 tc0 = tcp_connection_get (vnet_buffer (b[0])->tcp.connection_index,
2116 thread_index);
2117 tc1 = tcp_connection_get (vnet_buffer (b[1])->tcp.connection_index,
2118 thread_index);
2119
2120 tcp_output_push_ip (vm, b[0], tc0, is_ip4);
2121 tcp_output_push_ip (vm, b[1], tc1, is_ip4);
2122
2123 tcp_output_handle_packet (tc0, b[0], &error0, &next[0], is_ip4);
2124 tcp_output_handle_packet (tc1, b[1], &error1, &next[1], is_ip4);
2125
2126 b += 2;
2127 next += 2;
2128 n_left_from -= 2;
2129 }
2130 while (n_left_from > 0)
2131 {
2132 u32 error0 = TCP_ERROR_PKTS_SENT;
2133 tcp_connection_t *tc0;
2134
2135 if (n_left_from > 1)
2136 {
Florin Coras91ca4622018-06-30 11:27:59 -07002137 vlib_prefetch_buffer_header (b[1], STORE);
2138 CLIB_PREFETCH (b[1]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE);
Florin Coras8b20bf52018-06-14 14:55:50 -07002139 }
2140
2141 next[0] = TCP_OUTPUT_NEXT_IP_LOOKUP;
2142 tc0 = tcp_connection_get (vnet_buffer (b[0])->tcp.connection_index,
2143 thread_index);
2144
2145 tcp_output_push_ip (vm, b[0], tc0, is_ip4);
2146 tcp_output_handle_packet (tc0, b[0], &error0, &next[0], is_ip4);
2147
2148 b += 1;
2149 next += 1;
2150 n_left_from -= 1;
2151 }
2152
2153 vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
2154 return frame->n_vectors;
Dave Barach68b0fb02017-02-28 15:15:56 -05002155}
2156
2157static uword
2158tcp4_output (vlib_main_t * vm, vlib_node_runtime_t * node,
2159 vlib_frame_t * from_frame)
2160{
2161 return tcp46_output_inline (vm, node, from_frame, 1 /* is_ip4 */ );
2162}
2163
2164static uword
2165tcp6_output (vlib_main_t * vm, vlib_node_runtime_t * node,
2166 vlib_frame_t * from_frame)
2167{
2168 return tcp46_output_inline (vm, node, from_frame, 0 /* is_ip4 */ );
2169}
2170
Florin Corase69f4952017-03-07 10:06:24 -08002171/* *INDENT-OFF* */
Dave Barach68b0fb02017-02-28 15:15:56 -05002172VLIB_REGISTER_NODE (tcp4_output_node) =
2173{
2174 .function = tcp4_output,.name = "tcp4-output",
2175 /* Takes a vector of packets. */
Florin Corase69f4952017-03-07 10:06:24 -08002176 .vector_size = sizeof (u32),
2177 .n_errors = TCP_N_ERROR,
2178 .error_strings = tcp_error_strings,
2179 .n_next_nodes = TCP_OUTPUT_N_NEXT,
2180 .next_nodes = {
Dave Barach68b0fb02017-02-28 15:15:56 -05002181#define _(s,n) [TCP_OUTPUT_NEXT_##s] = n,
2182 foreach_tcp4_output_next
2183#undef _
Florin Corase69f4952017-03-07 10:06:24 -08002184 },
2185 .format_buffer = format_tcp_header,
2186 .format_trace = format_tcp_tx_trace,
2187};
2188/* *INDENT-ON* */
Dave Barach68b0fb02017-02-28 15:15:56 -05002189
Florin Corase69f4952017-03-07 10:06:24 -08002190VLIB_NODE_FUNCTION_MULTIARCH (tcp4_output_node, tcp4_output);
2191
2192/* *INDENT-OFF* */
Dave Barach68b0fb02017-02-28 15:15:56 -05002193VLIB_REGISTER_NODE (tcp6_output_node) =
2194{
Florin Corase69f4952017-03-07 10:06:24 -08002195 .function = tcp6_output,
2196 .name = "tcp6-output",
Dave Barach68b0fb02017-02-28 15:15:56 -05002197 /* Takes a vector of packets. */
Florin Corase69f4952017-03-07 10:06:24 -08002198 .vector_size = sizeof (u32),
2199 .n_errors = TCP_N_ERROR,
2200 .error_strings = tcp_error_strings,
2201 .n_next_nodes = TCP_OUTPUT_N_NEXT,
2202 .next_nodes = {
Dave Barach68b0fb02017-02-28 15:15:56 -05002203#define _(s,n) [TCP_OUTPUT_NEXT_##s] = n,
2204 foreach_tcp6_output_next
2205#undef _
Florin Corase69f4952017-03-07 10:06:24 -08002206 },
2207 .format_buffer = format_tcp_header,
2208 .format_trace = format_tcp_tx_trace,
2209};
2210/* *INDENT-ON* */
Dave Barach68b0fb02017-02-28 15:15:56 -05002211
Florin Corase69f4952017-03-07 10:06:24 -08002212VLIB_NODE_FUNCTION_MULTIARCH (tcp6_output_node, tcp6_output);
2213
Dave Barach68b0fb02017-02-28 15:15:56 -05002214typedef enum _tcp_reset_next
2215{
2216 TCP_RESET_NEXT_DROP,
2217 TCP_RESET_NEXT_IP_LOOKUP,
2218 TCP_RESET_N_NEXT
2219} tcp_reset_next_t;
2220
2221#define foreach_tcp4_reset_next \
2222 _(DROP, "error-drop") \
2223 _(IP_LOOKUP, "ip4-lookup")
2224
2225#define foreach_tcp6_reset_next \
2226 _(DROP, "error-drop") \
2227 _(IP_LOOKUP, "ip6-lookup")
2228
2229static uword
2230tcp46_send_reset_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
2231 vlib_frame_t * from_frame, u8 is_ip4)
2232{
2233 u32 n_left_from, next_index, *from, *to_next;
Damjan Marion586afd72017-04-05 19:18:20 +02002234 u32 my_thread_index = vm->thread_index;
Dave Barach68b0fb02017-02-28 15:15:56 -05002235
2236 from = vlib_frame_vector_args (from_frame);
2237 n_left_from = from_frame->n_vectors;
2238
2239 next_index = node->cached_next_index;
2240
2241 while (n_left_from > 0)
2242 {
2243 u32 n_left_to_next;
2244
2245 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2246
2247 while (n_left_from > 0 && n_left_to_next > 0)
2248 {
2249 u32 bi0;
2250 vlib_buffer_t *b0;
Clement Durand6cf260c2017-04-13 13:27:04 +02002251 tcp_tx_trace_t *t0;
2252 tcp_header_t *th0;
Dave Barach68b0fb02017-02-28 15:15:56 -05002253 u32 error0 = TCP_ERROR_RST_SENT, next0 = TCP_RESET_NEXT_IP_LOOKUP;
2254
2255 bi0 = from[0];
2256 to_next[0] = bi0;
2257 from += 1;
2258 to_next += 1;
2259 n_left_from -= 1;
2260 n_left_to_next -= 1;
2261
2262 b0 = vlib_get_buffer (vm, bi0);
2263
2264 if (tcp_make_reset_in_place (vm, b0, vnet_buffer (b0)->tcp.flags,
2265 my_thread_index, is_ip4))
2266 {
2267 error0 = TCP_ERROR_LOOKUP_DROPS;
2268 next0 = TCP_RESET_NEXT_DROP;
2269 goto done;
2270 }
2271
2272 /* Prepare to send to IP lookup */
Florin Corasf988e692017-11-27 04:34:14 -05002273 vnet_buffer (b0)->sw_if_index[VLIB_TX] = ~0;
Dave Barach68b0fb02017-02-28 15:15:56 -05002274 next0 = TCP_RESET_NEXT_IP_LOOKUP;
2275
2276 done:
Florin Corase69f4952017-03-07 10:06:24 -08002277 b0->error = node->errors[error0];
Damjan Marion213b5aa2017-07-13 21:19:27 +02002278 b0->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
Dave Barach68b0fb02017-02-28 15:15:56 -05002279 if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
2280 {
Clement Durand6cf260c2017-04-13 13:27:04 +02002281 th0 = vlib_buffer_get_current (b0);
2282 if (is_ip4)
2283 th0 = ip4_next_header ((ip4_header_t *) th0);
2284 else
2285 th0 = ip6_next_header ((ip6_header_t *) th0);
Clement Durand6cf260c2017-04-13 13:27:04 +02002286 t0 = vlib_add_trace (vm, node, b0, sizeof (*t0));
2287 clib_memcpy (&t0->tcp_header, th0, sizeof (t0->tcp_header));
Dave Barach68b0fb02017-02-28 15:15:56 -05002288 }
2289
2290 vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
2291 n_left_to_next, bi0, next0);
2292 }
2293 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2294 }
2295 return from_frame->n_vectors;
2296}
2297
2298static uword
2299tcp4_send_reset (vlib_main_t * vm, vlib_node_runtime_t * node,
2300 vlib_frame_t * from_frame)
2301{
2302 return tcp46_send_reset_inline (vm, node, from_frame, 1);
2303}
2304
2305static uword
2306tcp6_send_reset (vlib_main_t * vm, vlib_node_runtime_t * node,
2307 vlib_frame_t * from_frame)
2308{
2309 return tcp46_send_reset_inline (vm, node, from_frame, 0);
2310}
2311
2312/* *INDENT-OFF* */
2313VLIB_REGISTER_NODE (tcp4_reset_node) = {
2314 .function = tcp4_send_reset,
2315 .name = "tcp4-reset",
2316 .vector_size = sizeof (u32),
2317 .n_errors = TCP_N_ERROR,
2318 .error_strings = tcp_error_strings,
2319 .n_next_nodes = TCP_RESET_N_NEXT,
2320 .next_nodes = {
2321#define _(s,n) [TCP_RESET_NEXT_##s] = n,
2322 foreach_tcp4_reset_next
2323#undef _
2324 },
Clement Durand6cf260c2017-04-13 13:27:04 +02002325 .format_trace = format_tcp_tx_trace,
Dave Barach68b0fb02017-02-28 15:15:56 -05002326};
2327/* *INDENT-ON* */
2328
Florin Corase69f4952017-03-07 10:06:24 -08002329VLIB_NODE_FUNCTION_MULTIARCH (tcp4_reset_node, tcp4_send_reset);
2330
Dave Barach68b0fb02017-02-28 15:15:56 -05002331/* *INDENT-OFF* */
2332VLIB_REGISTER_NODE (tcp6_reset_node) = {
2333 .function = tcp6_send_reset,
2334 .name = "tcp6-reset",
2335 .vector_size = sizeof (u32),
2336 .n_errors = TCP_N_ERROR,
2337 .error_strings = tcp_error_strings,
2338 .n_next_nodes = TCP_RESET_N_NEXT,
2339 .next_nodes = {
2340#define _(s,n) [TCP_RESET_NEXT_##s] = n,
2341 foreach_tcp6_reset_next
2342#undef _
2343 },
Clement Durand6cf260c2017-04-13 13:27:04 +02002344 .format_trace = format_tcp_tx_trace,
Dave Barach68b0fb02017-02-28 15:15:56 -05002345};
2346/* *INDENT-ON* */
2347
Florin Corase69f4952017-03-07 10:06:24 -08002348VLIB_NODE_FUNCTION_MULTIARCH (tcp6_reset_node, tcp6_send_reset);
2349
Dave Barach68b0fb02017-02-28 15:15:56 -05002350/*
2351 * fd.io coding-style-patch-verification: ON
2352 *
2353 * Local Variables:
2354 * eval: (c-set-style "gnu")
2355 * End:
2356 */