blob: 725ffec0852ec567926ee59e82dfca242eb5f380 [file] [log] [blame]
Dave Barach68b0fb02017-02-28 15:15:56 -05001/*
Florin Coras222e1f412019-02-16 20:47:32 -08002 * Copyright (c) 2016-2019 Cisco and/or its affiliates.
Dave Barach68b0fb02017-02-28 15:15:56 -05003 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16#include <vnet/tcp/tcp.h>
Florin Corasb2215d62017-08-01 16:56:58 -070017#include <math.h>
Dave Barach68b0fb02017-02-28 15:15:56 -050018
19vlib_node_registration_t tcp4_output_node;
20vlib_node_registration_t tcp6_output_node;
21
Dave Barach2c25a622017-06-26 11:35:07 -040022typedef enum _tcp_output_next
Dave Barach68b0fb02017-02-28 15:15:56 -050023{
24 TCP_OUTPUT_NEXT_DROP,
Dave Barach2c25a622017-06-26 11:35:07 -040025 TCP_OUTPUT_NEXT_IP_LOOKUP,
Florin Corasf9d05682018-04-26 08:26:52 -070026 TCP_OUTPUT_NEXT_IP_REWRITE,
27 TCP_OUTPUT_NEXT_IP_ARP,
Dave Barach68b0fb02017-02-28 15:15:56 -050028 TCP_OUTPUT_N_NEXT
29} tcp_output_next_t;
30
31#define foreach_tcp4_output_next \
32 _ (DROP, "error-drop") \
Florin Corasf9d05682018-04-26 08:26:52 -070033 _ (IP_LOOKUP, "ip4-lookup") \
34 _ (IP_REWRITE, "ip4-rewrite") \
35 _ (IP_ARP, "ip4-arp")
Dave Barach68b0fb02017-02-28 15:15:56 -050036
37#define foreach_tcp6_output_next \
38 _ (DROP, "error-drop") \
Florin Corasf9d05682018-04-26 08:26:52 -070039 _ (IP_LOOKUP, "ip6-lookup") \
40 _ (IP_REWRITE, "ip6-rewrite") \
41 _ (IP_ARP, "ip6-discover-neighbor")
Dave Barach68b0fb02017-02-28 15:15:56 -050042
43static char *tcp_error_strings[] = {
44#define tcp_error(n,s) s,
45#include <vnet/tcp/tcp_error.def>
46#undef tcp_error
47};
48
49typedef struct
50{
Clement Durand6cf260c2017-04-13 13:27:04 +020051 tcp_header_t tcp_header;
52 tcp_connection_t tcp_connection;
Dave Barach68b0fb02017-02-28 15:15:56 -050053} tcp_tx_trace_t;
54
Florin Corasf6d68ed2017-05-07 19:12:02 -070055u16 dummy_mtu = 1460;
Dave Barach68b0fb02017-02-28 15:15:56 -050056
57u8 *
58format_tcp_tx_trace (u8 * s, va_list * args)
59{
60 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
61 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
Clement Durand6cf260c2017-04-13 13:27:04 +020062 tcp_tx_trace_t *t = va_arg (*args, tcp_tx_trace_t *);
Christophe Fontained3c008d2017-10-02 18:10:54 +020063 u32 indent = format_get_indent (s);
Dave Barach68b0fb02017-02-28 15:15:56 -050064
Clement Durand6cf260c2017-04-13 13:27:04 +020065 s = format (s, "%U\n%U%U",
66 format_tcp_header, &t->tcp_header, 128,
67 format_white_space, indent,
Florin Corasbb292f42017-05-19 09:49:19 -070068 format_tcp_connection, &t->tcp_connection, 1);
Dave Barach68b0fb02017-02-28 15:15:56 -050069
70 return s;
71}
72
Dave Barach68b0fb02017-02-28 15:15:56 -050073static u8
Florin Coras4eeeaaf2017-09-05 14:03:37 -040074tcp_window_compute_scale (u32 window)
Dave Barach68b0fb02017-02-28 15:15:56 -050075{
76 u8 wnd_scale = 0;
Florin Coras4eeeaaf2017-09-05 14:03:37 -040077 while (wnd_scale < TCP_MAX_WND_SCALE && (window >> wnd_scale) > TCP_WND_MAX)
Dave Barach68b0fb02017-02-28 15:15:56 -050078 wnd_scale++;
79 return wnd_scale;
80}
81
82/**
Florin Coras6534b7a2017-07-18 05:38:03 -040083 * Update max segment size we're able to process.
84 *
85 * The value is constrained by our interface's MTU and IP options. It is
86 * also what we advertise to our peer.
87 */
88void
89tcp_update_rcv_mss (tcp_connection_t * tc)
90{
91 /* TODO find our iface MTU */
Florin Corasb2215d62017-08-01 16:56:58 -070092 tc->mss = dummy_mtu - sizeof (tcp_header_t);
Florin Coras6534b7a2017-07-18 05:38:03 -040093}
94
95/**
96 * TCP's initial window
Florin Corase04c2992017-03-01 08:17:34 -080097 */
98always_inline u32
99tcp_initial_wnd_unscaled (tcp_connection_t * tc)
100{
Florin Coras6534b7a2017-07-18 05:38:03 -0400101 /* RFC 6928 recommends the value lower. However at the time our connections
102 * are initialized, fifos may not be allocated. Therefore, advertise the
103 * smallest possible unscaled window size and update once fifos are
104 * assigned to the session.
105 */
106 /*
107 tcp_update_rcv_mss (tc);
108 TCP_IW_N_SEGMENTS * tc->mss;
109 */
110 return TCP_MIN_RX_FIFO_SIZE;
Florin Corase04c2992017-03-01 08:17:34 -0800111}
112
113/**
Dave Barach68b0fb02017-02-28 15:15:56 -0500114 * Compute initial window and scale factor. As per RFC1323, window field in
115 * SYN and SYN-ACK segments is never scaled.
116 */
117u32
118tcp_initial_window_to_advertise (tcp_connection_t * tc)
119{
Florin Corasca031862018-09-24 13:58:05 -0700120 tcp_main_t *tm = &tcp_main;
Florin Corase04c2992017-03-01 08:17:34 -0800121 u32 max_fifo;
Dave Barach68b0fb02017-02-28 15:15:56 -0500122
123 /* Initial wnd for SYN. Fifos are not allocated yet.
Florin Corase04c2992017-03-01 08:17:34 -0800124 * Use some predefined value. For SYN-ACK we still want the
125 * scale to be computed in the same way */
Florin Corasca031862018-09-24 13:58:05 -0700126 max_fifo = tm->max_rx_fifo ? tm->max_rx_fifo : TCP_MAX_RX_FIFO_SIZE;
Dave Barach68b0fb02017-02-28 15:15:56 -0500127
Florin Corase80b5912018-12-12 19:25:43 -0800128 /* Compute rcv wscale only if peer advertised support for it */
129 if (tc->state != TCP_STATE_SYN_RCVD || tcp_opts_wscale (&tc->rcv_opts))
130 tc->rcv_wscale = tcp_window_compute_scale (max_fifo);
131
Florin Corase04c2992017-03-01 08:17:34 -0800132 tc->rcv_wnd = tcp_initial_wnd_unscaled (tc);
Dave Barach68b0fb02017-02-28 15:15:56 -0500133
134 return clib_min (tc->rcv_wnd, TCP_WND_MAX);
135}
136
Florin Coras0dbd5172018-06-25 16:19:34 -0700137static void
Florin Coras6792ec02017-03-13 03:49:51 -0700138tcp_update_rcv_wnd (tcp_connection_t * tc)
139{
140 i32 observed_wnd;
141 u32 available_space, max_fifo, wnd;
142
Florin Corase04c2992017-03-01 08:17:34 -0800143 /*
144 * Figure out how much space we have available
145 */
Florin Corasd2aab832018-05-22 11:39:59 -0700146 available_space = transport_max_rx_enqueue (&tc->connection);
147 max_fifo = transport_rx_fifo_size (&tc->connection);
Dave Barach68b0fb02017-02-28 15:15:56 -0500148
Florin Coras93992a92017-05-24 18:03:56 -0700149 ASSERT (tc->rcv_opts.mss < max_fifo);
150 if (available_space < tc->rcv_opts.mss && available_space < max_fifo >> 3)
Florin Corase04c2992017-03-01 08:17:34 -0800151 available_space = 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500152
Florin Corase04c2992017-03-01 08:17:34 -0800153 /*
154 * Use the above and what we know about what we've previously advertised
155 * to compute the new window
156 */
Florin Coras6792ec02017-03-13 03:49:51 -0700157 observed_wnd = (i32) tc->rcv_wnd - (tc->rcv_nxt - tc->rcv_las);
158 if (observed_wnd < 0)
159 observed_wnd = 0;
Florin Corase04c2992017-03-01 08:17:34 -0800160
161 /* Bad. Thou shalt not shrink */
162 if (available_space < observed_wnd)
163 {
Florin Coras6792ec02017-03-13 03:49:51 -0700164 wnd = observed_wnd;
Florin Coras3e350af2017-03-30 02:54:28 -0700165 TCP_EVT_DBG (TCP_EVT_RCV_WND_SHRUNK, tc, observed_wnd, available_space);
Florin Corase04c2992017-03-01 08:17:34 -0800166 }
Florin Coras6792ec02017-03-13 03:49:51 -0700167 else
Florin Corase04c2992017-03-01 08:17:34 -0800168 {
Florin Coras6792ec02017-03-13 03:49:51 -0700169 wnd = available_space;
Florin Corase04c2992017-03-01 08:17:34 -0800170 }
171
Florin Coras3e350af2017-03-30 02:54:28 -0700172 /* Make sure we have a multiple of rcv_wscale */
173 if (wnd && tc->rcv_wscale)
174 {
175 wnd &= ~(1 << tc->rcv_wscale);
176 if (wnd == 0)
177 wnd = 1 << tc->rcv_wscale;
178 }
Florin Coras6792ec02017-03-13 03:49:51 -0700179
180 tc->rcv_wnd = clib_min (wnd, TCP_WND_MAX << tc->rcv_wscale);
Dave Barach68b0fb02017-02-28 15:15:56 -0500181}
182
183/**
Florin Coras0dbd5172018-06-25 16:19:34 -0700184 * Compute and return window to advertise, scaled as per RFC1323
185 */
186static u32
187tcp_window_to_advertise (tcp_connection_t * tc, tcp_state_t state)
188{
189 if (state < TCP_STATE_ESTABLISHED)
190 return tcp_initial_window_to_advertise (tc);
191
192 tcp_update_rcv_wnd (tc);
193
194 if (tc->rcv_wnd == 0)
195 {
196 tc->flags |= TCP_CONN_SENT_RCV_WND0;
197 }
198 else
199 {
200 tc->flags &= ~TCP_CONN_SENT_RCV_WND0;
201 }
202
203 return tc->rcv_wnd >> tc->rcv_wscale;
204}
205
206/**
Dave Barach68b0fb02017-02-28 15:15:56 -0500207 * Write TCP options to segment.
208 */
Florin Coras0dbd5172018-06-25 16:19:34 -0700209static u32
Dave Barach68b0fb02017-02-28 15:15:56 -0500210tcp_options_write (u8 * data, tcp_options_t * opts)
211{
212 u32 opts_len = 0;
213 u32 buf, seq_len = 4;
214
215 if (tcp_opts_mss (opts))
216 {
217 *data++ = TCP_OPTION_MSS;
218 *data++ = TCP_OPTION_LEN_MSS;
219 buf = clib_host_to_net_u16 (opts->mss);
Dave Barach178cf492018-11-13 16:34:13 -0500220 clib_memcpy_fast (data, &buf, sizeof (opts->mss));
Dave Barach68b0fb02017-02-28 15:15:56 -0500221 data += sizeof (opts->mss);
222 opts_len += TCP_OPTION_LEN_MSS;
223 }
224
225 if (tcp_opts_wscale (opts))
226 {
227 *data++ = TCP_OPTION_WINDOW_SCALE;
228 *data++ = TCP_OPTION_LEN_WINDOW_SCALE;
229 *data++ = opts->wscale;
230 opts_len += TCP_OPTION_LEN_WINDOW_SCALE;
231 }
232
233 if (tcp_opts_sack_permitted (opts))
234 {
235 *data++ = TCP_OPTION_SACK_PERMITTED;
236 *data++ = TCP_OPTION_LEN_SACK_PERMITTED;
237 opts_len += TCP_OPTION_LEN_SACK_PERMITTED;
238 }
239
240 if (tcp_opts_tstamp (opts))
241 {
242 *data++ = TCP_OPTION_TIMESTAMP;
243 *data++ = TCP_OPTION_LEN_TIMESTAMP;
244 buf = clib_host_to_net_u32 (opts->tsval);
Dave Barach178cf492018-11-13 16:34:13 -0500245 clib_memcpy_fast (data, &buf, sizeof (opts->tsval));
Dave Barach68b0fb02017-02-28 15:15:56 -0500246 data += sizeof (opts->tsval);
247 buf = clib_host_to_net_u32 (opts->tsecr);
Dave Barach178cf492018-11-13 16:34:13 -0500248 clib_memcpy_fast (data, &buf, sizeof (opts->tsecr));
Dave Barach68b0fb02017-02-28 15:15:56 -0500249 data += sizeof (opts->tsecr);
250 opts_len += TCP_OPTION_LEN_TIMESTAMP;
251 }
252
253 if (tcp_opts_sack (opts))
254 {
255 int i;
256 u32 n_sack_blocks = clib_min (vec_len (opts->sacks),
257 TCP_OPTS_MAX_SACK_BLOCKS);
258
259 if (n_sack_blocks != 0)
260 {
261 *data++ = TCP_OPTION_SACK_BLOCK;
262 *data++ = 2 + n_sack_blocks * TCP_OPTION_LEN_SACK_BLOCK;
263 for (i = 0; i < n_sack_blocks; i++)
264 {
265 buf = clib_host_to_net_u32 (opts->sacks[i].start);
Dave Barach178cf492018-11-13 16:34:13 -0500266 clib_memcpy_fast (data, &buf, seq_len);
Dave Barach68b0fb02017-02-28 15:15:56 -0500267 data += seq_len;
268 buf = clib_host_to_net_u32 (opts->sacks[i].end);
Dave Barach178cf492018-11-13 16:34:13 -0500269 clib_memcpy_fast (data, &buf, seq_len);
Dave Barach68b0fb02017-02-28 15:15:56 -0500270 data += seq_len;
271 }
272 opts_len += 2 + n_sack_blocks * TCP_OPTION_LEN_SACK_BLOCK;
273 }
274 }
275
276 /* Terminate TCP options */
277 if (opts_len % 4)
278 {
279 *data++ = TCP_OPTION_EOL;
280 opts_len += TCP_OPTION_LEN_EOL;
281 }
282
283 /* Pad with zeroes to a u32 boundary */
284 while (opts_len % 4)
285 {
286 *data++ = TCP_OPTION_NOOP;
287 opts_len += TCP_OPTION_LEN_NOOP;
288 }
289 return opts_len;
290}
291
Florin Coras0dbd5172018-06-25 16:19:34 -0700292static int
Florin Corase04c2992017-03-01 08:17:34 -0800293tcp_make_syn_options (tcp_options_t * opts, u8 wnd_scale)
Dave Barach68b0fb02017-02-28 15:15:56 -0500294{
295 u8 len = 0;
296
297 opts->flags |= TCP_OPTS_FLAG_MSS;
298 opts->mss = dummy_mtu; /*XXX discover that */
299 len += TCP_OPTION_LEN_MSS;
300
301 opts->flags |= TCP_OPTS_FLAG_WSCALE;
Florin Corase04c2992017-03-01 08:17:34 -0800302 opts->wscale = wnd_scale;
Dave Barach68b0fb02017-02-28 15:15:56 -0500303 len += TCP_OPTION_LEN_WINDOW_SCALE;
304
305 opts->flags |= TCP_OPTS_FLAG_TSTAMP;
306 opts->tsval = tcp_time_now ();
307 opts->tsecr = 0;
308 len += TCP_OPTION_LEN_TIMESTAMP;
309
Florin Coras93992a92017-05-24 18:03:56 -0700310 if (TCP_USE_SACKS)
311 {
312 opts->flags |= TCP_OPTS_FLAG_SACK_PERMITTED;
313 len += TCP_OPTION_LEN_SACK_PERMITTED;
314 }
Dave Barach68b0fb02017-02-28 15:15:56 -0500315
316 /* Align to needed boundary */
317 len += (TCP_OPTS_ALIGN - len % TCP_OPTS_ALIGN) % TCP_OPTS_ALIGN;
318 return len;
319}
320
Florin Coras0dbd5172018-06-25 16:19:34 -0700321static int
Dave Barach68b0fb02017-02-28 15:15:56 -0500322tcp_make_synack_options (tcp_connection_t * tc, tcp_options_t * opts)
323{
324 u8 len = 0;
325
326 opts->flags |= TCP_OPTS_FLAG_MSS;
Florin Corasc8343412017-05-04 14:25:50 -0700327 opts->mss = tc->mss;
Dave Barach68b0fb02017-02-28 15:15:56 -0500328 len += TCP_OPTION_LEN_MSS;
329
Florin Coras93992a92017-05-24 18:03:56 -0700330 if (tcp_opts_wscale (&tc->rcv_opts))
Dave Barach68b0fb02017-02-28 15:15:56 -0500331 {
332 opts->flags |= TCP_OPTS_FLAG_WSCALE;
333 opts->wscale = tc->rcv_wscale;
334 len += TCP_OPTION_LEN_WINDOW_SCALE;
335 }
336
Florin Coras93992a92017-05-24 18:03:56 -0700337 if (tcp_opts_tstamp (&tc->rcv_opts))
Dave Barach68b0fb02017-02-28 15:15:56 -0500338 {
339 opts->flags |= TCP_OPTS_FLAG_TSTAMP;
340 opts->tsval = tcp_time_now ();
341 opts->tsecr = tc->tsval_recent;
342 len += TCP_OPTION_LEN_TIMESTAMP;
343 }
344
Florin Coras93992a92017-05-24 18:03:56 -0700345 if (tcp_opts_sack_permitted (&tc->rcv_opts))
Dave Barach68b0fb02017-02-28 15:15:56 -0500346 {
347 opts->flags |= TCP_OPTS_FLAG_SACK_PERMITTED;
348 len += TCP_OPTION_LEN_SACK_PERMITTED;
349 }
350
351 /* Align to needed boundary */
352 len += (TCP_OPTS_ALIGN - len % TCP_OPTS_ALIGN) % TCP_OPTS_ALIGN;
353 return len;
354}
355
Florin Coras0dbd5172018-06-25 16:19:34 -0700356static int
Dave Barach68b0fb02017-02-28 15:15:56 -0500357tcp_make_established_options (tcp_connection_t * tc, tcp_options_t * opts)
358{
359 u8 len = 0;
360
361 opts->flags = 0;
362
Florin Coras93992a92017-05-24 18:03:56 -0700363 if (tcp_opts_tstamp (&tc->rcv_opts))
Dave Barach68b0fb02017-02-28 15:15:56 -0500364 {
365 opts->flags |= TCP_OPTS_FLAG_TSTAMP;
Florin Corasbe72ae62018-11-01 11:23:03 -0700366 opts->tsval = tcp_time_now_w_thread (tc->c_thread_index);
Dave Barach68b0fb02017-02-28 15:15:56 -0500367 opts->tsecr = tc->tsval_recent;
368 len += TCP_OPTION_LEN_TIMESTAMP;
369 }
Florin Coras93992a92017-05-24 18:03:56 -0700370 if (tcp_opts_sack_permitted (&tc->rcv_opts))
Dave Barach68b0fb02017-02-28 15:15:56 -0500371 {
372 if (vec_len (tc->snd_sacks))
373 {
374 opts->flags |= TCP_OPTS_FLAG_SACK;
375 opts->sacks = tc->snd_sacks;
Florin Corasc28764f2017-04-26 00:08:42 -0700376 opts->n_sack_blocks = clib_min (vec_len (tc->snd_sacks),
377 TCP_OPTS_MAX_SACK_BLOCKS);
Dave Barach68b0fb02017-02-28 15:15:56 -0500378 len += 2 + TCP_OPTION_LEN_SACK_BLOCK * opts->n_sack_blocks;
379 }
380 }
381
382 /* Align to needed boundary */
383 len += (TCP_OPTS_ALIGN - len % TCP_OPTS_ALIGN) % TCP_OPTS_ALIGN;
384 return len;
385}
386
387always_inline int
388tcp_make_options (tcp_connection_t * tc, tcp_options_t * opts,
389 tcp_state_t state)
390{
391 switch (state)
392 {
393 case TCP_STATE_ESTABLISHED:
Florin Coras25579b42018-06-06 17:55:02 -0700394 case TCP_STATE_CLOSE_WAIT:
Florin Coras54ddf432018-12-21 13:54:09 -0800395 case TCP_STATE_FIN_WAIT_1:
396 case TCP_STATE_LAST_ACK:
397 case TCP_STATE_CLOSING:
398 case TCP_STATE_FIN_WAIT_2:
399 case TCP_STATE_TIME_WAIT:
400 case TCP_STATE_CLOSED:
Dave Barach68b0fb02017-02-28 15:15:56 -0500401 return tcp_make_established_options (tc, opts);
402 case TCP_STATE_SYN_RCVD:
403 return tcp_make_synack_options (tc, opts);
404 case TCP_STATE_SYN_SENT:
Florin Corase04c2992017-03-01 08:17:34 -0800405 return tcp_make_syn_options (opts, tc->rcv_wscale);
Dave Barach68b0fb02017-02-28 15:15:56 -0500406 default:
Florin Coras371ca502018-02-21 12:07:41 -0800407 clib_warning ("State not handled! %d", state);
Dave Barach68b0fb02017-02-28 15:15:56 -0500408 return 0;
409 }
410}
411
Florin Corasc8343412017-05-04 14:25:50 -0700412/**
Florin Corasb26743d2018-06-26 09:31:04 -0700413 * Update burst send vars
414 *
415 * - Updates snd_mss to reflect the effective segment size that we can send
416 * by taking into account all TCP options, including SACKs.
417 * - Cache 'on the wire' options for reuse
418 * - Updates receive window which can be reused for a burst.
419 *
420 * This should *only* be called when doing bursts
Florin Corasc8343412017-05-04 14:25:50 -0700421 */
422void
Florin Corasb26743d2018-06-26 09:31:04 -0700423tcp_update_burst_snd_vars (tcp_connection_t * tc)
Florin Corasc8343412017-05-04 14:25:50 -0700424{
Florin Corasb26743d2018-06-26 09:31:04 -0700425 tcp_main_t *tm = &tcp_main;
426
Florin Corasc8343412017-05-04 14:25:50 -0700427 /* Compute options to be used for connection. These may be reused when
428 * sending data or to compute the effective mss (snd_mss) */
Florin Corasb26743d2018-06-26 09:31:04 -0700429 tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts,
430 TCP_STATE_ESTABLISHED);
Florin Corasc8343412017-05-04 14:25:50 -0700431
432 /* XXX check if MTU has been updated */
Florin Coras93992a92017-05-24 18:03:56 -0700433 tc->snd_mss = clib_min (tc->mss, tc->rcv_opts.mss) - tc->snd_opts_len;
Florin Corasdb84e572017-05-09 18:54:52 -0700434 ASSERT (tc->snd_mss > 0);
Florin Corasb26743d2018-06-26 09:31:04 -0700435
436 tcp_options_write (tm->wrk_ctx[tc->c_thread_index].cached_opts,
437 &tc->snd_opts);
438
439 tcp_update_rcv_wnd (tc);
Florin Corasc8343412017-05-04 14:25:50 -0700440}
441
442void
443tcp_init_mss (tcp_connection_t * tc)
444{
Florin Corasdb84e572017-05-09 18:54:52 -0700445 u16 default_min_mss = 536;
Florin Corasc8343412017-05-04 14:25:50 -0700446 tcp_update_rcv_mss (tc);
447
448 /* TODO cache mss and consider PMTU discovery */
Florin Coras93992a92017-05-24 18:03:56 -0700449 tc->snd_mss = clib_min (tc->rcv_opts.mss, tc->mss);
Florin Corasc8343412017-05-04 14:25:50 -0700450
Florin Corasdb84e572017-05-09 18:54:52 -0700451 if (tc->snd_mss < 45)
Florin Corasc8343412017-05-04 14:25:50 -0700452 {
Florin Corasdb84e572017-05-09 18:54:52 -0700453 /* Assume that at least the min default mss works */
454 tc->snd_mss = default_min_mss;
Florin Coras93992a92017-05-24 18:03:56 -0700455 tc->rcv_opts.mss = default_min_mss;
Florin Corasc8343412017-05-04 14:25:50 -0700456 }
457
458 /* We should have enough space for 40 bytes of options */
459 ASSERT (tc->snd_mss > 45);
460
461 /* If we use timestamp option, account for it */
Florin Coras93992a92017-05-24 18:03:56 -0700462 if (tcp_opts_tstamp (&tc->rcv_opts))
Florin Corasc8343412017-05-04 14:25:50 -0700463 tc->snd_mss -= TCP_OPTION_LEN_TIMESTAMP;
464}
465
Florin Coras0dbd5172018-06-25 16:19:34 -0700466static void *
Dave Barach68b0fb02017-02-28 15:15:56 -0500467tcp_reuse_buffer (vlib_main_t * vm, vlib_buffer_t * b)
468{
Florin Corasb2215d62017-08-01 16:56:58 -0700469 if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
470 vlib_buffer_free_one (vm, b->next_buffer);
Florin Coras4eeeaaf2017-09-05 14:03:37 -0400471 /* Zero all flags but free list index and trace flag */
472 b->flags &= VLIB_BUFFER_NEXT_PRESENT - 1;
Florin Coras1f152cd2017-08-18 19:28:03 -0700473 b->current_data = 0;
474 b->current_length = 0;
475 b->total_length_not_including_first_buffer = 0;
476 vnet_buffer (b)->tcp.flags = 0;
Florin Corasb2215d62017-08-01 16:56:58 -0700477
Dave Barach68b0fb02017-02-28 15:15:56 -0500478 /* Leave enough space for headers */
Florin Coras1ee78302019-02-05 15:51:15 -0800479 return vlib_buffer_make_headroom (b, TRANSPORT_MAX_HDRS_LEN);
Florin Coras1f152cd2017-08-18 19:28:03 -0700480}
481
Florin Coras0dbd5172018-06-25 16:19:34 -0700482static void *
Florin Coras1f152cd2017-08-18 19:28:03 -0700483tcp_init_buffer (vlib_main_t * vm, vlib_buffer_t * b)
484{
485 ASSERT ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
Florin Coras4eeeaaf2017-09-05 14:03:37 -0400486 b->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
Florin Coras1f152cd2017-08-18 19:28:03 -0700487 b->total_length_not_including_first_buffer = 0;
Florin Coras24793e32018-05-09 11:34:25 -0700488 b->current_data = 0;
Florin Corasd79b41e2017-03-04 05:37:52 -0800489 vnet_buffer (b)->tcp.flags = 0;
Florin Corasf988e692017-11-27 04:34:14 -0500490 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b);
Florin Coras1f152cd2017-08-18 19:28:03 -0700491 /* Leave enough space for headers */
Florin Coras1ee78302019-02-05 15:51:15 -0800492 return vlib_buffer_make_headroom (b, TRANSPORT_MAX_HDRS_LEN);
Dave Barach68b0fb02017-02-28 15:15:56 -0500493}
494
495/**
496 * Prepare ACK
497 */
Florin Corasbdf7fd62019-01-31 17:31:01 -0800498static inline void
Dave Barach68b0fb02017-02-28 15:15:56 -0500499tcp_make_ack_i (tcp_connection_t * tc, vlib_buffer_t * b, tcp_state_t state,
500 u8 flags)
501{
502 tcp_options_t _snd_opts, *snd_opts = &_snd_opts;
503 u8 tcp_opts_len, tcp_hdr_opts_len;
504 tcp_header_t *th;
505 u16 wnd;
506
507 wnd = tcp_window_to_advertise (tc, state);
508
509 /* Make and write options */
510 tcp_opts_len = tcp_make_established_options (tc, snd_opts);
511 tcp_hdr_opts_len = tcp_opts_len + sizeof (tcp_header_t);
512
513 th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, tc->snd_nxt,
514 tc->rcv_nxt, tcp_hdr_opts_len, flags, wnd);
515
516 tcp_options_write ((u8 *) (th + 1), snd_opts);
Dave Barach68b0fb02017-02-28 15:15:56 -0500517 vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
518}
519
520/**
521 * Convert buffer to ACK
522 */
Florin Corasbdf7fd62019-01-31 17:31:01 -0800523static inline void
Dave Barach68b0fb02017-02-28 15:15:56 -0500524tcp_make_ack (tcp_connection_t * tc, vlib_buffer_t * b)
525{
Dave Barach68b0fb02017-02-28 15:15:56 -0500526 tcp_make_ack_i (tc, b, TCP_STATE_ESTABLISHED, TCP_FLAG_ACK);
Florin Coras6792ec02017-03-13 03:49:51 -0700527 TCP_EVT_DBG (TCP_EVT_ACK_SENT, tc);
Florin Coras3e350af2017-03-30 02:54:28 -0700528 tc->rcv_las = tc->rcv_nxt;
Dave Barach68b0fb02017-02-28 15:15:56 -0500529}
530
531/**
532 * Convert buffer to FIN-ACK
533 */
534void
Florin Corasd79b41e2017-03-04 05:37:52 -0800535tcp_make_fin (tcp_connection_t * tc, vlib_buffer_t * b)
Dave Barach68b0fb02017-02-28 15:15:56 -0500536{
Florin Corasbdf7fd62019-01-31 17:31:01 -0800537 tcp_make_ack_i (tc, b, TCP_STATE_ESTABLISHED, TCP_FLAG_FIN | TCP_FLAG_ACK);
Dave Barach68b0fb02017-02-28 15:15:56 -0500538
539 /* Reset flags, make sure ack is sent */
Dave Barach68b0fb02017-02-28 15:15:56 -0500540 vnet_buffer (b)->tcp.flags &= ~TCP_BUF_FLAG_DUPACK;
Florin Coras4eeeaaf2017-09-05 14:03:37 -0400541}
Dave Barach68b0fb02017-02-28 15:15:56 -0500542
Florin Coras4eeeaaf2017-09-05 14:03:37 -0400543/**
544 * Convert buffer to SYN
545 */
546void
547tcp_make_syn (tcp_connection_t * tc, vlib_buffer_t * b)
548{
549 u8 tcp_hdr_opts_len, tcp_opts_len;
550 tcp_header_t *th;
551 u16 initial_wnd;
552 tcp_options_t snd_opts;
553
554 initial_wnd = tcp_initial_window_to_advertise (tc);
555
556 /* Make and write options */
Dave Barachb7b92992018-10-17 10:38:51 -0400557 clib_memset (&snd_opts, 0, sizeof (snd_opts));
Florin Coras4eeeaaf2017-09-05 14:03:37 -0400558 tcp_opts_len = tcp_make_syn_options (&snd_opts, tc->rcv_wscale);
559 tcp_hdr_opts_len = tcp_opts_len + sizeof (tcp_header_t);
560
561 th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, tc->iss,
562 tc->rcv_nxt, tcp_hdr_opts_len, TCP_FLAG_SYN,
563 initial_wnd);
564 vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
565 tcp_options_write ((u8 *) (th + 1), &snd_opts);
Dave Barach68b0fb02017-02-28 15:15:56 -0500566}
567
568/**
569 * Convert buffer to SYN-ACK
570 */
571void
572tcp_make_synack (tcp_connection_t * tc, vlib_buffer_t * b)
573{
Dave Barach68b0fb02017-02-28 15:15:56 -0500574 tcp_options_t _snd_opts, *snd_opts = &_snd_opts;
575 u8 tcp_opts_len, tcp_hdr_opts_len;
576 tcp_header_t *th;
577 u16 initial_wnd;
Dave Barach68b0fb02017-02-28 15:15:56 -0500578
Dave Barachb7b92992018-10-17 10:38:51 -0400579 clib_memset (snd_opts, 0, sizeof (*snd_opts));
Dave Barach68b0fb02017-02-28 15:15:56 -0500580 initial_wnd = tcp_initial_window_to_advertise (tc);
Dave Barach68b0fb02017-02-28 15:15:56 -0500581 tcp_opts_len = tcp_make_synack_options (tc, snd_opts);
582 tcp_hdr_opts_len = tcp_opts_len + sizeof (tcp_header_t);
583
584 th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, tc->iss,
585 tc->rcv_nxt, tcp_hdr_opts_len,
586 TCP_FLAG_SYN | TCP_FLAG_ACK, initial_wnd);
Dave Barach68b0fb02017-02-28 15:15:56 -0500587 tcp_options_write ((u8 *) (th + 1), snd_opts);
588
589 vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
Dave Barach68b0fb02017-02-28 15:15:56 -0500590}
591
592always_inline void
Florin Corasbe72ae62018-11-01 11:23:03 -0700593tcp_enqueue_to_ip_lookup_i (tcp_worker_ctx_t * wrk, vlib_buffer_t * b, u32 bi,
Florin Coras56b39f62018-03-27 17:29:32 -0700594 u8 is_ip4, u32 fib_index, u8 flush)
Dave Barach68b0fb02017-02-28 15:15:56 -0500595{
Florin Corasbe72ae62018-11-01 11:23:03 -0700596 vlib_main_t *vm = wrk->vm;
Dave Barach68b0fb02017-02-28 15:15:56 -0500597 u32 *to_next, next_index;
598 vlib_frame_t *f;
599
Damjan Marion213b5aa2017-07-13 21:19:27 +0200600 b->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
Dave Barach68b0fb02017-02-28 15:15:56 -0500601 b->error = 0;
602
Florin Coras56b39f62018-03-27 17:29:32 -0700603 vnet_buffer (b)->sw_if_index[VLIB_TX] = fib_index;
604 vnet_buffer (b)->sw_if_index[VLIB_RX] = 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500605
606 /* Send to IP lookup */
607 next_index = is_ip4 ? ip4_lookup_node.index : ip6_lookup_node.index;
Florin Corasf988e692017-11-27 04:34:14 -0500608 tcp_trajectory_add_start (b, 1);
Dave Barach68b0fb02017-02-28 15:15:56 -0500609
Florin Corasbe72ae62018-11-01 11:23:03 -0700610 f = wrk->ip_lookup_tx_frames[!is_ip4];
Florin Coras9d063042017-09-14 03:08:00 -0400611 if (!f)
612 {
613 f = vlib_get_frame_to_node (vm, next_index);
614 ASSERT (f);
Florin Corasbe72ae62018-11-01 11:23:03 -0700615 wrk->ip_lookup_tx_frames[!is_ip4] = f;
Florin Coras9d063042017-09-14 03:08:00 -0400616 }
617
Dave Barach68b0fb02017-02-28 15:15:56 -0500618 to_next = vlib_frame_vector_args (f);
Florin Coras9d063042017-09-14 03:08:00 -0400619 to_next[f->n_vectors] = bi;
620 f->n_vectors += 1;
621 if (flush || f->n_vectors == VLIB_FRAME_SIZE)
622 {
623 vlib_put_frame_to_node (vm, next_index, f);
Florin Corasbe72ae62018-11-01 11:23:03 -0700624 wrk->ip_lookup_tx_frames[!is_ip4] = 0;
Florin Coras9d063042017-09-14 03:08:00 -0400625 }
626}
627
Florin Coras0dbd5172018-06-25 16:19:34 -0700628static void
Florin Corasbe72ae62018-11-01 11:23:03 -0700629tcp_enqueue_to_ip_lookup_now (tcp_worker_ctx_t * wrk, vlib_buffer_t * b,
630 u32 bi, u8 is_ip4, u32 fib_index)
Florin Coras9d063042017-09-14 03:08:00 -0400631{
Florin Corasbe72ae62018-11-01 11:23:03 -0700632 tcp_enqueue_to_ip_lookup_i (wrk, b, bi, is_ip4, fib_index, 1);
Florin Coras9d063042017-09-14 03:08:00 -0400633}
634
Florin Coras0dbd5172018-06-25 16:19:34 -0700635static void
Florin Corasbe72ae62018-11-01 11:23:03 -0700636tcp_enqueue_to_ip_lookup (tcp_worker_ctx_t * wrk, vlib_buffer_t * b, u32 bi,
Florin Coras56b39f62018-03-27 17:29:32 -0700637 u8 is_ip4, u32 fib_index)
Florin Coras9d063042017-09-14 03:08:00 -0400638{
Florin Corasbe72ae62018-11-01 11:23:03 -0700639 tcp_enqueue_to_ip_lookup_i (wrk, b, bi, is_ip4, fib_index, 0);
640 if (wrk->vm->thread_index == 0 && vlib_num_workers ())
641 session_flush_frames_main_thread (wrk->vm);
Dave Barach68b0fb02017-02-28 15:15:56 -0500642}
643
Florin Coras1f152cd2017-08-18 19:28:03 -0700644always_inline void
Florin Corasbe72ae62018-11-01 11:23:03 -0700645tcp_enqueue_to_output_i (tcp_worker_ctx_t * wrk, vlib_buffer_t * b, u32 bi,
Florin Coras1f152cd2017-08-18 19:28:03 -0700646 u8 is_ip4, u8 flush)
647{
Florin Coras1f152cd2017-08-18 19:28:03 -0700648 u32 *to_next, next_index;
649 vlib_frame_t *f;
650
651 b->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
652 b->error = 0;
653
654 /* Decide where to send the packet */
655 next_index = is_ip4 ? tcp4_output_node.index : tcp6_output_node.index;
Florin Corasf988e692017-11-27 04:34:14 -0500656 tcp_trajectory_add_start (b, 2);
Florin Coras1f152cd2017-08-18 19:28:03 -0700657
658 /* Get frame to v4/6 output node */
Florin Corasbe72ae62018-11-01 11:23:03 -0700659 f = wrk->tx_frames[!is_ip4];
Florin Coras1f152cd2017-08-18 19:28:03 -0700660 if (!f)
661 {
Florin Corasbe72ae62018-11-01 11:23:03 -0700662 f = vlib_get_frame_to_node (wrk->vm, next_index);
Florin Coras1f152cd2017-08-18 19:28:03 -0700663 ASSERT (f);
Florin Corasbe72ae62018-11-01 11:23:03 -0700664 wrk->tx_frames[!is_ip4] = f;
Florin Coras1f152cd2017-08-18 19:28:03 -0700665 }
666 to_next = vlib_frame_vector_args (f);
667 to_next[f->n_vectors] = bi;
668 f->n_vectors += 1;
669 if (flush || f->n_vectors == VLIB_FRAME_SIZE)
670 {
Florin Corasbe72ae62018-11-01 11:23:03 -0700671 vlib_put_frame_to_node (wrk->vm, next_index, f);
672 wrk->tx_frames[!is_ip4] = 0;
Florin Coras1f152cd2017-08-18 19:28:03 -0700673 }
674}
675
Florin Coras0dbd5172018-06-25 16:19:34 -0700676static void
Florin Corasbe72ae62018-11-01 11:23:03 -0700677tcp_enqueue_to_output (tcp_worker_ctx_t * wrk, vlib_buffer_t * b, u32 bi,
678 u8 is_ip4)
Florin Coras1f152cd2017-08-18 19:28:03 -0700679{
Florin Corasbe72ae62018-11-01 11:23:03 -0700680 tcp_enqueue_to_output_i (wrk, b, bi, is_ip4, 0);
Florin Coras1f152cd2017-08-18 19:28:03 -0700681}
682
Florin Coras0dbd5172018-06-25 16:19:34 -0700683static void
Florin Corasbe72ae62018-11-01 11:23:03 -0700684tcp_enqueue_to_output_now (tcp_worker_ctx_t * wrk, vlib_buffer_t * b, u32 bi,
Florin Coras1f152cd2017-08-18 19:28:03 -0700685 u8 is_ip4)
686{
Florin Corasbe72ae62018-11-01 11:23:03 -0700687 tcp_enqueue_to_output_i (wrk, b, bi, is_ip4, 1);
Florin Coras1f152cd2017-08-18 19:28:03 -0700688}
689
Florin Coras0dbd5172018-06-25 16:19:34 -0700690static int
Dave Barach68b0fb02017-02-28 15:15:56 -0500691tcp_make_reset_in_place (vlib_main_t * vm, vlib_buffer_t * b0,
Florin Corasdc629cd2017-05-09 00:52:37 -0700692 tcp_state_t state, u8 thread_index, u8 is_ip4)
Dave Barach68b0fb02017-02-28 15:15:56 -0500693{
Dave Barach68b0fb02017-02-28 15:15:56 -0500694 ip4_header_t *ih4;
695 ip6_header_t *ih6;
696 tcp_header_t *th0;
Florin Corasdc629cd2017-05-09 00:52:37 -0700697 ip4_address_t src_ip40, dst_ip40;
698 ip6_address_t src_ip60, dst_ip60;
699 u16 src_port, dst_port;
Dave Barach68b0fb02017-02-28 15:15:56 -0500700 u32 tmp;
Florin Corasdc629cd2017-05-09 00:52:37 -0700701 u32 seq, ack;
702 u8 flags;
Dave Barach68b0fb02017-02-28 15:15:56 -0500703
704 /* Find IP and TCP headers */
Florin Corasdc629cd2017-05-09 00:52:37 -0700705 th0 = tcp_buffer_hdr (b0);
706
707 /* Save src and dst ip */
Dave Barach68b0fb02017-02-28 15:15:56 -0500708 if (is_ip4)
709 {
710 ih4 = vlib_buffer_get_current (b0);
Florin Corasdc629cd2017-05-09 00:52:37 -0700711 ASSERT ((ih4->ip_version_and_header_length & 0xF0) == 0x40);
712 src_ip40.as_u32 = ih4->src_address.as_u32;
713 dst_ip40.as_u32 = ih4->dst_address.as_u32;
Dave Barach68b0fb02017-02-28 15:15:56 -0500714 }
715 else
716 {
717 ih6 = vlib_buffer_get_current (b0);
Dave Barach68b0fb02017-02-28 15:15:56 -0500718 ASSERT ((ih6->ip_version_traffic_class_and_flow_label & 0xF0) == 0x60);
Dave Barach178cf492018-11-13 16:34:13 -0500719 clib_memcpy_fast (&src_ip60, &ih6->src_address, sizeof (ip6_address_t));
720 clib_memcpy_fast (&dst_ip60, &ih6->dst_address, sizeof (ip6_address_t));
Dave Barach68b0fb02017-02-28 15:15:56 -0500721 }
722
Florin Corasdc629cd2017-05-09 00:52:37 -0700723 src_port = th0->src_port;
724 dst_port = th0->dst_port;
725
726 /* Try to determine what/why we're actually resetting */
Dave Barach68b0fb02017-02-28 15:15:56 -0500727 if (state == TCP_STATE_CLOSED)
728 {
729 if (!tcp_syn (th0))
730 return -1;
731
732 tmp = clib_net_to_host_u32 (th0->seq_number);
733
734 /* Got a SYN for no listener. */
Florin Corasdc629cd2017-05-09 00:52:37 -0700735 flags = TCP_FLAG_RST | TCP_FLAG_ACK;
736 ack = clib_host_to_net_u32 (tmp + 1);
737 seq = 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500738 }
Florin Corasdc629cd2017-05-09 00:52:37 -0700739 else
Dave Barach68b0fb02017-02-28 15:15:56 -0500740 {
Florin Corasdc629cd2017-05-09 00:52:37 -0700741 flags = TCP_FLAG_RST;
742 seq = th0->ack_number;
743 ack = 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500744 }
745
Florin Corasdc629cd2017-05-09 00:52:37 -0700746 tcp_reuse_buffer (vm, b0);
Florin Corasf988e692017-11-27 04:34:14 -0500747 tcp_trajectory_add_start (b0, 4);
Florin Corasdc629cd2017-05-09 00:52:37 -0700748 th0 = vlib_buffer_push_tcp_net_order (b0, dst_port, src_port, seq, ack,
749 sizeof (tcp_header_t), flags, 0);
Dave Barach68b0fb02017-02-28 15:15:56 -0500750
Dave Barach68b0fb02017-02-28 15:15:56 -0500751 if (is_ip4)
752 {
Florin Corasdc629cd2017-05-09 00:52:37 -0700753 ih4 = vlib_buffer_push_ip4 (vm, b0, &dst_ip40, &src_ip40,
Florin Corasfdbc3822017-07-27 00:34:12 -0700754 IP_PROTOCOL_TCP, 1);
Dave Barach68b0fb02017-02-28 15:15:56 -0500755 th0->checksum = ip4_tcp_udp_compute_checksum (vm, b0, ih4);
756 }
757 else
758 {
759 int bogus = ~0;
Florin Corasdc629cd2017-05-09 00:52:37 -0700760 ih6 = vlib_buffer_push_ip6 (vm, b0, &dst_ip60, &src_ip60,
761 IP_PROTOCOL_TCP);
Dave Barach68b0fb02017-02-28 15:15:56 -0500762 th0->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b0, ih6, &bogus);
763 ASSERT (!bogus);
764 }
765
766 return 0;
767}
768
769/**
770 * Send reset without reusing existing buffer
Florin Coras1f152cd2017-08-18 19:28:03 -0700771 *
772 * It extracts connection info out of original packet
Dave Barach68b0fb02017-02-28 15:15:56 -0500773 */
774void
Florin Corasd4c49be2019-02-07 00:15:53 -0800775tcp_send_reset_w_pkt (tcp_connection_t * tc, vlib_buffer_t * pkt,
776 u32 thread_index, u8 is_ip4)
Dave Barach68b0fb02017-02-28 15:15:56 -0500777{
Florin Corasd4c49be2019-02-07 00:15:53 -0800778 tcp_worker_ctx_t *wrk = tcp_get_worker (thread_index);
Florin Corasbe72ae62018-11-01 11:23:03 -0700779 vlib_main_t *vm = wrk->vm;
Dave Barach68b0fb02017-02-28 15:15:56 -0500780 vlib_buffer_t *b;
Florin Coras56b39f62018-03-27 17:29:32 -0700781 u32 bi, sw_if_index, fib_index;
Dave Barach68b0fb02017-02-28 15:15:56 -0500782 u8 tcp_hdr_len, flags = 0;
783 tcp_header_t *th, *pkt_th;
784 u32 seq, ack;
785 ip4_header_t *ih4, *pkt_ih4;
786 ip6_header_t *ih6, *pkt_ih6;
Florin Coras56b39f62018-03-27 17:29:32 -0700787 fib_protocol_t fib_proto;
Dave Barach68b0fb02017-02-28 15:15:56 -0500788
Florin Corasbdf7fd62019-01-31 17:31:01 -0800789 if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
Florin Coras66b11312017-07-31 17:18:03 -0700790 return;
791
Dave Barach68b0fb02017-02-28 15:15:56 -0500792 b = vlib_get_buffer (vm, bi);
Florin Coras56b39f62018-03-27 17:29:32 -0700793 sw_if_index = vnet_buffer (pkt)->sw_if_index[VLIB_RX];
794 fib_proto = is_ip4 ? FIB_PROTOCOL_IP4 : FIB_PROTOCOL_IP6;
795 fib_index = fib_table_get_index_for_sw_if_index (fib_proto, sw_if_index);
Florin Coras1f152cd2017-08-18 19:28:03 -0700796 tcp_init_buffer (vm, b);
Dave Barach68b0fb02017-02-28 15:15:56 -0500797
798 /* Make and write options */
799 tcp_hdr_len = sizeof (tcp_header_t);
800
801 if (is_ip4)
802 {
803 pkt_ih4 = vlib_buffer_get_current (pkt);
804 pkt_th = ip4_next_header (pkt_ih4);
805 }
806 else
807 {
808 pkt_ih6 = vlib_buffer_get_current (pkt);
809 pkt_th = ip6_next_header (pkt_ih6);
810 }
811
812 if (tcp_ack (pkt_th))
813 {
814 flags = TCP_FLAG_RST;
815 seq = pkt_th->ack_number;
Florin Coras776f3d82018-11-02 08:23:58 -0700816 ack = (tc->state >= TCP_STATE_SYN_RCVD) ? tc->rcv_nxt : 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500817 }
818 else
819 {
820 flags = TCP_FLAG_RST | TCP_FLAG_ACK;
821 seq = 0;
822 ack = clib_host_to_net_u32 (vnet_buffer (pkt)->tcp.seq_end);
823 }
824
825 th = vlib_buffer_push_tcp_net_order (b, pkt_th->dst_port, pkt_th->src_port,
826 seq, ack, tcp_hdr_len, flags, 0);
827
828 /* Swap src and dst ip */
829 if (is_ip4)
830 {
831 ASSERT ((pkt_ih4->ip_version_and_header_length & 0xF0) == 0x40);
832 ih4 = vlib_buffer_push_ip4 (vm, b, &pkt_ih4->dst_address,
Florin Corasfdbc3822017-07-27 00:34:12 -0700833 &pkt_ih4->src_address, IP_PROTOCOL_TCP, 1);
Dave Barach68b0fb02017-02-28 15:15:56 -0500834 th->checksum = ip4_tcp_udp_compute_checksum (vm, b, ih4);
835 }
836 else
837 {
838 int bogus = ~0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500839 ASSERT ((pkt_ih6->ip_version_traffic_class_and_flow_label & 0xF0) ==
840 0x60);
rootc9d1c5b2017-08-15 12:58:31 -0400841 ih6 = vlib_buffer_push_ip6 (vm, b, &pkt_ih6->dst_address,
842 &pkt_ih6->src_address, IP_PROTOCOL_TCP);
Dave Barach68b0fb02017-02-28 15:15:56 -0500843 th->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ih6, &bogus);
844 ASSERT (!bogus);
845 }
846
Florin Corasbe72ae62018-11-01 11:23:03 -0700847 tcp_enqueue_to_ip_lookup_now (wrk, b, bi, is_ip4, fib_index);
Florin Coras6534b7a2017-07-18 05:38:03 -0400848 TCP_EVT_DBG (TCP_EVT_RST_SENT, tc);
Dave Barach68b0fb02017-02-28 15:15:56 -0500849}
850
Florin Coras1f152cd2017-08-18 19:28:03 -0700851/**
852 * Build and set reset packet for connection
853 */
854void
855tcp_send_reset (tcp_connection_t * tc)
856{
Florin Corasbe72ae62018-11-01 11:23:03 -0700857 tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
858 vlib_main_t *vm = wrk->vm;
Florin Coras1f152cd2017-08-18 19:28:03 -0700859 vlib_buffer_t *b;
860 u32 bi;
861 tcp_header_t *th;
862 u16 tcp_hdr_opts_len, advertise_wnd, opts_write_len;
863 u8 flags;
864
Florin Corasbdf7fd62019-01-31 17:31:01 -0800865 if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
Florin Coras1f152cd2017-08-18 19:28:03 -0700866 return;
867 b = vlib_get_buffer (vm, bi);
868 tcp_init_buffer (vm, b);
869
870 tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts, tc->state);
871 tcp_hdr_opts_len = tc->snd_opts_len + sizeof (tcp_header_t);
872 advertise_wnd = tcp_window_to_advertise (tc, TCP_STATE_ESTABLISHED);
873 flags = TCP_FLAG_RST;
874 th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, tc->snd_nxt,
875 tc->rcv_nxt, tcp_hdr_opts_len, flags,
876 advertise_wnd);
877 opts_write_len = tcp_options_write ((u8 *) (th + 1), &tc->snd_opts);
878 ASSERT (opts_write_len == tc->snd_opts_len);
879 vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
Florin Corasf1762d62017-09-24 19:43:08 -0400880 if (tc->c_is_ip4)
881 {
882 ip4_header_t *ih4;
883 ih4 = vlib_buffer_push_ip4 (vm, b, &tc->c_lcl_ip.ip4,
884 &tc->c_rmt_ip.ip4, IP_PROTOCOL_TCP, 0);
885 th->checksum = ip4_tcp_udp_compute_checksum (vm, b, ih4);
886 }
887 else
888 {
889 int bogus = ~0;
890 ip6_header_t *ih6;
891 ih6 = vlib_buffer_push_ip6 (vm, b, &tc->c_lcl_ip.ip6,
892 &tc->c_rmt_ip.ip6, IP_PROTOCOL_TCP);
893 th->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ih6, &bogus);
894 ASSERT (!bogus);
895 }
Florin Corasbe72ae62018-11-01 11:23:03 -0700896 tcp_enqueue_to_ip_lookup_now (wrk, b, bi, tc->c_is_ip4, tc->c_fib_index);
Florin Corasf1762d62017-09-24 19:43:08 -0400897 TCP_EVT_DBG (TCP_EVT_RST_SENT, tc);
Florin Coras1f152cd2017-08-18 19:28:03 -0700898}
899
Florin Coras0dbd5172018-06-25 16:19:34 -0700900static void
Florin Corasbe72ae62018-11-01 11:23:03 -0700901tcp_push_ip_hdr (tcp_worker_ctx_t * wrk, tcp_connection_t * tc,
902 vlib_buffer_t * b)
Dave Barach68b0fb02017-02-28 15:15:56 -0500903{
904 tcp_header_t *th = vlib_buffer_get_current (b);
Florin Corasbe72ae62018-11-01 11:23:03 -0700905 vlib_main_t *vm = wrk->vm;
Dave Barach68b0fb02017-02-28 15:15:56 -0500906 if (tc->c_is_ip4)
907 {
908 ip4_header_t *ih;
Dave Barach2c25a622017-06-26 11:35:07 -0400909 ih = vlib_buffer_push_ip4 (vm, b, &tc->c_lcl_ip4,
Florin Corasfdbc3822017-07-27 00:34:12 -0700910 &tc->c_rmt_ip4, IP_PROTOCOL_TCP, 1);
Dave Barach2c25a622017-06-26 11:35:07 -0400911 th->checksum = ip4_tcp_udp_compute_checksum (vm, b, ih);
Dave Barach68b0fb02017-02-28 15:15:56 -0500912 }
913 else
914 {
915 ip6_header_t *ih;
916 int bogus = ~0;
917
Dave Barach2c25a622017-06-26 11:35:07 -0400918 ih = vlib_buffer_push_ip6 (vm, b, &tc->c_lcl_ip6,
Dave Barach68b0fb02017-02-28 15:15:56 -0500919 &tc->c_rmt_ip6, IP_PROTOCOL_TCP);
Dave Barach2c25a622017-06-26 11:35:07 -0400920 th->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ih, &bogus);
Dave Barach68b0fb02017-02-28 15:15:56 -0500921 ASSERT (!bogus);
922 }
923}
924
925/**
926 * Send SYN
927 *
928 * Builds a SYN packet for a half-open connection and sends it to ipx_lookup.
929 * The packet is not forwarded through tcpx_output to avoid doing lookups
930 * in the half_open pool.
931 */
932void
933tcp_send_syn (tcp_connection_t * tc)
934{
Florin Corasbe72ae62018-11-01 11:23:03 -0700935 tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
936 vlib_main_t *vm = wrk->vm;
Dave Barach68b0fb02017-02-28 15:15:56 -0500937 vlib_buffer_t *b;
938 u32 bi;
Dave Barach68b0fb02017-02-28 15:15:56 -0500939
Florin Corasf988e692017-11-27 04:34:14 -0500940 /*
941 * Setup retransmit and establish timers before requesting buffer
942 * such that we can return if we've ran out.
943 */
Florin Coras85a3ddd2018-12-24 16:54:34 -0800944 tcp_timer_set (tc, TCP_TIMER_ESTABLISH_AO, TCP_ESTABLISH_TIME);
Florin Corasf988e692017-11-27 04:34:14 -0500945 tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN,
946 tc->rto * TCP_TO_TIMER_TICK);
947
Florin Corasbdf7fd62019-01-31 17:31:01 -0800948 if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
Florin Coras222e1f412019-02-16 20:47:32 -0800949 {
950 tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN, 1);
951 return;
952 }
Florin Coras66b11312017-07-31 17:18:03 -0700953
Dave Barach68b0fb02017-02-28 15:15:56 -0500954 b = vlib_get_buffer (vm, bi);
Florin Coras1f152cd2017-08-18 19:28:03 -0700955 tcp_init_buffer (vm, b);
Florin Coras4eeeaaf2017-09-05 14:03:37 -0400956 tcp_make_syn (tc, b);
Dave Barach68b0fb02017-02-28 15:15:56 -0500957
958 /* Measure RTT with this */
Florin Corasefefc6b2018-11-07 12:49:19 -0800959 tc->rtt_ts = tcp_time_now_us (vlib_num_workers ()? 1 : 0);
Dave Barach68b0fb02017-02-28 15:15:56 -0500960 tc->rtt_seq = tc->snd_nxt;
Dave Barach68b0fb02017-02-28 15:15:56 -0500961 tc->rto_boff = 0;
962
Florin Corasbe72ae62018-11-01 11:23:03 -0700963 tcp_push_ip_hdr (wrk, tc, b);
964 tcp_enqueue_to_ip_lookup (wrk, b, bi, tc->c_is_ip4, tc->c_fib_index);
Florin Coras6534b7a2017-07-18 05:38:03 -0400965 TCP_EVT_DBG (TCP_EVT_SYN_SENT, tc);
Dave Barach68b0fb02017-02-28 15:15:56 -0500966}
967
Florin Coras7ac053b2018-11-05 15:57:21 -0800968void
969tcp_send_synack (tcp_connection_t * tc)
970{
971 tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
972 vlib_main_t *vm = wrk->vm;
973 vlib_buffer_t *b;
974 u32 bi;
975
Florin Coras222e1f412019-02-16 20:47:32 -0800976 tcp_retransmit_timer_force_update (tc);
977
Florin Corasbdf7fd62019-01-31 17:31:01 -0800978 if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
Florin Coras222e1f412019-02-16 20:47:32 -0800979 {
980 tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1);
981 return;
982 }
Florin Coras7ac053b2018-11-05 15:57:21 -0800983
Florin Corasdf084782018-12-06 17:41:10 -0800984 tc->rtt_ts = tcp_time_now_us (tc->c_thread_index);
Florin Coras7ac053b2018-11-05 15:57:21 -0800985 b = vlib_get_buffer (vm, bi);
Florin Corasbdf7fd62019-01-31 17:31:01 -0800986 tcp_init_buffer (vm, b);
Florin Coras7ac053b2018-11-05 15:57:21 -0800987 tcp_make_synack (tc, b);
988 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Florin Coras222e1f412019-02-16 20:47:32 -0800989 TCP_EVT_DBG (TCP_EVT_SYNACK_SENT, tc);
Florin Coras7ac053b2018-11-05 15:57:21 -0800990}
991
Florin Coras66b11312017-07-31 17:18:03 -0700992/**
993 * Flush tx frame populated by retransmits and timer pops
994 */
995void
Florin Corasbe72ae62018-11-01 11:23:03 -0700996tcp_flush_frame_to_output (tcp_worker_ctx_t * wrk, u8 is_ip4)
Florin Coras66b11312017-07-31 17:18:03 -0700997{
Florin Corasbe72ae62018-11-01 11:23:03 -0700998 if (wrk->tx_frames[!is_ip4])
Florin Coras66b11312017-07-31 17:18:03 -0700999 {
1000 u32 next_index;
1001 next_index = is_ip4 ? tcp4_output_node.index : tcp6_output_node.index;
Florin Corasbe72ae62018-11-01 11:23:03 -07001002 vlib_put_frame_to_node (wrk->vm, next_index, wrk->tx_frames[!is_ip4]);
1003 wrk->tx_frames[!is_ip4] = 0;
Florin Coras66b11312017-07-31 17:18:03 -07001004 }
1005}
1006
1007/**
Florin Coras9d063042017-09-14 03:08:00 -04001008 * Flush ip lookup tx frames populated by timer pops
1009 */
Florin Coras0dbd5172018-06-25 16:19:34 -07001010static void
Florin Corasbe72ae62018-11-01 11:23:03 -07001011tcp_flush_frame_to_ip_lookup (tcp_worker_ctx_t * wrk, u8 is_ip4)
Florin Coras9d063042017-09-14 03:08:00 -04001012{
Florin Corasbe72ae62018-11-01 11:23:03 -07001013 if (wrk->ip_lookup_tx_frames[!is_ip4])
Florin Coras9d063042017-09-14 03:08:00 -04001014 {
1015 u32 next_index;
1016 next_index = is_ip4 ? ip4_lookup_node.index : ip6_lookup_node.index;
Florin Corasbe72ae62018-11-01 11:23:03 -07001017 vlib_put_frame_to_node (wrk->vm, next_index,
1018 wrk->ip_lookup_tx_frames[!is_ip4]);
1019 wrk->ip_lookup_tx_frames[!is_ip4] = 0;
Florin Coras9d063042017-09-14 03:08:00 -04001020 }
1021}
1022
1023/**
1024 * Flush v4 and v6 tcp and ip-lookup tx frames for thread index
Florin Coras66b11312017-07-31 17:18:03 -07001025 */
1026void
Florin Corasbe72ae62018-11-01 11:23:03 -07001027tcp_flush_frames_to_output (tcp_worker_ctx_t * wrk)
Florin Coras66b11312017-07-31 17:18:03 -07001028{
Florin Corasbe72ae62018-11-01 11:23:03 -07001029 tcp_flush_frame_to_output (wrk, 1);
1030 tcp_flush_frame_to_output (wrk, 0);
1031 tcp_flush_frame_to_ip_lookup (wrk, 1);
1032 tcp_flush_frame_to_ip_lookup (wrk, 0);
Dave Barach68b0fb02017-02-28 15:15:56 -05001033}
1034
1035/**
1036 * Send FIN
1037 */
1038void
1039tcp_send_fin (tcp_connection_t * tc)
1040{
Florin Corasbe72ae62018-11-01 11:23:03 -07001041 tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
1042 vlib_main_t *vm = wrk->vm;
Florin Coras9d063042017-09-14 03:08:00 -04001043 vlib_buffer_t *b;
1044 u32 bi;
1045 u8 fin_snt = 0;
1046
Florin Corasc977e7c2018-10-16 20:30:31 -07001047 fin_snt = tc->flags & TCP_CONN_FINSNT;
1048 if (fin_snt)
1049 tc->snd_nxt = tc->snd_una;
1050
Florin Corasbdf7fd62019-01-31 17:31:01 -08001051 if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
Florin Coraseb97e5f2018-10-15 21:35:42 -07001052 {
1053 /* Out of buffers so program fin retransmit ASAP */
1054 tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1);
Florin Coras85a3ddd2018-12-24 16:54:34 -08001055 if (fin_snt)
1056 tc->snd_nxt = tc->snd_una_max;
Florin Coras222e1f412019-02-16 20:47:32 -08001057 else
1058 /* Make sure retransmit retries a fin not data */
1059 tc->flags |= TCP_CONN_FINSNT;
Florin Coras85a3ddd2018-12-24 16:54:34 -08001060 return;
Florin Coraseb97e5f2018-10-15 21:35:42 -07001061 }
1062
Florin Corasc977e7c2018-10-16 20:30:31 -07001063 tcp_retransmit_timer_force_update (tc);
Dave Barach68b0fb02017-02-28 15:15:56 -05001064 b = vlib_get_buffer (vm, bi);
Florin Coras24793e32018-05-09 11:34:25 -07001065 tcp_init_buffer (vm, b);
Florin Corasd79b41e2017-03-04 05:37:52 -08001066 tcp_make_fin (tc, b);
Florin Corasbe72ae62018-11-01 11:23:03 -07001067 tcp_enqueue_to_output_now (wrk, b, bi, tc->c_is_ip4);
Florin Corasc977e7c2018-10-16 20:30:31 -07001068 TCP_EVT_DBG (TCP_EVT_FIN_SENT, tc);
1069
Florin Coras9d063042017-09-14 03:08:00 -04001070 if (!fin_snt)
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001071 {
1072 tc->flags |= TCP_CONN_FINSNT;
1073 tc->flags &= ~TCP_CONN_FINPNDG;
Florin Coras9d063042017-09-14 03:08:00 -04001074 /* Account for the FIN */
1075 tc->snd_una_max += 1;
1076 tc->snd_nxt = tc->snd_una_max;
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001077 }
Florin Corasa096f2d2017-09-28 23:49:42 -04001078 else
1079 {
1080 tc->snd_nxt = tc->snd_una_max;
1081 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001082}
1083
1084always_inline u8
Florin Corasb2215d62017-08-01 16:56:58 -07001085tcp_make_state_flags (tcp_connection_t * tc, tcp_state_t next_state)
Dave Barach68b0fb02017-02-28 15:15:56 -05001086{
1087 switch (next_state)
1088 {
1089 case TCP_STATE_ESTABLISHED:
Florin Coras00dfe542018-06-11 13:15:11 -07001090 case TCP_STATE_CLOSE_WAIT:
Florin Coras54ddf432018-12-21 13:54:09 -08001091 case TCP_STATE_TIME_WAIT:
1092 case TCP_STATE_FIN_WAIT_2:
Dave Barach68b0fb02017-02-28 15:15:56 -05001093 return TCP_FLAG_ACK;
1094 case TCP_STATE_SYN_RCVD:
1095 return TCP_FLAG_SYN | TCP_FLAG_ACK;
1096 case TCP_STATE_SYN_SENT:
1097 return TCP_FLAG_SYN;
1098 case TCP_STATE_LAST_ACK:
1099 case TCP_STATE_FIN_WAIT_1:
Florin Coras54ddf432018-12-21 13:54:09 -08001100 case TCP_STATE_CLOSING:
Florin Corasb2215d62017-08-01 16:56:58 -07001101 if (tc->snd_nxt + 1 < tc->snd_una_max)
1102 return TCP_FLAG_ACK;
1103 else
1104 return TCP_FLAG_FIN;
Dave Barach68b0fb02017-02-28 15:15:56 -05001105 default:
1106 clib_warning ("Shouldn't be here!");
1107 }
1108 return 0;
1109}
1110
1111/**
1112 * Push TCP header and update connection variables
1113 */
Florin Coras0dbd5172018-06-25 16:19:34 -07001114always_inline void
Dave Barach68b0fb02017-02-28 15:15:56 -05001115tcp_push_hdr_i (tcp_connection_t * tc, vlib_buffer_t * b,
Florin Corasb26743d2018-06-26 09:31:04 -07001116 tcp_state_t next_state, u8 compute_opts, u8 maybe_burst)
Dave Barach68b0fb02017-02-28 15:15:56 -05001117{
1118 u32 advertise_wnd, data_len;
Florin Corasb26743d2018-06-26 09:31:04 -07001119 u8 tcp_hdr_opts_len, flags;
1120 tcp_main_t *tm = &tcp_main;
Dave Barach68b0fb02017-02-28 15:15:56 -05001121 tcp_header_t *th;
1122
Florin Corasb26743d2018-06-26 09:31:04 -07001123 data_len = b->current_length;
1124 if (PREDICT_FALSE (b->flags & VLIB_BUFFER_NEXT_PRESENT))
1125 data_len += b->total_length_not_including_first_buffer;
1126
Dave Barach68b0fb02017-02-28 15:15:56 -05001127 vnet_buffer (b)->tcp.flags = 0;
Florin Corasb26743d2018-06-26 09:31:04 -07001128 vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
Dave Barach68b0fb02017-02-28 15:15:56 -05001129
Florin Corasc8343412017-05-04 14:25:50 -07001130 if (compute_opts)
1131 tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts, tc->state);
1132
Florin Corasc8343412017-05-04 14:25:50 -07001133 tcp_hdr_opts_len = tc->snd_opts_len + sizeof (tcp_header_t);
Florin Corasb26743d2018-06-26 09:31:04 -07001134
1135 if (maybe_burst)
1136 advertise_wnd = tc->rcv_wnd >> tc->rcv_wscale;
1137 else
1138 advertise_wnd = tcp_window_to_advertise (tc, next_state);
1139
Florin Corasb2215d62017-08-01 16:56:58 -07001140 flags = tcp_make_state_flags (tc, next_state);
Florin Coras42ceddb2018-12-12 10:56:01 -08001141 if (PREDICT_FALSE (tc->flags & TCP_CONN_PSH_PENDING))
1142 {
1143 if (seq_geq (tc->psh_seq, tc->snd_nxt)
1144 && seq_lt (tc->psh_seq, tc->snd_nxt + data_len))
1145 flags |= TCP_FLAG_PSH;
1146 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001147 th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, tc->snd_nxt,
1148 tc->rcv_nxt, tcp_hdr_opts_len, flags,
1149 advertise_wnd);
Dave Barach68b0fb02017-02-28 15:15:56 -05001150
Florin Corasb26743d2018-06-26 09:31:04 -07001151 if (maybe_burst)
1152 {
Dave Barach178cf492018-11-13 16:34:13 -05001153 clib_memcpy_fast ((u8 *) (th + 1),
1154 tm->wrk_ctx[tc->c_thread_index].cached_opts,
1155 tc->snd_opts_len);
Florin Corasb26743d2018-06-26 09:31:04 -07001156 }
1157 else
1158 {
1159 u8 len = tcp_options_write ((u8 *) (th + 1), &tc->snd_opts);
1160 ASSERT (len == tc->snd_opts_len);
1161 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001162
Florin Coras93992a92017-05-24 18:03:56 -07001163 /*
1164 * Update connection variables
1165 */
1166
Dave Barach68b0fb02017-02-28 15:15:56 -05001167 tc->snd_nxt += data_len;
Florin Corasc28764f2017-04-26 00:08:42 -07001168 tc->rcv_las = tc->rcv_nxt;
Florin Coras3e350af2017-03-30 02:54:28 -07001169
Florin Corase69f4952017-03-07 10:06:24 -08001170 TCP_EVT_DBG (TCP_EVT_PKTIZE, tc);
Dave Barach68b0fb02017-02-28 15:15:56 -05001171}
1172
Florin Coras0dbd5172018-06-25 16:19:34 -07001173u32
1174tcp_push_header (tcp_connection_t * tc, vlib_buffer_t * b)
1175{
Florin Corasb26743d2018-06-26 09:31:04 -07001176 tcp_push_hdr_i (tc, b, TCP_STATE_ESTABLISHED, /* compute opts */ 0,
1177 /* burst */ 1);
Florin Coras0dbd5172018-06-25 16:19:34 -07001178 tc->snd_una_max = tc->snd_nxt;
Florin Corasb11175d2018-11-09 14:34:08 -08001179 ASSERT (seq_leq (tc->snd_una_max, tc->snd_una + tc->snd_wnd));
Florin Coras0dbd5172018-06-25 16:19:34 -07001180 tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una);
1181 /* If not tracking an ACK, start tracking */
1182 if (tc->rtt_ts == 0 && !tcp_in_cong_recovery (tc))
1183 {
Florin Corasd67f1122018-05-21 17:47:40 -07001184 tc->rtt_ts = tcp_time_now_us (tc->c_thread_index);
Florin Coras0dbd5172018-06-25 16:19:34 -07001185 tc->rtt_seq = tc->snd_nxt;
1186 }
1187 if (PREDICT_FALSE (!tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT)))
1188 {
1189 tcp_retransmit_timer_set (tc);
1190 tc->rto_boff = 0;
1191 }
1192 tcp_trajectory_add_start (b, 3);
1193 return 0;
1194}
1195
Dave Barach68b0fb02017-02-28 15:15:56 -05001196void
Florin Coras6792ec02017-03-13 03:49:51 -07001197tcp_send_ack (tcp_connection_t * tc)
Dave Barach68b0fb02017-02-28 15:15:56 -05001198{
Florin Corasbe72ae62018-11-01 11:23:03 -07001199 tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
1200 vlib_main_t *vm = wrk->vm;
Dave Barach68b0fb02017-02-28 15:15:56 -05001201 vlib_buffer_t *b;
1202 u32 bi;
1203
Florin Corasbdf7fd62019-01-31 17:31:01 -08001204 if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
Florin Coras222e1f412019-02-16 20:47:32 -08001205 {
1206 tcp_update_rcv_wnd (tc);
1207 return;
1208 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001209 b = vlib_get_buffer (vm, bi);
Florin Coras24793e32018-05-09 11:34:25 -07001210 tcp_init_buffer (vm, b);
Dave Barach68b0fb02017-02-28 15:15:56 -05001211 tcp_make_ack (tc, b);
Florin Corasbe72ae62018-11-01 11:23:03 -07001212 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Dave Barach68b0fb02017-02-28 15:15:56 -05001213}
1214
Florin Coras7ac053b2018-11-05 15:57:21 -08001215void
1216tcp_program_ack (tcp_worker_ctx_t * wrk, tcp_connection_t * tc)
1217{
1218 if (!(tc->flags & TCP_CONN_SNDACK))
1219 {
1220 vec_add1 (wrk->pending_acks, tc->c_c_index);
1221 tc->flags |= TCP_CONN_SNDACK;
1222 }
1223}
1224
1225void
1226tcp_program_dupack (tcp_worker_ctx_t * wrk, tcp_connection_t * tc)
1227{
1228 if (!(tc->flags & TCP_CONN_SNDACK))
1229 {
1230 vec_add1 (wrk->pending_acks, tc->c_c_index);
1231 tc->flags |= TCP_CONN_SNDACK;
1232 }
1233 if (tc->pending_dupacks < 255)
1234 tc->pending_dupacks += 1;
1235}
1236
1237void
1238tcp_send_acks (tcp_worker_ctx_t * wrk)
1239{
1240 u32 thread_index, *pending_acks;
1241 tcp_connection_t *tc;
1242 int i, j, n_acks;
1243
1244 if (!vec_len (wrk->pending_acks))
1245 return;
1246
1247 thread_index = wrk->vm->thread_index;
1248 pending_acks = wrk->pending_acks;
1249 for (i = 0; i < vec_len (pending_acks); i++)
1250 {
1251 tc = tcp_connection_get (pending_acks[i], thread_index);
1252 tc->flags &= ~TCP_CONN_SNDACK;
1253 n_acks = clib_max (1, tc->pending_dupacks);
1254 /* If we're supposed to send dupacks but have no ooo data
1255 * send only one ack */
1256 if (tc->pending_dupacks && !vec_len (tc->snd_sacks))
1257 n_acks = 1;
1258 for (j = 0; j < n_acks; j++)
1259 tcp_send_ack (tc);
1260 tc->pending_dupacks = 0;
1261 }
1262 _vec_len (wrk->pending_acks) = 0;
1263}
1264
Florin Corasb2215d62017-08-01 16:56:58 -07001265/**
1266 * Delayed ack timer handler
1267 *
1268 * Sends delayed ACK when timer expires
1269 */
Florin Coras6792ec02017-03-13 03:49:51 -07001270void
1271tcp_timer_delack_handler (u32 index)
1272{
Damjan Marion586afd72017-04-05 19:18:20 +02001273 u32 thread_index = vlib_get_thread_index ();
Florin Coras6792ec02017-03-13 03:49:51 -07001274 tcp_connection_t *tc;
1275
1276 tc = tcp_connection_get (index, thread_index);
1277 tc->timers[TCP_TIMER_DELACK] = TCP_TIMER_HANDLE_INVALID;
Florin Coras6792ec02017-03-13 03:49:51 -07001278 tcp_send_ack (tc);
1279}
1280
Florin Corasb2215d62017-08-01 16:56:58 -07001281/**
Florin Coras36ee9f12018-11-02 12:52:10 -07001282 * Allocate a new buffer and build a new tcp segment
Dave Barach68b0fb02017-02-28 15:15:56 -05001283 *
Florin Coras36ee9f12018-11-02 12:52:10 -07001284 * @param wrk tcp worker
1285 * @param tc connection for which the segment will be allocated
1286 * @param offset offset of the first byte in the tx fifo
1287 * @param max_deq_byte segment size
1288 * @param[out] b pointer to buffer allocated
1289 *
1290 * @return the number of bytes in the segment or 0 if buffer cannot be
1291 * allocated or no data available
Florin Coras93992a92017-05-24 18:03:56 -07001292 */
Florin Coras36ee9f12018-11-02 12:52:10 -07001293static int
1294tcp_prepare_segment (tcp_worker_ctx_t * wrk, tcp_connection_t * tc,
1295 u32 offset, u32 max_deq_bytes, vlib_buffer_t ** b)
Dave Barach68b0fb02017-02-28 15:15:56 -05001296{
Florin Corasbe72ae62018-11-01 11:23:03 -07001297 u32 bytes_per_buffer = vnet_get_tcp_main ()->bytes_per_buffer;
1298 vlib_main_t *vm = wrk->vm;
Florin Corasbdf7fd62019-01-31 17:31:01 -08001299 u32 bi, seg_size;
Florin Coras93992a92017-05-24 18:03:56 -07001300 int n_bytes = 0;
Florin Corase87216f2017-08-17 16:59:22 -07001301 u8 *data;
Dave Barach68b0fb02017-02-28 15:15:56 -05001302
Florin Coras1ee78302019-02-05 15:51:15 -08001303 seg_size = max_deq_bytes + TRANSPORT_MAX_HDRS_LEN;
Florin Coras1f152cd2017-08-18 19:28:03 -07001304
Florin Corasb2215d62017-08-01 16:56:58 -07001305 /*
1306 * Prepare options
1307 */
Florin Corasc8343412017-05-04 14:25:50 -07001308 tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts, tc->state);
1309
Florin Corasb2215d62017-08-01 16:56:58 -07001310 /*
1311 * Allocate and fill in buffer(s)
1312 */
Dave Barach68b0fb02017-02-28 15:15:56 -05001313
Florin Corasb2215d62017-08-01 16:56:58 -07001314 /* Easy case, buffer size greater than mss */
Florin Corasbe72ae62018-11-01 11:23:03 -07001315 if (PREDICT_TRUE (seg_size <= bytes_per_buffer))
Florin Corasb2215d62017-08-01 16:56:58 -07001316 {
Florin Corasbdf7fd62019-01-31 17:31:01 -08001317 if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
Florin Coras24793e32018-05-09 11:34:25 -07001318 return 0;
1319 *b = vlib_get_buffer (vm, bi);
1320 data = tcp_init_buffer (vm, *b);
Florin Corase87216f2017-08-17 16:59:22 -07001321 n_bytes = stream_session_peek_bytes (&tc->connection, data, offset,
1322 max_deq_bytes);
Florin Corasb2215d62017-08-01 16:56:58 -07001323 ASSERT (n_bytes == max_deq_bytes);
1324 b[0]->current_length = n_bytes;
Florin Corasb26743d2018-06-26 09:31:04 -07001325 tcp_push_hdr_i (tc, *b, tc->state, /* compute opts */ 0, /* burst */ 0);
Florin Coras208daa12018-07-02 01:30:51 -07001326 if (seq_gt (tc->snd_nxt, tc->snd_una_max))
1327 tc->snd_una_max = tc->snd_nxt;
Florin Corasb2215d62017-08-01 16:56:58 -07001328 }
1329 /* Split mss into multiple buffers */
1330 else
1331 {
Florin Corasbdf7fd62019-01-31 17:31:01 -08001332 u32 chain_bi = ~0, n_bufs_per_seg, n_bufs;
1333 u16 n_peeked, len_to_deq;
Florin Corasb2215d62017-08-01 16:56:58 -07001334 vlib_buffer_t *chain_b, *prev_b;
Florin Corasb2215d62017-08-01 16:56:58 -07001335 int i;
1336
Florin Corasb2215d62017-08-01 16:56:58 -07001337 /* Make sure we have enough buffers */
Florin Corasbe72ae62018-11-01 11:23:03 -07001338 n_bufs_per_seg = ceil ((double) seg_size / bytes_per_buffer);
Florin Corasbdf7fd62019-01-31 17:31:01 -08001339 vec_validate_aligned (wrk->tx_buffers, n_bufs_per_seg - 1,
1340 CLIB_CACHE_LINE_BYTES);
1341 n_bufs = vlib_buffer_alloc (vm, wrk->tx_buffers, n_bufs_per_seg);
1342 if (PREDICT_FALSE (n_bufs != n_bufs_per_seg))
Florin Corasb2215d62017-08-01 16:56:58 -07001343 {
Florin Corasbdf7fd62019-01-31 17:31:01 -08001344 if (n_bufs)
1345 vlib_buffer_free (vm, wrk->tx_buffers, n_bufs);
1346 return 0;
Florin Corasb2215d62017-08-01 16:56:58 -07001347 }
1348
Florin Corasbdf7fd62019-01-31 17:31:01 -08001349 *b = vlib_get_buffer (vm, wrk->tx_buffers[--n_bufs]);
Florin Coras24793e32018-05-09 11:34:25 -07001350 data = tcp_init_buffer (vm, *b);
Florin Corase87216f2017-08-17 16:59:22 -07001351 n_bytes = stream_session_peek_bytes (&tc->connection, data, offset,
Florin Coras1ee78302019-02-05 15:51:15 -08001352 bytes_per_buffer -
1353 TRANSPORT_MAX_HDRS_LEN);
Florin Corasb2215d62017-08-01 16:56:58 -07001354 b[0]->current_length = n_bytes;
1355 b[0]->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
1356 b[0]->total_length_not_including_first_buffer = 0;
Florin Corasb2215d62017-08-01 16:56:58 -07001357 max_deq_bytes -= n_bytes;
1358
1359 chain_b = *b;
1360 for (i = 1; i < n_bufs_per_seg; i++)
1361 {
1362 prev_b = chain_b;
Florin Corasbe72ae62018-11-01 11:23:03 -07001363 len_to_deq = clib_min (max_deq_bytes, bytes_per_buffer);
Florin Corasbdf7fd62019-01-31 17:31:01 -08001364 chain_bi = wrk->tx_buffers[--n_bufs];
Florin Corasb2215d62017-08-01 16:56:58 -07001365 chain_b = vlib_get_buffer (vm, chain_bi);
1366 chain_b->current_data = 0;
Florin Corase87216f2017-08-17 16:59:22 -07001367 data = vlib_buffer_get_current (chain_b);
1368 n_peeked = stream_session_peek_bytes (&tc->connection, data,
Florin Coras1f152cd2017-08-18 19:28:03 -07001369 offset + n_bytes, len_to_deq);
Florin Corasb2215d62017-08-01 16:56:58 -07001370 ASSERT (n_peeked == len_to_deq);
Florin Coras1f152cd2017-08-18 19:28:03 -07001371 n_bytes += n_peeked;
Florin Corasb2215d62017-08-01 16:56:58 -07001372 chain_b->current_length = n_peeked;
Florin Coras1f152cd2017-08-18 19:28:03 -07001373 chain_b->next_buffer = 0;
Florin Corasb2215d62017-08-01 16:56:58 -07001374
1375 /* update previous buffer */
1376 prev_b->next_buffer = chain_bi;
1377 prev_b->flags |= VLIB_BUFFER_NEXT_PRESENT;
1378
Florin Corasb2215d62017-08-01 16:56:58 -07001379 max_deq_bytes -= n_peeked;
Florin Coras1f152cd2017-08-18 19:28:03 -07001380 b[0]->total_length_not_including_first_buffer += n_peeked;
Florin Corasb2215d62017-08-01 16:56:58 -07001381 }
Florin Coras1f152cd2017-08-18 19:28:03 -07001382
Florin Corasb26743d2018-06-26 09:31:04 -07001383 tcp_push_hdr_i (tc, *b, tc->state, /* compute opts */ 0, /* burst */ 0);
Florin Coras208daa12018-07-02 01:30:51 -07001384 if (seq_gt (tc->snd_nxt, tc->snd_una_max))
1385 tc->snd_una_max = tc->snd_nxt;
Florin Corasbdf7fd62019-01-31 17:31:01 -08001386
1387 if (PREDICT_FALSE (n_bufs))
1388 {
1389 clib_warning ("not all buffers consumed");
1390 vlib_buffer_free (vm, wrk->tx_buffers, n_bufs);
1391 }
Florin Corasb2215d62017-08-01 16:56:58 -07001392 }
1393
Florin Coras93992a92017-05-24 18:03:56 -07001394 ASSERT (n_bytes > 0);
Florin Corasbe72ae62018-11-01 11:23:03 -07001395 ASSERT (((*b)->current_data + (*b)->current_length) <= bytes_per_buffer);
Florin Corasbb292f42017-05-19 09:49:19 -07001396
Florin Coras36ee9f12018-11-02 12:52:10 -07001397 return n_bytes;
1398}
1399
1400/**
1401 * Build a retransmit segment
1402 *
1403 * @return the number of bytes in the segment or 0 if there's nothing to
1404 * retransmit
1405 */
1406static u32
1407tcp_prepare_retransmit_segment (tcp_worker_ctx_t * wrk,
1408 tcp_connection_t * tc, u32 offset,
1409 u32 max_deq_bytes, vlib_buffer_t ** b)
1410{
1411 u32 start, available_bytes;
1412 int n_bytes = 0;
1413
1414 ASSERT (tc->state >= TCP_STATE_ESTABLISHED);
1415 ASSERT (max_deq_bytes != 0);
1416
1417 /*
1418 * Make sure we can retransmit something
1419 */
1420 available_bytes = session_tx_fifo_max_dequeue (&tc->connection);
1421 ASSERT (available_bytes >= offset);
1422 available_bytes -= offset;
1423 if (!available_bytes)
1424 return 0;
1425
1426 max_deq_bytes = clib_min (tc->snd_mss, max_deq_bytes);
1427 max_deq_bytes = clib_min (available_bytes, max_deq_bytes);
1428
1429 /* Start is beyond snd_congestion */
1430 start = tc->snd_una + offset;
1431 if (seq_geq (start, tc->snd_congestion))
1432 goto done;
1433
1434 /* Don't overshoot snd_congestion */
1435 if (seq_gt (start + max_deq_bytes, tc->snd_congestion))
1436 {
1437 max_deq_bytes = tc->snd_congestion - start;
1438 if (max_deq_bytes == 0)
1439 goto done;
1440 }
1441
1442 n_bytes = tcp_prepare_segment (wrk, tc, offset, max_deq_bytes, b);
1443 if (!n_bytes)
1444 return 0;
1445
Florin Coras93992a92017-05-24 18:03:56 -07001446 if (tcp_in_fastrecovery (tc))
1447 tc->snd_rxt_bytes += n_bytes;
Dave Barach68b0fb02017-02-28 15:15:56 -05001448
Florin Coras6792ec02017-03-13 03:49:51 -07001449done:
1450 TCP_EVT_DBG (TCP_EVT_CC_RTX, tc, offset, n_bytes);
Dave Barach68b0fb02017-02-28 15:15:56 -05001451 return n_bytes;
1452}
1453
Florin Coras6792ec02017-03-13 03:49:51 -07001454/**
1455 * Reset congestion control, switch cwnd to loss window and try again.
1456 */
1457static void
Florin Corasca1c8f32018-05-23 21:01:30 -07001458tcp_rxt_timeout_cc (tcp_connection_t * tc)
Florin Coras6792ec02017-03-13 03:49:51 -07001459{
Florin Corasca1c8f32018-05-23 21:01:30 -07001460 TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 6);
Florin Coras93992a92017-05-24 18:03:56 -07001461 tc->prev_ssthresh = tc->ssthresh;
1462 tc->prev_cwnd = tc->cwnd;
1463
Florin Coras6792ec02017-03-13 03:49:51 -07001464 /* Cleanly recover cc (also clears up fast retransmit) */
1465 if (tcp_in_fastrecovery (tc))
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001466 {
1467 /* TODO be less aggressive about this */
1468 scoreboard_clear (&tc->sack_sb);
1469 tcp_cc_fastrecovery_exit (tc);
1470 }
Florin Coras222e1f412019-02-16 20:47:32 -08001471 else
1472 tc->rcv_dupacks = 0;
Florin Coras6792ec02017-03-13 03:49:51 -07001473
1474 /* Start again from the beginning */
Florin Corasd2aab832018-05-22 11:39:59 -07001475 tc->cc_algo->congestion (tc);
Florin Coras6792ec02017-03-13 03:49:51 -07001476 tc->cwnd = tcp_loss_wnd (tc);
1477 tc->snd_congestion = tc->snd_una_max;
Florin Corasf1762d62017-09-24 19:43:08 -04001478 tc->rtt_ts = 0;
Florin Corasd2aab832018-05-22 11:39:59 -07001479 tc->cwnd_acc_bytes = 0;
Florin Corasc44a5582018-11-01 16:30:54 -07001480 tcp_connection_tx_pacer_reset (tc, tc->cwnd, 2 * tc->snd_mss);
Florin Coras3af90fc2017-05-03 21:09:42 -07001481 tcp_recovery_on (tc);
Florin Coras6792ec02017-03-13 03:49:51 -07001482}
1483
Florin Coras0dbd5172018-06-25 16:19:34 -07001484static inline void
Dave Barach68b0fb02017-02-28 15:15:56 -05001485tcp_timer_retransmit_handler_i (u32 index, u8 is_syn)
1486{
Damjan Marion586afd72017-04-05 19:18:20 +02001487 u32 thread_index = vlib_get_thread_index ();
Florin Corasbe72ae62018-11-01 11:23:03 -07001488 tcp_worker_ctx_t *wrk = tcp_get_worker (thread_index);
1489 vlib_main_t *vm = wrk->vm;
Dave Barach68b0fb02017-02-28 15:15:56 -05001490 tcp_connection_t *tc;
Florin Corasb2215d62017-08-01 16:56:58 -07001491 vlib_buffer_t *b = 0;
Florin Coras3af90fc2017-05-03 21:09:42 -07001492 u32 bi, n_bytes;
Dave Barach68b0fb02017-02-28 15:15:56 -05001493
1494 if (is_syn)
1495 {
1496 tc = tcp_half_open_connection_get (index);
Dave Barachb7f1faa2017-08-29 11:43:37 -04001497 /* Note: the connection may have transitioned to ESTABLISHED... */
Florin Coras311e11b2018-12-05 11:19:14 -08001498 if (PREDICT_FALSE (tc == 0 || tc->state != TCP_STATE_SYN_SENT))
Dave Barachb7f1faa2017-08-29 11:43:37 -04001499 return;
Florin Coras68810622017-07-24 17:40:28 -07001500 tc->timers[TCP_TIMER_RETRANSMIT_SYN] = TCP_TIMER_HANDLE_INVALID;
Dave Barach68b0fb02017-02-28 15:15:56 -05001501 }
1502 else
1503 {
1504 tc = tcp_connection_get (index, thread_index);
Dave Barachb7f1faa2017-08-29 11:43:37 -04001505 /* Note: the connection may have been closed and pool_put */
Florin Coras85a3ddd2018-12-24 16:54:34 -08001506 if (PREDICT_FALSE (tc == 0 || tc->state == TCP_STATE_SYN_SENT))
Dave Barachb7f1faa2017-08-29 11:43:37 -04001507 return;
Florin Coras68810622017-07-24 17:40:28 -07001508 tc->timers[TCP_TIMER_RETRANSMIT] = TCP_TIMER_HANDLE_INVALID;
Florin Coras85a3ddd2018-12-24 16:54:34 -08001509 /* Wait-close and retransmit could pop at the same time */
1510 if (tc->state == TCP_STATE_CLOSED)
1511 return;
Dave Barach68b0fb02017-02-28 15:15:56 -05001512 }
1513
Florin Corase04c2992017-03-01 08:17:34 -08001514 if (tc->state >= TCP_STATE_ESTABLISHED)
Dave Barach68b0fb02017-02-28 15:15:56 -05001515 {
Florin Corase55a6d72018-10-31 23:09:22 -07001516 TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 2);
1517
Florin Coras93992a92017-05-24 18:03:56 -07001518 /* Lost FIN, retransmit and return */
Florin Coras222e1f412019-02-16 20:47:32 -08001519 if (tc->flags & TCP_CONN_FINSNT)
Florin Coras93992a92017-05-24 18:03:56 -07001520 {
1521 tcp_send_fin (tc);
Florin Corasf988e692017-11-27 04:34:14 -05001522 tc->rto_boff += 1;
1523 tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
Florin Coras93992a92017-05-24 18:03:56 -07001524 return;
1525 }
1526
Florin Coras3ec66b02018-08-23 16:27:05 -07001527 /* Shouldn't be here. This condition is tricky because it has to take
1528 * into account boff > 0 due to persist timeout. */
Florin Coras887b7ba2018-06-09 06:49:59 -07001529 if ((tc->rto_boff == 0 && tc->snd_una == tc->snd_una_max)
Florin Coras3ec66b02018-08-23 16:27:05 -07001530 || (tc->rto_boff > 0 && seq_geq (tc->snd_una, tc->snd_congestion)
1531 && !tcp_flight_size (tc)))
Florin Corasa096f2d2017-09-28 23:49:42 -04001532 {
Florin Coras3ec66b02018-08-23 16:27:05 -07001533 ASSERT (!tcp_in_recovery (tc));
1534 tc->rto_boff = 0;
Florin Corasa096f2d2017-09-28 23:49:42 -04001535 return;
1536 }
1537
Florin Coras3ec66b02018-08-23 16:27:05 -07001538 /* We're not in recovery so make sure rto_boff is 0. Can be non 0 due
1539 * to persist timer timeout */
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001540 if (!tcp_in_recovery (tc) && tc->rto_boff > 0)
1541 {
1542 tc->rto_boff = 0;
1543 tcp_update_rto (tc);
1544 }
1545
1546 /* Increment RTO backoff (also equal to number of retries) and go back
1547 * to first un-acked byte */
1548 tc->rto_boff += 1;
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001549
Florin Coras6792ec02017-03-13 03:49:51 -07001550 /* First retransmit timeout */
1551 if (tc->rto_boff == 1)
Florin Corasca1c8f32018-05-23 21:01:30 -07001552 tcp_rxt_timeout_cc (tc);
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001553 else
1554 scoreboard_clear (&tc->sack_sb);
Dave Barach68b0fb02017-02-28 15:15:56 -05001555
Florin Coras3ec66b02018-08-23 16:27:05 -07001556 /* If we've sent beyond snd_congestion, update it */
1557 if (seq_gt (tc->snd_una_max, tc->snd_congestion))
1558 tc->snd_congestion = tc->snd_una_max;
1559
Florin Corasd2aab832018-05-22 11:39:59 -07001560 tc->snd_una_max = tc->snd_nxt = tc->snd_una;
Dave Barach68b0fb02017-02-28 15:15:56 -05001561 tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
1562
Florin Coras3ec66b02018-08-23 16:27:05 -07001563 /* Send one segment. Note that n_bytes may be zero due to buffer
1564 * shortfall */
Florin Corasbe72ae62018-11-01 11:23:03 -07001565 n_bytes = tcp_prepare_retransmit_segment (wrk, tc, 0, tc->snd_mss, &b);
Florin Coras85a3ddd2018-12-24 16:54:34 -08001566 if (!n_bytes)
Florin Corase04c2992017-03-01 08:17:34 -08001567 {
Florin Coras222e1f412019-02-16 20:47:32 -08001568 tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1);
Florin Corase04c2992017-03-01 08:17:34 -08001569 return;
1570 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001571
Dave Barachb7f1faa2017-08-29 11:43:37 -04001572 bi = vlib_get_buffer_index (vm, b);
1573
Florin Coras93992a92017-05-24 18:03:56 -07001574 /* For first retransmit, record timestamp (Eifel detection RFC3522) */
1575 if (tc->rto_boff == 1)
Florin Corasbe72ae62018-11-01 11:23:03 -07001576 tc->snd_rxt_ts = tcp_time_now_w_thread (tc->c_thread_index);
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001577
Florin Corasbe72ae62018-11-01 11:23:03 -07001578 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Florin Coras3ec66b02018-08-23 16:27:05 -07001579 tcp_retransmit_timer_force_update (tc);
Florin Coras93992a92017-05-24 18:03:56 -07001580 }
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001581 /* Retransmit for SYN */
1582 else if (tc->state == TCP_STATE_SYN_SENT)
Florin Coras93992a92017-05-24 18:03:56 -07001583 {
Florin Coras68810622017-07-24 17:40:28 -07001584 /* Half-open connection actually moved to established but we were
1585 * waiting for syn retransmit to pop to call cleanup from the right
1586 * thread. */
1587 if (tc->flags & TCP_CONN_HALF_OPEN_DONE)
1588 {
Florin Coras68810622017-07-24 17:40:28 -07001589 if (tcp_half_open_connection_cleanup (tc))
Florin Coras85a3ddd2018-12-24 16:54:34 -08001590 TCP_DBG ("could not remove half-open connection");
Florin Coras68810622017-07-24 17:40:28 -07001591 return;
1592 }
1593
Florin Corase55a6d72018-10-31 23:09:22 -07001594 TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 2);
1595
Dave Barach68b0fb02017-02-28 15:15:56 -05001596 /* Try without increasing RTO a number of times. If this fails,
1597 * start growing RTO exponentially */
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001598 tc->rto_boff += 1;
Dave Barach68b0fb02017-02-28 15:15:56 -05001599 if (tc->rto_boff > TCP_RTO_SYN_RETRIES)
1600 tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
1601
Florin Corasf988e692017-11-27 04:34:14 -05001602 tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN,
1603 tc->rto * TCP_TO_TIMER_TICK);
1604
Florin Corasbdf7fd62019-01-31 17:31:01 -08001605 if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
Florin Coras222e1f412019-02-16 20:47:32 -08001606 {
1607 tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN, 1);
1608 return;
1609 }
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001610
Florin Corasb2215d62017-08-01 16:56:58 -07001611 b = vlib_get_buffer (vm, bi);
Florin Coras1f152cd2017-08-18 19:28:03 -07001612 tcp_init_buffer (vm, b);
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001613 tcp_make_syn (tc, b);
Florin Corase04c2992017-03-01 08:17:34 -08001614
Dave Barach2c25a622017-06-26 11:35:07 -04001615 tc->rtt_ts = 0;
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001616 TCP_EVT_DBG (TCP_EVT_SYN_RXT, tc, 0);
1617
1618 /* This goes straight to ipx_lookup. Retransmit timer set already */
Florin Corasbe72ae62018-11-01 11:23:03 -07001619 tcp_push_ip_hdr (wrk, tc, b);
1620 tcp_enqueue_to_ip_lookup (wrk, b, bi, tc->c_is_ip4, tc->c_fib_index);
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001621 }
1622 /* Retransmit SYN-ACK */
1623 else if (tc->state == TCP_STATE_SYN_RCVD)
1624 {
Florin Corase55a6d72018-10-31 23:09:22 -07001625 TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 2);
1626
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001627 tc->rto_boff += 1;
Florin Coras9d063042017-09-14 03:08:00 -04001628 if (tc->rto_boff > TCP_RTO_SYN_RETRIES)
1629 tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001630 tc->rtt_ts = 0;
1631
Florin Coras222e1f412019-02-16 20:47:32 -08001632 tcp_retransmit_timer_force_update (tc);
1633
Florin Corasbdf7fd62019-01-31 17:31:01 -08001634 if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
Florin Corasf988e692017-11-27 04:34:14 -05001635 {
Florin Coras222e1f412019-02-16 20:47:32 -08001636 tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1);
Florin Corasf988e692017-11-27 04:34:14 -05001637 return;
1638 }
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001639
1640 b = vlib_get_buffer (vm, bi);
Florin Coras24793e32018-05-09 11:34:25 -07001641 tcp_init_buffer (vm, b);
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001642 tcp_make_synack (tc, b);
1643 TCP_EVT_DBG (TCP_EVT_SYN_RXT, tc, 1);
1644
1645 /* Retransmit timer already updated, just enqueue to output */
Florin Corasbe72ae62018-11-01 11:23:03 -07001646 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Dave Barach68b0fb02017-02-28 15:15:56 -05001647 }
Florin Coras93992a92017-05-24 18:03:56 -07001648 else
1649 {
1650 ASSERT (tc->state == TCP_STATE_CLOSED);
Florin Coras93992a92017-05-24 18:03:56 -07001651 return;
1652 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001653}
1654
1655void
1656tcp_timer_retransmit_handler (u32 index)
1657{
1658 tcp_timer_retransmit_handler_i (index, 0);
1659}
1660
1661void
1662tcp_timer_retransmit_syn_handler (u32 index)
1663{
1664 tcp_timer_retransmit_handler_i (index, 1);
1665}
1666
1667/**
Florin Coras3e350af2017-03-30 02:54:28 -07001668 * Got 0 snd_wnd from peer, try to do something about it.
1669 *
1670 */
1671void
1672tcp_timer_persist_handler (u32 index)
1673{
Damjan Marion586afd72017-04-05 19:18:20 +02001674 u32 thread_index = vlib_get_thread_index ();
Florin Corasbe72ae62018-11-01 11:23:03 -07001675 tcp_worker_ctx_t *wrk = tcp_get_worker (thread_index);
1676 u32 bi, max_snd_bytes, available_bytes, offset;
1677 tcp_main_t *tm = vnet_get_tcp_main ();
1678 vlib_main_t *vm = wrk->vm;
Florin Coras3e350af2017-03-30 02:54:28 -07001679 tcp_connection_t *tc;
1680 vlib_buffer_t *b;
Florin Coras93992a92017-05-24 18:03:56 -07001681 int n_bytes = 0;
Florin Coras1f152cd2017-08-18 19:28:03 -07001682 u8 *data;
Florin Coras3e350af2017-03-30 02:54:28 -07001683
Florin Corasdb84e572017-05-09 18:54:52 -07001684 tc = tcp_connection_get_if_valid (index, thread_index);
1685
1686 if (!tc)
1687 return;
Florin Coras3e350af2017-03-30 02:54:28 -07001688
1689 /* Make sure timer handle is set to invalid */
1690 tc->timers[TCP_TIMER_PERSIST] = TCP_TIMER_HANDLE_INVALID;
1691
1692 /* Problem already solved or worse */
Florin Corasf03a59a2017-06-09 21:07:32 -07001693 if (tc->state == TCP_STATE_CLOSED || tc->state > TCP_STATE_ESTABLISHED
Florin Coras2e31cc32018-09-25 14:00:34 -07001694 || tc->snd_wnd > tc->snd_mss)
Florin Coras3e350af2017-03-30 02:54:28 -07001695 return;
1696
Florin Coras25579b42018-06-06 17:55:02 -07001697 available_bytes = session_tx_fifo_max_dequeue (&tc->connection);
Florin Coras50958952017-08-29 14:50:13 -07001698 offset = tc->snd_una_max - tc->snd_una;
1699
1700 /* Reprogram persist if no new bytes available to send. We may have data
1701 * next time */
1702 if (!available_bytes)
1703 {
1704 tcp_persist_timer_set (tc);
1705 return;
1706 }
1707
1708 if (available_bytes <= offset)
1709 {
1710 ASSERT (tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT));
1711 return;
1712 }
1713
Florin Coras3e350af2017-03-30 02:54:28 -07001714 /* Increment RTO backoff */
1715 tc->rto_boff += 1;
1716 tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
1717
Florin Corasb2215d62017-08-01 16:56:58 -07001718 /*
1719 * Try to force the first unsent segment (or buffer)
1720 */
Florin Corasbdf7fd62019-01-31 17:31:01 -08001721 if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
Florin Coras3ec66b02018-08-23 16:27:05 -07001722 {
1723 tcp_persist_timer_set (tc);
1724 return;
1725 }
Florin Coras3e350af2017-03-30 02:54:28 -07001726 b = vlib_get_buffer (vm, bi);
Florin Coras1f152cd2017-08-18 19:28:03 -07001727 data = tcp_init_buffer (vm, b);
Florin Coras93992a92017-05-24 18:03:56 -07001728
Florin Coras1f152cd2017-08-18 19:28:03 -07001729 tcp_validate_txf_size (tc, offset);
Florin Corasc8343412017-05-04 14:25:50 -07001730 tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts, tc->state);
Florin Coras1ee78302019-02-05 15:51:15 -08001731 max_snd_bytes =
1732 clib_min (tc->snd_mss, tm->bytes_per_buffer - TRANSPORT_MAX_HDRS_LEN);
1733 n_bytes =
1734 stream_session_peek_bytes (&tc->connection, data, offset, max_snd_bytes);
Florin Coras3e350af2017-03-30 02:54:28 -07001735 b->current_length = n_bytes;
Florin Coras84275e92017-09-26 12:30:40 -04001736 ASSERT (n_bytes != 0 && (tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT)
1737 || tc->snd_nxt == tc->snd_una_max
1738 || tc->rto_boff > 1));
Florin Coras93992a92017-05-24 18:03:56 -07001739
Florin Corasb26743d2018-06-26 09:31:04 -07001740 tcp_push_hdr_i (tc, b, tc->state, /* compute opts */ 0, /* burst */ 0);
Florin Coras8b20bf52018-06-14 14:55:50 -07001741 tc->snd_una_max = tc->snd_nxt;
1742 tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una);
Florin Corasbe72ae62018-11-01 11:23:03 -07001743 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Florin Coras3e350af2017-03-30 02:54:28 -07001744
Florin Coras1f152cd2017-08-18 19:28:03 -07001745 /* Just sent new data, enable retransmit */
1746 tcp_retransmit_timer_update (tc);
Florin Coras3e350af2017-03-30 02:54:28 -07001747}
1748
1749/**
Florin Coras6792ec02017-03-13 03:49:51 -07001750 * Retransmit first unacked segment
1751 */
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001752int
Florin Corasbe72ae62018-11-01 11:23:03 -07001753tcp_retransmit_first_unacked (tcp_worker_ctx_t * wrk, tcp_connection_t * tc)
Dave Barach68b0fb02017-02-28 15:15:56 -05001754{
Florin Corasb2215d62017-08-01 16:56:58 -07001755 u32 bi, old_snd_nxt, n_bytes;
Florin Corasbe72ae62018-11-01 11:23:03 -07001756 vlib_main_t *vm = wrk->vm;
1757 vlib_buffer_t *b;
Dave Barach68b0fb02017-02-28 15:15:56 -05001758
Florin Coras93992a92017-05-24 18:03:56 -07001759 old_snd_nxt = tc->snd_nxt;
Dave Barach68b0fb02017-02-28 15:15:56 -05001760 tc->snd_nxt = tc->snd_una;
1761
Florin Corase55a6d72018-10-31 23:09:22 -07001762 TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 1);
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001763
Florin Corasbe72ae62018-11-01 11:23:03 -07001764 n_bytes = tcp_prepare_retransmit_segment (wrk, tc, 0, tc->snd_mss, &b);
Florin Corasb2215d62017-08-01 16:56:58 -07001765 if (!n_bytes)
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001766 return -1;
1767
Florin Corasb2215d62017-08-01 16:56:58 -07001768 bi = vlib_get_buffer_index (vm, b);
Florin Corasbe72ae62018-11-01 11:23:03 -07001769 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Florin Coras93992a92017-05-24 18:03:56 -07001770 tc->snd_nxt = old_snd_nxt;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001771
1772 return 0;
Florin Coras6792ec02017-03-13 03:49:51 -07001773}
1774
Florin Coras36ee9f12018-11-02 12:52:10 -07001775static int
1776tcp_fast_retransmit_unsent (tcp_worker_ctx_t * wrk, tcp_connection_t * tc,
1777 u32 burst_size)
1778{
1779 u32 offset, n_segs = 0, n_written, bi;
1780 vlib_main_t *vm = wrk->vm;
1781 vlib_buffer_t *b = 0;
1782
1783 tc->snd_nxt = tc->snd_una_max;
1784 offset = tc->snd_una_max - tc->snd_una;
1785 while (n_segs < burst_size)
1786 {
1787 n_written = tcp_prepare_segment (wrk, tc, offset, tc->snd_mss, &b);
1788 if (!n_written)
1789 goto done;
1790
1791 bi = vlib_get_buffer_index (vm, b);
1792 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
1793 offset += n_written;
1794 n_segs += 1;
1795 }
1796
1797done:
1798 return n_segs;
1799}
1800
1801#define scoreboard_rescue_rxt_valid(_sb, _tc) \
1802 (seq_geq (_sb->rescue_rxt, _tc->snd_una) \
1803 && seq_leq (_sb->rescue_rxt, _tc->snd_congestion))
1804
Florin Coras6792ec02017-03-13 03:49:51 -07001805/**
Florin Coras93992a92017-05-24 18:03:56 -07001806 * Do fast retransmit with SACKs
Florin Coras6792ec02017-03-13 03:49:51 -07001807 */
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001808int
Florin Corasbe72ae62018-11-01 11:23:03 -07001809tcp_fast_retransmit_sack (tcp_worker_ctx_t * wrk, tcp_connection_t * tc,
1810 u32 burst_size)
Dave Barach68b0fb02017-02-28 15:15:56 -05001811{
Florin Coras36ee9f12018-11-02 12:52:10 -07001812 u32 n_written = 0, offset, max_bytes, n_segs = 0, n_segs_now;
Florin Coras93992a92017-05-24 18:03:56 -07001813 sack_scoreboard_hole_t *hole;
Florin Coras36ee9f12018-11-02 12:52:10 -07001814 vlib_main_t *vm = wrk->vm;
1815 vlib_buffer_t *b = 0;
Florin Coras93992a92017-05-24 18:03:56 -07001816 sack_scoreboard_t *sb;
1817 u32 bi, old_snd_nxt;
1818 int snd_space;
Florin Coras36ee9f12018-11-02 12:52:10 -07001819 u32 max_deq;
Florin Coras93992a92017-05-24 18:03:56 -07001820 u8 snd_limited = 0, can_rescue = 0;
Dave Barach68b0fb02017-02-28 15:15:56 -05001821
1822 ASSERT (tcp_in_fastrecovery (tc));
Dave Barach68b0fb02017-02-28 15:15:56 -05001823
Florin Corasca1c8f32018-05-23 21:01:30 -07001824 snd_space = tcp_available_cc_snd_space (tc);
Florin Corasca1c8f32018-05-23 21:01:30 -07001825 if (snd_space < tc->snd_mss)
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001826 {
Florin Corasbe72ae62018-11-01 11:23:03 -07001827 tcp_program_fastretransmit (wrk, tc);
Florin Coras36ee9f12018-11-02 12:52:10 -07001828 return 0;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001829 }
Florin Corasca1c8f32018-05-23 21:01:30 -07001830
1831 TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 0);
Florin Coras36ee9f12018-11-02 12:52:10 -07001832 old_snd_nxt = tc->snd_nxt;
1833 sb = &tc->sack_sb;
1834 hole = scoreboard_get_hole (sb, sb->cur_rxt_hole);
1835
1836 max_deq = session_tx_fifo_max_dequeue (&tc->connection);
1837 max_deq -= tc->snd_una_max - tc->snd_una;
1838
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001839 while (snd_space > 0 && n_segs < burst_size)
Florin Coras93992a92017-05-24 18:03:56 -07001840 {
Florin Coras36ee9f12018-11-02 12:52:10 -07001841 hole = scoreboard_next_rxt_hole (sb, hole, max_deq, &can_rescue,
1842 &snd_limited);
Florin Coras93992a92017-05-24 18:03:56 -07001843 if (!hole)
1844 {
Florin Coras36ee9f12018-11-02 12:52:10 -07001845 if (max_deq)
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001846 {
Florin Coras36ee9f12018-11-02 12:52:10 -07001847 snd_space = clib_min (max_deq, snd_space);
1848 burst_size = clib_min (burst_size - n_segs,
1849 snd_space / tc->snd_mss);
1850 n_segs_now = tcp_fast_retransmit_unsent (wrk, tc, burst_size);
1851 if (max_deq > n_segs_now * tc->snd_mss)
1852 tcp_program_fastretransmit (wrk, tc);
1853 n_segs += n_segs_now;
1854 goto done;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001855 }
Florin Coras93992a92017-05-24 18:03:56 -07001856
Florin Coras36ee9f12018-11-02 12:52:10 -07001857 if (!can_rescue || scoreboard_rescue_rxt_valid (sb, tc))
1858 break;
1859
Florin Coras93992a92017-05-24 18:03:56 -07001860 /* If rescue rxt undefined or less than snd_una then one segment of
1861 * up to SMSS octets that MUST include the highest outstanding
1862 * unSACKed sequence number SHOULD be returned, and RescueRxt set to
1863 * RecoveryPoint. HighRxt MUST NOT be updated.
1864 */
Florin Coras1f152cd2017-08-18 19:28:03 -07001865 max_bytes = clib_min (tc->snd_mss,
1866 tc->snd_congestion - tc->snd_una);
1867 max_bytes = clib_min (max_bytes, snd_space);
Florin Coras93992a92017-05-24 18:03:56 -07001868 offset = tc->snd_congestion - tc->snd_una - max_bytes;
1869 sb->rescue_rxt = tc->snd_congestion;
1870 tc->snd_nxt = tc->snd_una + offset;
Florin Corasbe72ae62018-11-01 11:23:03 -07001871 n_written = tcp_prepare_retransmit_segment (wrk, tc, offset,
1872 max_bytes, &b);
Florin Coras24793e32018-05-09 11:34:25 -07001873 if (!n_written)
1874 goto done;
1875
Florin Corasb2215d62017-08-01 16:56:58 -07001876 bi = vlib_get_buffer_index (vm, b);
Florin Corasbe72ae62018-11-01 11:23:03 -07001877 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001878 n_segs += 1;
Florin Coras93992a92017-05-24 18:03:56 -07001879 break;
1880 }
1881
Florin Coras1f152cd2017-08-18 19:28:03 -07001882 max_bytes = clib_min (hole->end - sb->high_rxt, snd_space);
1883 max_bytes = snd_limited ? clib_min (max_bytes, tc->snd_mss) : max_bytes;
1884 if (max_bytes == 0)
1885 break;
Florin Coras36ee9f12018-11-02 12:52:10 -07001886
Florin Coras93992a92017-05-24 18:03:56 -07001887 offset = sb->high_rxt - tc->snd_una;
Florin Coras1f152cd2017-08-18 19:28:03 -07001888 tc->snd_nxt = sb->high_rxt;
Florin Corasbe72ae62018-11-01 11:23:03 -07001889 n_written = tcp_prepare_retransmit_segment (wrk, tc, offset, max_bytes,
1890 &b);
Florin Coras36ee9f12018-11-02 12:52:10 -07001891 ASSERT (n_written <= snd_space);
Florin Coras93992a92017-05-24 18:03:56 -07001892
1893 /* Nothing left to retransmit */
1894 if (n_written == 0)
Florin Corasb2215d62017-08-01 16:56:58 -07001895 break;
Florin Coras93992a92017-05-24 18:03:56 -07001896
Florin Corasb2215d62017-08-01 16:56:58 -07001897 bi = vlib_get_buffer_index (vm, b);
Florin Corasbe72ae62018-11-01 11:23:03 -07001898 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Florin Coras36ee9f12018-11-02 12:52:10 -07001899
1900 sb->high_rxt += n_written;
Florin Coras93992a92017-05-24 18:03:56 -07001901 snd_space -= n_written;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001902 n_segs += 1;
Florin Coras93992a92017-05-24 18:03:56 -07001903 }
1904
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001905 if (hole)
Florin Corasbe72ae62018-11-01 11:23:03 -07001906 tcp_program_fastretransmit (wrk, tc);
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001907
Florin Coras24793e32018-05-09 11:34:25 -07001908done:
Florin Coras93992a92017-05-24 18:03:56 -07001909 /* If window allows, send 1 SMSS of new data */
1910 tc->snd_nxt = old_snd_nxt;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001911 return n_segs;
Florin Coras93992a92017-05-24 18:03:56 -07001912}
1913
1914/**
1915 * Fast retransmit without SACK info
1916 */
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001917int
Florin Corasbe72ae62018-11-01 11:23:03 -07001918tcp_fast_retransmit_no_sack (tcp_worker_ctx_t * wrk, tcp_connection_t * tc,
1919 u32 burst_size)
Florin Coras93992a92017-05-24 18:03:56 -07001920{
Florin Coras36ee9f12018-11-02 12:52:10 -07001921 u32 n_written = 0, offset = 0, bi, old_snd_nxt, max_deq, n_segs_now;
Florin Corasbe72ae62018-11-01 11:23:03 -07001922 vlib_main_t *vm = wrk->vm;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001923 int snd_space, n_segs = 0;
Florin Coras93992a92017-05-24 18:03:56 -07001924 vlib_buffer_t *b;
1925
1926 ASSERT (tcp_in_fastrecovery (tc));
1927 TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 0);
Florin Coras93992a92017-05-24 18:03:56 -07001928 old_snd_nxt = tc->snd_nxt;
Florin Coras6792ec02017-03-13 03:49:51 -07001929
Florin Coras36ee9f12018-11-02 12:52:10 -07001930 if (!tcp_fastrecovery_first (tc))
1931 goto send_unsent;
1932
1933 /* RFC 6582: [If a partial ack], retransmit the first unacknowledged
1934 * segment. */
1935 snd_space = tc->sack_sb.last_bytes_delivered;
1936 tc->snd_nxt = tc->snd_una;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001937 while (snd_space > 0 && n_segs < burst_size)
Dave Barach68b0fb02017-02-28 15:15:56 -05001938 {
Florin Coras36ee9f12018-11-02 12:52:10 -07001939 n_written = tcp_prepare_retransmit_segment (wrk, tc, offset,
1940 tc->snd_mss, &b);
Dave Barach68b0fb02017-02-28 15:15:56 -05001941
1942 /* Nothing left to retransmit */
Florin Coras6792ec02017-03-13 03:49:51 -07001943 if (n_written == 0)
Florin Corasb2215d62017-08-01 16:56:58 -07001944 break;
Dave Barach68b0fb02017-02-28 15:15:56 -05001945
Florin Corasb2215d62017-08-01 16:56:58 -07001946 bi = vlib_get_buffer_index (vm, b);
Florin Corasbe72ae62018-11-01 11:23:03 -07001947 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Florin Coras6792ec02017-03-13 03:49:51 -07001948 snd_space -= n_written;
Florin Coras36ee9f12018-11-02 12:52:10 -07001949 offset += n_written;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001950 n_segs += 1;
Dave Barach68b0fb02017-02-28 15:15:56 -05001951 }
1952
Florin Coras36ee9f12018-11-02 12:52:10 -07001953 if (n_segs == burst_size)
1954 goto done;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001955
Florin Coras36ee9f12018-11-02 12:52:10 -07001956send_unsent:
1957
1958 /* RFC 6582: Send a new segment if permitted by the new value of cwnd. */
1959 snd_space = tcp_available_cc_snd_space (tc);
Florin Coras9ece3c02018-11-05 11:06:53 -08001960 if (snd_space < tc->snd_mss || tc->snd_mss == 0)
Florin Coras36ee9f12018-11-02 12:52:10 -07001961 goto done;
1962
1963 max_deq = session_tx_fifo_max_dequeue (&tc->connection);
1964 max_deq -= tc->snd_una_max - tc->snd_una;
1965 if (max_deq)
1966 {
1967 snd_space = clib_min (max_deq, snd_space);
1968 burst_size = clib_min (burst_size - n_segs, snd_space / tc->snd_mss);
1969 n_segs_now = tcp_fast_retransmit_unsent (wrk, tc, burst_size);
1970 if (max_deq > n_segs_now * tc->snd_mss)
1971 tcp_program_fastretransmit (wrk, tc);
1972 n_segs += n_segs_now;
1973 }
1974
1975 /* Restore snd_nxt */
Florin Coras93992a92017-05-24 18:03:56 -07001976 tc->snd_nxt = old_snd_nxt;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001977
Florin Coras36ee9f12018-11-02 12:52:10 -07001978done:
1979 tcp_fastrecovery_first_off (tc);
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001980 return n_segs;
Florin Coras93992a92017-05-24 18:03:56 -07001981}
1982
1983/**
1984 * Do fast retransmit
1985 */
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001986int
Florin Corasbe72ae62018-11-01 11:23:03 -07001987tcp_fast_retransmit (tcp_worker_ctx_t * wrk, tcp_connection_t * tc,
1988 u32 burst_size)
Florin Coras93992a92017-05-24 18:03:56 -07001989{
Florin Corasca1c8f32018-05-23 21:01:30 -07001990 if (tcp_opts_sack_permitted (&tc->rcv_opts))
Florin Corasbe72ae62018-11-01 11:23:03 -07001991 return tcp_fast_retransmit_sack (wrk, tc, burst_size);
Florin Coras93992a92017-05-24 18:03:56 -07001992 else
Florin Corasbe72ae62018-11-01 11:23:03 -07001993 return tcp_fast_retransmit_no_sack (wrk, tc, burst_size);
Dave Barach68b0fb02017-02-28 15:15:56 -05001994}
1995
Florin Corasf9d05682018-04-26 08:26:52 -07001996static void
1997tcp_output_handle_link_local (tcp_connection_t * tc0, vlib_buffer_t * b0,
Florin Coras8b20bf52018-06-14 14:55:50 -07001998 u16 * next0, u32 * error0)
Florin Corasf9d05682018-04-26 08:26:52 -07001999{
2000 ip_adjacency_t *adj;
2001 adj_index_t ai;
2002
Florin Coras1c8ff632018-05-17 13:28:34 -07002003 /* Not thread safe but as long as the connection exists the adj should
2004 * not be removed */
Florin Corasf9d05682018-04-26 08:26:52 -07002005 ai = adj_nbr_find (FIB_PROTOCOL_IP6, VNET_LINK_IP6, &tc0->c_rmt_ip,
2006 tc0->sw_if_index);
2007 if (ai == ADJ_INDEX_INVALID)
2008 {
Florin Corasf9d05682018-04-26 08:26:52 -07002009 vnet_buffer (b0)->sw_if_index[VLIB_TX] = ~0;
2010 *next0 = TCP_OUTPUT_NEXT_DROP;
2011 *error0 = TCP_ERROR_LINK_LOCAL_RW;
2012 return;
2013 }
2014
2015 adj = adj_get (ai);
Florin Coras1c8ff632018-05-17 13:28:34 -07002016 if (PREDICT_TRUE (adj->lookup_next_index == IP_LOOKUP_NEXT_REWRITE))
Florin Corasf9d05682018-04-26 08:26:52 -07002017 *next0 = TCP_OUTPUT_NEXT_IP_REWRITE;
2018 else if (adj->lookup_next_index == IP_LOOKUP_NEXT_ARP)
2019 *next0 = TCP_OUTPUT_NEXT_IP_ARP;
Florin Coras1c8ff632018-05-17 13:28:34 -07002020 else
2021 {
2022 *next0 = TCP_OUTPUT_NEXT_DROP;
2023 *error0 = TCP_ERROR_LINK_LOCAL_RW;
2024 }
2025 vnet_buffer (b0)->ip.adj_index[VLIB_TX] = ai;
Florin Corasf9d05682018-04-26 08:26:52 -07002026}
2027
Florin Coras8b20bf52018-06-14 14:55:50 -07002028static void
2029tcp46_output_trace_frame (vlib_main_t * vm, vlib_node_runtime_t * node,
2030 u32 * to_next, u32 n_bufs)
Dave Barach68b0fb02017-02-28 15:15:56 -05002031{
Florin Coras8b20bf52018-06-14 14:55:50 -07002032 u32 n_trace = vlib_get_trace_count (vm, node);
2033 tcp_connection_t *tc;
2034 tcp_tx_trace_t *t;
2035 vlib_buffer_t *b;
2036 tcp_header_t *th;
2037 int i;
Dave Barach68b0fb02017-02-28 15:15:56 -05002038
Florin Coras8b20bf52018-06-14 14:55:50 -07002039 for (i = 0; i < clib_min (n_trace, n_bufs); i++)
Dave Barach68b0fb02017-02-28 15:15:56 -05002040 {
Florin Coras8b20bf52018-06-14 14:55:50 -07002041 b = vlib_get_buffer (vm, to_next[i]);
2042 th = vlib_buffer_get_current (b);
2043 tc = tcp_connection_get (vnet_buffer (b)->tcp.connection_index,
2044 vm->thread_index);
2045 t = vlib_add_trace (vm, node, b, sizeof (*t));
Dave Barach178cf492018-11-13 16:34:13 -05002046 clib_memcpy_fast (&t->tcp_header, th, sizeof (t->tcp_header));
2047 clib_memcpy_fast (&t->tcp_connection, tc, sizeof (t->tcp_connection));
Florin Coras8b20bf52018-06-14 14:55:50 -07002048 }
2049}
Dave Barach68b0fb02017-02-28 15:15:56 -05002050
Florin Coras0dbd5172018-06-25 16:19:34 -07002051always_inline void
Florin Coras8b20bf52018-06-14 14:55:50 -07002052tcp_output_push_ip (vlib_main_t * vm, vlib_buffer_t * b0,
2053 tcp_connection_t * tc0, u8 is_ip4)
2054{
2055 tcp_header_t *th0 = 0;
Dave Barach68b0fb02017-02-28 15:15:56 -05002056
Florin Coras8b20bf52018-06-14 14:55:50 -07002057 th0 = vlib_buffer_get_current (b0);
2058 TCP_EVT_DBG (TCP_EVT_OUTPUT, tc0, th0->flags, b0->current_length);
2059 if (is_ip4)
2060 {
2061 vlib_buffer_push_ip4 (vm, b0, &tc0->c_lcl_ip4, &tc0->c_rmt_ip4,
2062 IP_PROTOCOL_TCP, 1);
2063 b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
2064 vnet_buffer (b0)->l4_hdr_offset = (u8 *) th0 - b0->data;
2065 th0->checksum = 0;
2066 }
2067 else
2068 {
2069 ip6_header_t *ih0;
2070 ih0 = vlib_buffer_push_ip6 (vm, b0, &tc0->c_lcl_ip6,
2071 &tc0->c_rmt_ip6, IP_PROTOCOL_TCP);
2072 b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
2073 vnet_buffer (b0)->l3_hdr_offset = (u8 *) ih0 - b0->data;
2074 vnet_buffer (b0)->l4_hdr_offset = (u8 *) th0 - b0->data;
2075 th0->checksum = 0;
2076 }
2077}
Dave Barach68b0fb02017-02-28 15:15:56 -05002078
Florin Coras0dbd5172018-06-25 16:19:34 -07002079always_inline void
Florin Coras8b20bf52018-06-14 14:55:50 -07002080tcp_output_handle_packet (tcp_connection_t * tc0, vlib_buffer_t * b0,
2081 u32 * error0, u16 * next0, u8 is_ip4)
2082{
Florin Coras81a13db2018-03-16 08:48:31 -07002083
Florin Coras8b20bf52018-06-14 14:55:50 -07002084 if (PREDICT_FALSE (tc0->state == TCP_STATE_CLOSED))
2085 {
2086 *error0 = TCP_ERROR_INVALID_CONNECTION;
2087 *next0 = TCP_OUTPUT_NEXT_DROP;
2088 return;
Dave Barach68b0fb02017-02-28 15:15:56 -05002089 }
2090
Florin Coras8b20bf52018-06-14 14:55:50 -07002091 vnet_buffer (b0)->sw_if_index[VLIB_TX] = tc0->c_fib_index;
2092 vnet_buffer (b0)->sw_if_index[VLIB_RX] = 0;
2093
2094 if (!is_ip4)
2095 {
2096 if (PREDICT_FALSE (ip6_address_is_link_local_unicast (&tc0->c_rmt_ip6)))
2097 tcp_output_handle_link_local (tc0, b0, next0, error0);
2098 }
2099
Florin Coras8b20bf52018-06-14 14:55:50 -07002100 if (!TCP_ALWAYS_ACK)
2101 tcp_timer_reset (tc0, TCP_TIMER_DELACK);
2102}
2103
2104always_inline uword
2105tcp46_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
2106 vlib_frame_t * frame, int is_ip4)
2107{
2108 u32 n_left_from, *from, thread_index = vm->thread_index;
2109 vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
2110 u16 nexts[VLIB_FRAME_SIZE], *next;
2111
2112 from = vlib_frame_vector_args (frame);
2113 n_left_from = frame->n_vectors;
Florin Corasbe72ae62018-11-01 11:23:03 -07002114 tcp_set_time_now (tcp_get_worker (thread_index));
Florin Coras8b20bf52018-06-14 14:55:50 -07002115
2116 if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
2117 tcp46_output_trace_frame (vm, node, from, n_left_from);
2118
2119 vlib_get_buffers (vm, from, bufs, n_left_from);
2120 b = bufs;
2121 next = nexts;
2122
2123 while (n_left_from >= 4)
2124 {
2125 u32 error0 = TCP_ERROR_PKTS_SENT, error1 = TCP_ERROR_PKTS_SENT;
2126 tcp_connection_t *tc0, *tc1;
2127
2128 {
2129 vlib_prefetch_buffer_header (b[2], STORE);
2130 CLIB_PREFETCH (b[2]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE);
2131
2132 vlib_prefetch_buffer_header (b[3], STORE);
2133 CLIB_PREFETCH (b[3]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE);
2134 }
2135
2136 next[0] = next[1] = TCP_OUTPUT_NEXT_IP_LOOKUP;
2137
2138 tc0 = tcp_connection_get (vnet_buffer (b[0])->tcp.connection_index,
2139 thread_index);
2140 tc1 = tcp_connection_get (vnet_buffer (b[1])->tcp.connection_index,
2141 thread_index);
2142
2143 tcp_output_push_ip (vm, b[0], tc0, is_ip4);
2144 tcp_output_push_ip (vm, b[1], tc1, is_ip4);
2145
2146 tcp_output_handle_packet (tc0, b[0], &error0, &next[0], is_ip4);
2147 tcp_output_handle_packet (tc1, b[1], &error1, &next[1], is_ip4);
2148
2149 b += 2;
2150 next += 2;
2151 n_left_from -= 2;
2152 }
2153 while (n_left_from > 0)
2154 {
2155 u32 error0 = TCP_ERROR_PKTS_SENT;
2156 tcp_connection_t *tc0;
2157
2158 if (n_left_from > 1)
2159 {
Florin Coras91ca4622018-06-30 11:27:59 -07002160 vlib_prefetch_buffer_header (b[1], STORE);
2161 CLIB_PREFETCH (b[1]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE);
Florin Coras8b20bf52018-06-14 14:55:50 -07002162 }
2163
2164 next[0] = TCP_OUTPUT_NEXT_IP_LOOKUP;
2165 tc0 = tcp_connection_get (vnet_buffer (b[0])->tcp.connection_index,
2166 thread_index);
2167
2168 tcp_output_push_ip (vm, b[0], tc0, is_ip4);
2169 tcp_output_handle_packet (tc0, b[0], &error0, &next[0], is_ip4);
2170
2171 b += 1;
2172 next += 1;
2173 n_left_from -= 1;
2174 }
2175
2176 vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
2177 return frame->n_vectors;
Dave Barach68b0fb02017-02-28 15:15:56 -05002178}
2179
2180static uword
2181tcp4_output (vlib_main_t * vm, vlib_node_runtime_t * node,
2182 vlib_frame_t * from_frame)
2183{
2184 return tcp46_output_inline (vm, node, from_frame, 1 /* is_ip4 */ );
2185}
2186
2187static uword
2188tcp6_output (vlib_main_t * vm, vlib_node_runtime_t * node,
2189 vlib_frame_t * from_frame)
2190{
2191 return tcp46_output_inline (vm, node, from_frame, 0 /* is_ip4 */ );
2192}
2193
Florin Corase69f4952017-03-07 10:06:24 -08002194/* *INDENT-OFF* */
Dave Barach68b0fb02017-02-28 15:15:56 -05002195VLIB_REGISTER_NODE (tcp4_output_node) =
2196{
Dave Barachf970d2f2018-11-26 18:34:33 -05002197 .function = tcp4_output,
2198 .name = "tcp4-output",
2199 /* Takes a vector of packets. */
2200 .vector_size = sizeof (u32),
2201 .n_errors = TCP_N_ERROR,
Dave Barach7fff3d22018-11-27 16:52:59 -05002202 .protocol_hint = VLIB_NODE_PROTO_HINT_TCP,
Dave Barachf970d2f2018-11-26 18:34:33 -05002203 .error_strings = tcp_error_strings,
2204 .n_next_nodes = TCP_OUTPUT_N_NEXT,
2205 .next_nodes = {
Dave Barach68b0fb02017-02-28 15:15:56 -05002206#define _(s,n) [TCP_OUTPUT_NEXT_##s] = n,
2207 foreach_tcp4_output_next
2208#undef _
Dave Barachf970d2f2018-11-26 18:34:33 -05002209 },
2210 .format_buffer = format_tcp_header,
2211 .format_trace = format_tcp_tx_trace,
Florin Corase69f4952017-03-07 10:06:24 -08002212};
2213/* *INDENT-ON* */
Dave Barach68b0fb02017-02-28 15:15:56 -05002214
Florin Corase69f4952017-03-07 10:06:24 -08002215VLIB_NODE_FUNCTION_MULTIARCH (tcp4_output_node, tcp4_output);
2216
2217/* *INDENT-OFF* */
Dave Barach68b0fb02017-02-28 15:15:56 -05002218VLIB_REGISTER_NODE (tcp6_output_node) =
2219{
Florin Corase69f4952017-03-07 10:06:24 -08002220 .function = tcp6_output,
2221 .name = "tcp6-output",
Dave Barach68b0fb02017-02-28 15:15:56 -05002222 /* Takes a vector of packets. */
Florin Corase69f4952017-03-07 10:06:24 -08002223 .vector_size = sizeof (u32),
2224 .n_errors = TCP_N_ERROR,
Dave Barach7fff3d22018-11-27 16:52:59 -05002225 .protocol_hint = VLIB_NODE_PROTO_HINT_TCP,
Florin Corase69f4952017-03-07 10:06:24 -08002226 .error_strings = tcp_error_strings,
2227 .n_next_nodes = TCP_OUTPUT_N_NEXT,
2228 .next_nodes = {
Dave Barach68b0fb02017-02-28 15:15:56 -05002229#define _(s,n) [TCP_OUTPUT_NEXT_##s] = n,
2230 foreach_tcp6_output_next
2231#undef _
Florin Corase69f4952017-03-07 10:06:24 -08002232 },
2233 .format_buffer = format_tcp_header,
2234 .format_trace = format_tcp_tx_trace,
2235};
2236/* *INDENT-ON* */
Dave Barach68b0fb02017-02-28 15:15:56 -05002237
Florin Corase69f4952017-03-07 10:06:24 -08002238VLIB_NODE_FUNCTION_MULTIARCH (tcp6_output_node, tcp6_output);
2239
Dave Barach68b0fb02017-02-28 15:15:56 -05002240typedef enum _tcp_reset_next
2241{
2242 TCP_RESET_NEXT_DROP,
2243 TCP_RESET_NEXT_IP_LOOKUP,
2244 TCP_RESET_N_NEXT
2245} tcp_reset_next_t;
2246
2247#define foreach_tcp4_reset_next \
2248 _(DROP, "error-drop") \
2249 _(IP_LOOKUP, "ip4-lookup")
2250
2251#define foreach_tcp6_reset_next \
2252 _(DROP, "error-drop") \
2253 _(IP_LOOKUP, "ip6-lookup")
2254
2255static uword
2256tcp46_send_reset_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
2257 vlib_frame_t * from_frame, u8 is_ip4)
2258{
2259 u32 n_left_from, next_index, *from, *to_next;
Damjan Marion586afd72017-04-05 19:18:20 +02002260 u32 my_thread_index = vm->thread_index;
Dave Barach68b0fb02017-02-28 15:15:56 -05002261
2262 from = vlib_frame_vector_args (from_frame);
2263 n_left_from = from_frame->n_vectors;
2264
2265 next_index = node->cached_next_index;
2266
2267 while (n_left_from > 0)
2268 {
2269 u32 n_left_to_next;
2270
2271 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2272
2273 while (n_left_from > 0 && n_left_to_next > 0)
2274 {
2275 u32 bi0;
2276 vlib_buffer_t *b0;
Clement Durand6cf260c2017-04-13 13:27:04 +02002277 tcp_tx_trace_t *t0;
2278 tcp_header_t *th0;
Dave Barach68b0fb02017-02-28 15:15:56 -05002279 u32 error0 = TCP_ERROR_RST_SENT, next0 = TCP_RESET_NEXT_IP_LOOKUP;
2280
2281 bi0 = from[0];
2282 to_next[0] = bi0;
2283 from += 1;
2284 to_next += 1;
2285 n_left_from -= 1;
2286 n_left_to_next -= 1;
2287
2288 b0 = vlib_get_buffer (vm, bi0);
2289
2290 if (tcp_make_reset_in_place (vm, b0, vnet_buffer (b0)->tcp.flags,
2291 my_thread_index, is_ip4))
2292 {
2293 error0 = TCP_ERROR_LOOKUP_DROPS;
2294 next0 = TCP_RESET_NEXT_DROP;
2295 goto done;
2296 }
2297
2298 /* Prepare to send to IP lookup */
Florin Corasf988e692017-11-27 04:34:14 -05002299 vnet_buffer (b0)->sw_if_index[VLIB_TX] = ~0;
Dave Barach68b0fb02017-02-28 15:15:56 -05002300 next0 = TCP_RESET_NEXT_IP_LOOKUP;
2301
2302 done:
Florin Corase69f4952017-03-07 10:06:24 -08002303 b0->error = node->errors[error0];
Damjan Marion213b5aa2017-07-13 21:19:27 +02002304 b0->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
Dave Barach68b0fb02017-02-28 15:15:56 -05002305 if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
2306 {
Clement Durand6cf260c2017-04-13 13:27:04 +02002307 th0 = vlib_buffer_get_current (b0);
2308 if (is_ip4)
2309 th0 = ip4_next_header ((ip4_header_t *) th0);
2310 else
2311 th0 = ip6_next_header ((ip6_header_t *) th0);
Clement Durand6cf260c2017-04-13 13:27:04 +02002312 t0 = vlib_add_trace (vm, node, b0, sizeof (*t0));
Dave Barach178cf492018-11-13 16:34:13 -05002313 clib_memcpy_fast (&t0->tcp_header, th0,
2314 sizeof (t0->tcp_header));
Dave Barach68b0fb02017-02-28 15:15:56 -05002315 }
2316
2317 vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
2318 n_left_to_next, bi0, next0);
2319 }
2320 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2321 }
2322 return from_frame->n_vectors;
2323}
2324
2325static uword
2326tcp4_send_reset (vlib_main_t * vm, vlib_node_runtime_t * node,
2327 vlib_frame_t * from_frame)
2328{
2329 return tcp46_send_reset_inline (vm, node, from_frame, 1);
2330}
2331
2332static uword
2333tcp6_send_reset (vlib_main_t * vm, vlib_node_runtime_t * node,
2334 vlib_frame_t * from_frame)
2335{
2336 return tcp46_send_reset_inline (vm, node, from_frame, 0);
2337}
2338
2339/* *INDENT-OFF* */
2340VLIB_REGISTER_NODE (tcp4_reset_node) = {
2341 .function = tcp4_send_reset,
2342 .name = "tcp4-reset",
2343 .vector_size = sizeof (u32),
2344 .n_errors = TCP_N_ERROR,
2345 .error_strings = tcp_error_strings,
2346 .n_next_nodes = TCP_RESET_N_NEXT,
2347 .next_nodes = {
2348#define _(s,n) [TCP_RESET_NEXT_##s] = n,
2349 foreach_tcp4_reset_next
2350#undef _
2351 },
Clement Durand6cf260c2017-04-13 13:27:04 +02002352 .format_trace = format_tcp_tx_trace,
Dave Barach68b0fb02017-02-28 15:15:56 -05002353};
2354/* *INDENT-ON* */
2355
Florin Corase69f4952017-03-07 10:06:24 -08002356VLIB_NODE_FUNCTION_MULTIARCH (tcp4_reset_node, tcp4_send_reset);
2357
Dave Barach68b0fb02017-02-28 15:15:56 -05002358/* *INDENT-OFF* */
2359VLIB_REGISTER_NODE (tcp6_reset_node) = {
2360 .function = tcp6_send_reset,
2361 .name = "tcp6-reset",
2362 .vector_size = sizeof (u32),
2363 .n_errors = TCP_N_ERROR,
2364 .error_strings = tcp_error_strings,
2365 .n_next_nodes = TCP_RESET_N_NEXT,
2366 .next_nodes = {
2367#define _(s,n) [TCP_RESET_NEXT_##s] = n,
2368 foreach_tcp6_reset_next
2369#undef _
2370 },
Clement Durand6cf260c2017-04-13 13:27:04 +02002371 .format_trace = format_tcp_tx_trace,
Dave Barach68b0fb02017-02-28 15:15:56 -05002372};
2373/* *INDENT-ON* */
2374
Florin Corase69f4952017-03-07 10:06:24 -08002375VLIB_NODE_FUNCTION_MULTIARCH (tcp6_reset_node, tcp6_send_reset);
2376
Dave Barach68b0fb02017-02-28 15:15:56 -05002377/*
2378 * fd.io coding-style-patch-verification: ON
2379 *
2380 * Local Variables:
2381 * eval: (c-set-style "gnu")
2382 * End:
2383 */