blob: abd674d94be416e255fd73447f07cef5ee338251 [file] [log] [blame]
Dave Barach68b0fb02017-02-28 15:15:56 -05001/*
Florin Coras222e1f412019-02-16 20:47:32 -08002 * Copyright (c) 2016-2019 Cisco and/or its affiliates.
Dave Barach68b0fb02017-02-28 15:15:56 -05003 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16#include <vnet/tcp/tcp.h>
Florin Corasb2215d62017-08-01 16:56:58 -070017#include <math.h>
Dave Barach68b0fb02017-02-28 15:15:56 -050018
Dave Barach2c25a622017-06-26 11:35:07 -040019typedef enum _tcp_output_next
Dave Barach68b0fb02017-02-28 15:15:56 -050020{
21 TCP_OUTPUT_NEXT_DROP,
Dave Barach2c25a622017-06-26 11:35:07 -040022 TCP_OUTPUT_NEXT_IP_LOOKUP,
Florin Corasf9d05682018-04-26 08:26:52 -070023 TCP_OUTPUT_NEXT_IP_REWRITE,
24 TCP_OUTPUT_NEXT_IP_ARP,
Dave Barach68b0fb02017-02-28 15:15:56 -050025 TCP_OUTPUT_N_NEXT
26} tcp_output_next_t;
27
28#define foreach_tcp4_output_next \
29 _ (DROP, "error-drop") \
Florin Corasf9d05682018-04-26 08:26:52 -070030 _ (IP_LOOKUP, "ip4-lookup") \
31 _ (IP_REWRITE, "ip4-rewrite") \
32 _ (IP_ARP, "ip4-arp")
Dave Barach68b0fb02017-02-28 15:15:56 -050033
34#define foreach_tcp6_output_next \
35 _ (DROP, "error-drop") \
Florin Corasf9d05682018-04-26 08:26:52 -070036 _ (IP_LOOKUP, "ip6-lookup") \
37 _ (IP_REWRITE, "ip6-rewrite") \
38 _ (IP_ARP, "ip6-discover-neighbor")
Dave Barach68b0fb02017-02-28 15:15:56 -050039
40static char *tcp_error_strings[] = {
41#define tcp_error(n,s) s,
42#include <vnet/tcp/tcp_error.def>
43#undef tcp_error
44};
45
46typedef struct
47{
Clement Durand6cf260c2017-04-13 13:27:04 +020048 tcp_header_t tcp_header;
49 tcp_connection_t tcp_connection;
Dave Barach68b0fb02017-02-28 15:15:56 -050050} tcp_tx_trace_t;
51
Filip Tehlare275bed2019-03-06 00:06:56 -080052static u8 *
Dave Barach68b0fb02017-02-28 15:15:56 -050053format_tcp_tx_trace (u8 * s, va_list * args)
54{
55 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
56 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
Clement Durand6cf260c2017-04-13 13:27:04 +020057 tcp_tx_trace_t *t = va_arg (*args, tcp_tx_trace_t *);
Christophe Fontained3c008d2017-10-02 18:10:54 +020058 u32 indent = format_get_indent (s);
Dave Barach68b0fb02017-02-28 15:15:56 -050059
Clement Durand6cf260c2017-04-13 13:27:04 +020060 s = format (s, "%U\n%U%U",
61 format_tcp_header, &t->tcp_header, 128,
62 format_white_space, indent,
Florin Corasbb292f42017-05-19 09:49:19 -070063 format_tcp_connection, &t->tcp_connection, 1);
Dave Barach68b0fb02017-02-28 15:15:56 -050064
65 return s;
66}
67
Filip Tehlare275bed2019-03-06 00:06:56 -080068#ifndef CLIB_MARCH_VARIANT
Dave Barach68b0fb02017-02-28 15:15:56 -050069static u8
Florin Coras4eeeaaf2017-09-05 14:03:37 -040070tcp_window_compute_scale (u32 window)
Dave Barach68b0fb02017-02-28 15:15:56 -050071{
72 u8 wnd_scale = 0;
Florin Coras4eeeaaf2017-09-05 14:03:37 -040073 while (wnd_scale < TCP_MAX_WND_SCALE && (window >> wnd_scale) > TCP_WND_MAX)
Dave Barach68b0fb02017-02-28 15:15:56 -050074 wnd_scale++;
75 return wnd_scale;
76}
77
78/**
Florin Coras6534b7a2017-07-18 05:38:03 -040079 * Update max segment size we're able to process.
80 *
81 * The value is constrained by our interface's MTU and IP options. It is
82 * also what we advertise to our peer.
83 */
84void
85tcp_update_rcv_mss (tcp_connection_t * tc)
86{
87 /* TODO find our iface MTU */
Florin Coras537e85d2019-03-22 15:42:18 -070088 tc->mss = tcp_main.default_mtu - sizeof (tcp_header_t);
Florin Coras6534b7a2017-07-18 05:38:03 -040089}
90
91/**
92 * TCP's initial window
Florin Corase04c2992017-03-01 08:17:34 -080093 */
94always_inline u32
95tcp_initial_wnd_unscaled (tcp_connection_t * tc)
96{
Florin Coras6534b7a2017-07-18 05:38:03 -040097 /* RFC 6928 recommends the value lower. However at the time our connections
98 * are initialized, fifos may not be allocated. Therefore, advertise the
99 * smallest possible unscaled window size and update once fifos are
100 * assigned to the session.
101 */
102 /*
103 tcp_update_rcv_mss (tc);
104 TCP_IW_N_SEGMENTS * tc->mss;
105 */
106 return TCP_MIN_RX_FIFO_SIZE;
Florin Corase04c2992017-03-01 08:17:34 -0800107}
108
109/**
Dave Barach68b0fb02017-02-28 15:15:56 -0500110 * Compute initial window and scale factor. As per RFC1323, window field in
111 * SYN and SYN-ACK segments is never scaled.
112 */
113u32
114tcp_initial_window_to_advertise (tcp_connection_t * tc)
115{
Florin Corasca031862018-09-24 13:58:05 -0700116 tcp_main_t *tm = &tcp_main;
Florin Corase04c2992017-03-01 08:17:34 -0800117 u32 max_fifo;
Dave Barach68b0fb02017-02-28 15:15:56 -0500118
119 /* Initial wnd for SYN. Fifos are not allocated yet.
Florin Corase04c2992017-03-01 08:17:34 -0800120 * Use some predefined value. For SYN-ACK we still want the
121 * scale to be computed in the same way */
Florin Corasca031862018-09-24 13:58:05 -0700122 max_fifo = tm->max_rx_fifo ? tm->max_rx_fifo : TCP_MAX_RX_FIFO_SIZE;
Dave Barach68b0fb02017-02-28 15:15:56 -0500123
Florin Corase80b5912018-12-12 19:25:43 -0800124 /* Compute rcv wscale only if peer advertised support for it */
125 if (tc->state != TCP_STATE_SYN_RCVD || tcp_opts_wscale (&tc->rcv_opts))
126 tc->rcv_wscale = tcp_window_compute_scale (max_fifo);
127
Florin Corase04c2992017-03-01 08:17:34 -0800128 tc->rcv_wnd = tcp_initial_wnd_unscaled (tc);
Dave Barach68b0fb02017-02-28 15:15:56 -0500129
130 return clib_min (tc->rcv_wnd, TCP_WND_MAX);
131}
132
Florin Corasf20fd1a2019-03-28 13:21:19 -0700133static inline void
Florin Coras6792ec02017-03-13 03:49:51 -0700134tcp_update_rcv_wnd (tcp_connection_t * tc)
135{
Florin Corasf20fd1a2019-03-28 13:21:19 -0700136 u32 available_space, wnd;
Florin Coras6792ec02017-03-13 03:49:51 -0700137 i32 observed_wnd;
Florin Corasf20fd1a2019-03-28 13:21:19 -0700138
139 ASSERT (tc->rcv_opts.mss < transport_rx_fifo_size (&tc->connection));
Florin Coras6792ec02017-03-13 03:49:51 -0700140
Florin Corase04c2992017-03-01 08:17:34 -0800141 /*
142 * Figure out how much space we have available
143 */
Florin Corasd2aab832018-05-22 11:39:59 -0700144 available_space = transport_max_rx_enqueue (&tc->connection);
Florin Corasf20fd1a2019-03-28 13:21:19 -0700145 if (PREDICT_FALSE (available_space < tc->rcv_opts.mss))
Florin Corase04c2992017-03-01 08:17:34 -0800146 available_space = 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500147
Florin Corase04c2992017-03-01 08:17:34 -0800148 /*
149 * Use the above and what we know about what we've previously advertised
150 * to compute the new window
151 */
Florin Coras6792ec02017-03-13 03:49:51 -0700152 observed_wnd = (i32) tc->rcv_wnd - (tc->rcv_nxt - tc->rcv_las);
Florin Corase04c2992017-03-01 08:17:34 -0800153
154 /* Bad. Thou shalt not shrink */
Florin Corasf20fd1a2019-03-28 13:21:19 -0700155 if (PREDICT_FALSE ((i32) available_space < observed_wnd))
Florin Corase04c2992017-03-01 08:17:34 -0800156 {
Florin Corasf20fd1a2019-03-28 13:21:19 -0700157 wnd = clib_max (observed_wnd, 0);
Florin Coras3e350af2017-03-30 02:54:28 -0700158 TCP_EVT_DBG (TCP_EVT_RCV_WND_SHRUNK, tc, observed_wnd, available_space);
Florin Corase04c2992017-03-01 08:17:34 -0800159 }
Florin Coras6792ec02017-03-13 03:49:51 -0700160 else
Florin Corase04c2992017-03-01 08:17:34 -0800161 {
Florin Coras6792ec02017-03-13 03:49:51 -0700162 wnd = available_space;
Florin Corase04c2992017-03-01 08:17:34 -0800163 }
164
Florin Coras3e350af2017-03-30 02:54:28 -0700165 /* Make sure we have a multiple of rcv_wscale */
166 if (wnd && tc->rcv_wscale)
167 {
Florin Coras679b78f2019-03-06 21:16:26 -0800168 wnd &= ~((1 << tc->rcv_wscale) - 1);
Florin Coras3e350af2017-03-30 02:54:28 -0700169 if (wnd == 0)
170 wnd = 1 << tc->rcv_wscale;
171 }
Florin Coras6792ec02017-03-13 03:49:51 -0700172
173 tc->rcv_wnd = clib_min (wnd, TCP_WND_MAX << tc->rcv_wscale);
Dave Barach68b0fb02017-02-28 15:15:56 -0500174}
175
176/**
Florin Coras0dbd5172018-06-25 16:19:34 -0700177 * Compute and return window to advertise, scaled as per RFC1323
178 */
Florin Coras47596832019-03-12 18:58:54 -0700179static inline u32
Florin Coras0dbd5172018-06-25 16:19:34 -0700180tcp_window_to_advertise (tcp_connection_t * tc, tcp_state_t state)
181{
182 if (state < TCP_STATE_ESTABLISHED)
183 return tcp_initial_window_to_advertise (tc);
184
185 tcp_update_rcv_wnd (tc);
Florin Coras0dbd5172018-06-25 16:19:34 -0700186 return tc->rcv_wnd >> tc->rcv_wscale;
187}
188
189/**
Dave Barach68b0fb02017-02-28 15:15:56 -0500190 * Write TCP options to segment.
191 */
Florin Coras0dbd5172018-06-25 16:19:34 -0700192static u32
Dave Barach68b0fb02017-02-28 15:15:56 -0500193tcp_options_write (u8 * data, tcp_options_t * opts)
194{
195 u32 opts_len = 0;
196 u32 buf, seq_len = 4;
197
198 if (tcp_opts_mss (opts))
199 {
200 *data++ = TCP_OPTION_MSS;
201 *data++ = TCP_OPTION_LEN_MSS;
202 buf = clib_host_to_net_u16 (opts->mss);
Dave Barach178cf492018-11-13 16:34:13 -0500203 clib_memcpy_fast (data, &buf, sizeof (opts->mss));
Dave Barach68b0fb02017-02-28 15:15:56 -0500204 data += sizeof (opts->mss);
205 opts_len += TCP_OPTION_LEN_MSS;
206 }
207
208 if (tcp_opts_wscale (opts))
209 {
210 *data++ = TCP_OPTION_WINDOW_SCALE;
211 *data++ = TCP_OPTION_LEN_WINDOW_SCALE;
212 *data++ = opts->wscale;
213 opts_len += TCP_OPTION_LEN_WINDOW_SCALE;
214 }
215
216 if (tcp_opts_sack_permitted (opts))
217 {
218 *data++ = TCP_OPTION_SACK_PERMITTED;
219 *data++ = TCP_OPTION_LEN_SACK_PERMITTED;
220 opts_len += TCP_OPTION_LEN_SACK_PERMITTED;
221 }
222
223 if (tcp_opts_tstamp (opts))
224 {
225 *data++ = TCP_OPTION_TIMESTAMP;
226 *data++ = TCP_OPTION_LEN_TIMESTAMP;
227 buf = clib_host_to_net_u32 (opts->tsval);
Dave Barach178cf492018-11-13 16:34:13 -0500228 clib_memcpy_fast (data, &buf, sizeof (opts->tsval));
Dave Barach68b0fb02017-02-28 15:15:56 -0500229 data += sizeof (opts->tsval);
230 buf = clib_host_to_net_u32 (opts->tsecr);
Dave Barach178cf492018-11-13 16:34:13 -0500231 clib_memcpy_fast (data, &buf, sizeof (opts->tsecr));
Dave Barach68b0fb02017-02-28 15:15:56 -0500232 data += sizeof (opts->tsecr);
233 opts_len += TCP_OPTION_LEN_TIMESTAMP;
234 }
235
236 if (tcp_opts_sack (opts))
237 {
238 int i;
Dave Barach68b0fb02017-02-28 15:15:56 -0500239
Florin Corase5b17912019-02-21 16:46:24 -0800240 if (opts->n_sack_blocks != 0)
Dave Barach68b0fb02017-02-28 15:15:56 -0500241 {
242 *data++ = TCP_OPTION_SACK_BLOCK;
Florin Corase5b17912019-02-21 16:46:24 -0800243 *data++ = 2 + opts->n_sack_blocks * TCP_OPTION_LEN_SACK_BLOCK;
244 for (i = 0; i < opts->n_sack_blocks; i++)
Dave Barach68b0fb02017-02-28 15:15:56 -0500245 {
246 buf = clib_host_to_net_u32 (opts->sacks[i].start);
Dave Barach178cf492018-11-13 16:34:13 -0500247 clib_memcpy_fast (data, &buf, seq_len);
Dave Barach68b0fb02017-02-28 15:15:56 -0500248 data += seq_len;
249 buf = clib_host_to_net_u32 (opts->sacks[i].end);
Dave Barach178cf492018-11-13 16:34:13 -0500250 clib_memcpy_fast (data, &buf, seq_len);
Dave Barach68b0fb02017-02-28 15:15:56 -0500251 data += seq_len;
252 }
Florin Corase5b17912019-02-21 16:46:24 -0800253 opts_len += 2 + opts->n_sack_blocks * TCP_OPTION_LEN_SACK_BLOCK;
Dave Barach68b0fb02017-02-28 15:15:56 -0500254 }
255 }
256
257 /* Terminate TCP options */
258 if (opts_len % 4)
259 {
260 *data++ = TCP_OPTION_EOL;
261 opts_len += TCP_OPTION_LEN_EOL;
262 }
263
264 /* Pad with zeroes to a u32 boundary */
265 while (opts_len % 4)
266 {
267 *data++ = TCP_OPTION_NOOP;
268 opts_len += TCP_OPTION_LEN_NOOP;
269 }
270 return opts_len;
271}
272
Florin Coras0dbd5172018-06-25 16:19:34 -0700273static int
Florin Corase04c2992017-03-01 08:17:34 -0800274tcp_make_syn_options (tcp_options_t * opts, u8 wnd_scale)
Dave Barach68b0fb02017-02-28 15:15:56 -0500275{
276 u8 len = 0;
277
278 opts->flags |= TCP_OPTS_FLAG_MSS;
Florin Coras537e85d2019-03-22 15:42:18 -0700279 opts->mss = tcp_main.default_mtu; /*XXX discover that */
Dave Barach68b0fb02017-02-28 15:15:56 -0500280 len += TCP_OPTION_LEN_MSS;
281
282 opts->flags |= TCP_OPTS_FLAG_WSCALE;
Florin Corase04c2992017-03-01 08:17:34 -0800283 opts->wscale = wnd_scale;
Dave Barach68b0fb02017-02-28 15:15:56 -0500284 len += TCP_OPTION_LEN_WINDOW_SCALE;
285
286 opts->flags |= TCP_OPTS_FLAG_TSTAMP;
287 opts->tsval = tcp_time_now ();
288 opts->tsecr = 0;
289 len += TCP_OPTION_LEN_TIMESTAMP;
290
Florin Coras93992a92017-05-24 18:03:56 -0700291 if (TCP_USE_SACKS)
292 {
293 opts->flags |= TCP_OPTS_FLAG_SACK_PERMITTED;
294 len += TCP_OPTION_LEN_SACK_PERMITTED;
295 }
Dave Barach68b0fb02017-02-28 15:15:56 -0500296
297 /* Align to needed boundary */
298 len += (TCP_OPTS_ALIGN - len % TCP_OPTS_ALIGN) % TCP_OPTS_ALIGN;
299 return len;
300}
301
Florin Coras0dbd5172018-06-25 16:19:34 -0700302static int
Dave Barach68b0fb02017-02-28 15:15:56 -0500303tcp_make_synack_options (tcp_connection_t * tc, tcp_options_t * opts)
304{
305 u8 len = 0;
306
307 opts->flags |= TCP_OPTS_FLAG_MSS;
Florin Corasc8343412017-05-04 14:25:50 -0700308 opts->mss = tc->mss;
Dave Barach68b0fb02017-02-28 15:15:56 -0500309 len += TCP_OPTION_LEN_MSS;
310
Florin Coras93992a92017-05-24 18:03:56 -0700311 if (tcp_opts_wscale (&tc->rcv_opts))
Dave Barach68b0fb02017-02-28 15:15:56 -0500312 {
313 opts->flags |= TCP_OPTS_FLAG_WSCALE;
314 opts->wscale = tc->rcv_wscale;
315 len += TCP_OPTION_LEN_WINDOW_SCALE;
316 }
317
Florin Coras93992a92017-05-24 18:03:56 -0700318 if (tcp_opts_tstamp (&tc->rcv_opts))
Dave Barach68b0fb02017-02-28 15:15:56 -0500319 {
320 opts->flags |= TCP_OPTS_FLAG_TSTAMP;
321 opts->tsval = tcp_time_now ();
322 opts->tsecr = tc->tsval_recent;
323 len += TCP_OPTION_LEN_TIMESTAMP;
324 }
325
Florin Coras93992a92017-05-24 18:03:56 -0700326 if (tcp_opts_sack_permitted (&tc->rcv_opts))
Dave Barach68b0fb02017-02-28 15:15:56 -0500327 {
328 opts->flags |= TCP_OPTS_FLAG_SACK_PERMITTED;
329 len += TCP_OPTION_LEN_SACK_PERMITTED;
330 }
331
332 /* Align to needed boundary */
333 len += (TCP_OPTS_ALIGN - len % TCP_OPTS_ALIGN) % TCP_OPTS_ALIGN;
334 return len;
335}
336
Florin Coras0dbd5172018-06-25 16:19:34 -0700337static int
Dave Barach68b0fb02017-02-28 15:15:56 -0500338tcp_make_established_options (tcp_connection_t * tc, tcp_options_t * opts)
339{
340 u8 len = 0;
341
342 opts->flags = 0;
343
Florin Coras93992a92017-05-24 18:03:56 -0700344 if (tcp_opts_tstamp (&tc->rcv_opts))
Dave Barach68b0fb02017-02-28 15:15:56 -0500345 {
346 opts->flags |= TCP_OPTS_FLAG_TSTAMP;
Florin Corasbe72ae62018-11-01 11:23:03 -0700347 opts->tsval = tcp_time_now_w_thread (tc->c_thread_index);
Dave Barach68b0fb02017-02-28 15:15:56 -0500348 opts->tsecr = tc->tsval_recent;
349 len += TCP_OPTION_LEN_TIMESTAMP;
350 }
Florin Coras93992a92017-05-24 18:03:56 -0700351 if (tcp_opts_sack_permitted (&tc->rcv_opts))
Dave Barach68b0fb02017-02-28 15:15:56 -0500352 {
353 if (vec_len (tc->snd_sacks))
354 {
355 opts->flags |= TCP_OPTS_FLAG_SACK;
Florin Corase5b17912019-02-21 16:46:24 -0800356 if (tc->snd_sack_pos >= vec_len (tc->snd_sacks))
357 tc->snd_sack_pos = 0;
358 opts->sacks = &tc->snd_sacks[tc->snd_sack_pos];
359 opts->n_sack_blocks = vec_len (tc->snd_sacks) - tc->snd_sack_pos;
360 opts->n_sack_blocks = clib_min (opts->n_sack_blocks,
Florin Corasc28764f2017-04-26 00:08:42 -0700361 TCP_OPTS_MAX_SACK_BLOCKS);
Florin Corase5b17912019-02-21 16:46:24 -0800362 tc->snd_sack_pos += opts->n_sack_blocks;
Dave Barach68b0fb02017-02-28 15:15:56 -0500363 len += 2 + TCP_OPTION_LEN_SACK_BLOCK * opts->n_sack_blocks;
364 }
365 }
366
367 /* Align to needed boundary */
368 len += (TCP_OPTS_ALIGN - len % TCP_OPTS_ALIGN) % TCP_OPTS_ALIGN;
369 return len;
370}
371
372always_inline int
373tcp_make_options (tcp_connection_t * tc, tcp_options_t * opts,
374 tcp_state_t state)
375{
376 switch (state)
377 {
378 case TCP_STATE_ESTABLISHED:
Florin Coras25579b42018-06-06 17:55:02 -0700379 case TCP_STATE_CLOSE_WAIT:
Florin Coras54ddf432018-12-21 13:54:09 -0800380 case TCP_STATE_FIN_WAIT_1:
381 case TCP_STATE_LAST_ACK:
382 case TCP_STATE_CLOSING:
383 case TCP_STATE_FIN_WAIT_2:
384 case TCP_STATE_TIME_WAIT:
385 case TCP_STATE_CLOSED:
Dave Barach68b0fb02017-02-28 15:15:56 -0500386 return tcp_make_established_options (tc, opts);
387 case TCP_STATE_SYN_RCVD:
388 return tcp_make_synack_options (tc, opts);
389 case TCP_STATE_SYN_SENT:
Florin Corase04c2992017-03-01 08:17:34 -0800390 return tcp_make_syn_options (opts, tc->rcv_wscale);
Dave Barach68b0fb02017-02-28 15:15:56 -0500391 default:
Florin Coras371ca502018-02-21 12:07:41 -0800392 clib_warning ("State not handled! %d", state);
Dave Barach68b0fb02017-02-28 15:15:56 -0500393 return 0;
394 }
395}
396
Florin Corasc8343412017-05-04 14:25:50 -0700397/**
Florin Corasb26743d2018-06-26 09:31:04 -0700398 * Update burst send vars
399 *
400 * - Updates snd_mss to reflect the effective segment size that we can send
401 * by taking into account all TCP options, including SACKs.
402 * - Cache 'on the wire' options for reuse
403 * - Updates receive window which can be reused for a burst.
404 *
405 * This should *only* be called when doing bursts
Florin Corasc8343412017-05-04 14:25:50 -0700406 */
407void
Florin Corasb26743d2018-06-26 09:31:04 -0700408tcp_update_burst_snd_vars (tcp_connection_t * tc)
Florin Corasc8343412017-05-04 14:25:50 -0700409{
Florin Corasb26743d2018-06-26 09:31:04 -0700410 tcp_main_t *tm = &tcp_main;
411
Florin Corasc8343412017-05-04 14:25:50 -0700412 /* Compute options to be used for connection. These may be reused when
413 * sending data or to compute the effective mss (snd_mss) */
Florin Corasb26743d2018-06-26 09:31:04 -0700414 tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts,
415 TCP_STATE_ESTABLISHED);
Florin Corasc8343412017-05-04 14:25:50 -0700416
417 /* XXX check if MTU has been updated */
Florin Coras93992a92017-05-24 18:03:56 -0700418 tc->snd_mss = clib_min (tc->mss, tc->rcv_opts.mss) - tc->snd_opts_len;
Florin Corasdb84e572017-05-09 18:54:52 -0700419 ASSERT (tc->snd_mss > 0);
Florin Corasb26743d2018-06-26 09:31:04 -0700420
421 tcp_options_write (tm->wrk_ctx[tc->c_thread_index].cached_opts,
422 &tc->snd_opts);
423
424 tcp_update_rcv_wnd (tc);
Florin Corasc8343412017-05-04 14:25:50 -0700425}
426
427void
428tcp_init_mss (tcp_connection_t * tc)
429{
Florin Corasdb84e572017-05-09 18:54:52 -0700430 u16 default_min_mss = 536;
Florin Corasc8343412017-05-04 14:25:50 -0700431 tcp_update_rcv_mss (tc);
432
433 /* TODO cache mss and consider PMTU discovery */
Florin Coras93992a92017-05-24 18:03:56 -0700434 tc->snd_mss = clib_min (tc->rcv_opts.mss, tc->mss);
Florin Corasc8343412017-05-04 14:25:50 -0700435
Florin Corasdb84e572017-05-09 18:54:52 -0700436 if (tc->snd_mss < 45)
Florin Corasc8343412017-05-04 14:25:50 -0700437 {
Florin Corasdb84e572017-05-09 18:54:52 -0700438 /* Assume that at least the min default mss works */
439 tc->snd_mss = default_min_mss;
Florin Coras93992a92017-05-24 18:03:56 -0700440 tc->rcv_opts.mss = default_min_mss;
Florin Corasc8343412017-05-04 14:25:50 -0700441 }
442
443 /* We should have enough space for 40 bytes of options */
444 ASSERT (tc->snd_mss > 45);
445
446 /* If we use timestamp option, account for it */
Florin Coras93992a92017-05-24 18:03:56 -0700447 if (tcp_opts_tstamp (&tc->rcv_opts))
Florin Corasc8343412017-05-04 14:25:50 -0700448 tc->snd_mss -= TCP_OPTION_LEN_TIMESTAMP;
449}
Filip Tehlare275bed2019-03-06 00:06:56 -0800450#endif /* CLIB_MARCH_VARIANT */
Florin Corasc8343412017-05-04 14:25:50 -0700451
Florin Coras0dbd5172018-06-25 16:19:34 -0700452static void *
Dave Barach68b0fb02017-02-28 15:15:56 -0500453tcp_reuse_buffer (vlib_main_t * vm, vlib_buffer_t * b)
454{
Florin Corasb2215d62017-08-01 16:56:58 -0700455 if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
456 vlib_buffer_free_one (vm, b->next_buffer);
Florin Coras4eeeaaf2017-09-05 14:03:37 -0400457 /* Zero all flags but free list index and trace flag */
458 b->flags &= VLIB_BUFFER_NEXT_PRESENT - 1;
Florin Coras1f152cd2017-08-18 19:28:03 -0700459 b->current_data = 0;
460 b->current_length = 0;
461 b->total_length_not_including_first_buffer = 0;
462 vnet_buffer (b)->tcp.flags = 0;
Florin Corasb2215d62017-08-01 16:56:58 -0700463
Dave Barach68b0fb02017-02-28 15:15:56 -0500464 /* Leave enough space for headers */
Florin Coras1ee78302019-02-05 15:51:15 -0800465 return vlib_buffer_make_headroom (b, TRANSPORT_MAX_HDRS_LEN);
Florin Coras1f152cd2017-08-18 19:28:03 -0700466}
467
Filip Tehlare275bed2019-03-06 00:06:56 -0800468#ifndef CLIB_MARCH_VARIANT
Florin Coras0dbd5172018-06-25 16:19:34 -0700469static void *
Florin Coras1f152cd2017-08-18 19:28:03 -0700470tcp_init_buffer (vlib_main_t * vm, vlib_buffer_t * b)
471{
472 ASSERT ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
Florin Coras4eeeaaf2017-09-05 14:03:37 -0400473 b->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
Florin Coras1f152cd2017-08-18 19:28:03 -0700474 b->total_length_not_including_first_buffer = 0;
Florin Coras24793e32018-05-09 11:34:25 -0700475 b->current_data = 0;
Florin Corasd79b41e2017-03-04 05:37:52 -0800476 vnet_buffer (b)->tcp.flags = 0;
Florin Corasf988e692017-11-27 04:34:14 -0500477 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b);
Florin Coras1f152cd2017-08-18 19:28:03 -0700478 /* Leave enough space for headers */
Florin Coras1ee78302019-02-05 15:51:15 -0800479 return vlib_buffer_make_headroom (b, TRANSPORT_MAX_HDRS_LEN);
Dave Barach68b0fb02017-02-28 15:15:56 -0500480}
481
482/**
483 * Prepare ACK
484 */
Florin Corasbdf7fd62019-01-31 17:31:01 -0800485static inline void
Dave Barach68b0fb02017-02-28 15:15:56 -0500486tcp_make_ack_i (tcp_connection_t * tc, vlib_buffer_t * b, tcp_state_t state,
487 u8 flags)
488{
489 tcp_options_t _snd_opts, *snd_opts = &_snd_opts;
490 u8 tcp_opts_len, tcp_hdr_opts_len;
491 tcp_header_t *th;
492 u16 wnd;
493
494 wnd = tcp_window_to_advertise (tc, state);
495
496 /* Make and write options */
497 tcp_opts_len = tcp_make_established_options (tc, snd_opts);
498 tcp_hdr_opts_len = tcp_opts_len + sizeof (tcp_header_t);
499
500 th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, tc->snd_nxt,
501 tc->rcv_nxt, tcp_hdr_opts_len, flags, wnd);
502
503 tcp_options_write ((u8 *) (th + 1), snd_opts);
Dave Barach68b0fb02017-02-28 15:15:56 -0500504 vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
505}
506
507/**
508 * Convert buffer to ACK
509 */
Florin Corasbdf7fd62019-01-31 17:31:01 -0800510static inline void
Dave Barach68b0fb02017-02-28 15:15:56 -0500511tcp_make_ack (tcp_connection_t * tc, vlib_buffer_t * b)
512{
Dave Barach68b0fb02017-02-28 15:15:56 -0500513 tcp_make_ack_i (tc, b, TCP_STATE_ESTABLISHED, TCP_FLAG_ACK);
Florin Coras6792ec02017-03-13 03:49:51 -0700514 TCP_EVT_DBG (TCP_EVT_ACK_SENT, tc);
Florin Coras3e350af2017-03-30 02:54:28 -0700515 tc->rcv_las = tc->rcv_nxt;
Dave Barach68b0fb02017-02-28 15:15:56 -0500516}
517
518/**
519 * Convert buffer to FIN-ACK
520 */
521void
Florin Corasd79b41e2017-03-04 05:37:52 -0800522tcp_make_fin (tcp_connection_t * tc, vlib_buffer_t * b)
Dave Barach68b0fb02017-02-28 15:15:56 -0500523{
Florin Corasbdf7fd62019-01-31 17:31:01 -0800524 tcp_make_ack_i (tc, b, TCP_STATE_ESTABLISHED, TCP_FLAG_FIN | TCP_FLAG_ACK);
Florin Coras4eeeaaf2017-09-05 14:03:37 -0400525}
Dave Barach68b0fb02017-02-28 15:15:56 -0500526
Florin Coras4eeeaaf2017-09-05 14:03:37 -0400527/**
528 * Convert buffer to SYN
529 */
530void
531tcp_make_syn (tcp_connection_t * tc, vlib_buffer_t * b)
532{
533 u8 tcp_hdr_opts_len, tcp_opts_len;
534 tcp_header_t *th;
535 u16 initial_wnd;
536 tcp_options_t snd_opts;
537
538 initial_wnd = tcp_initial_window_to_advertise (tc);
539
540 /* Make and write options */
Dave Barachb7b92992018-10-17 10:38:51 -0400541 clib_memset (&snd_opts, 0, sizeof (snd_opts));
Florin Coras4eeeaaf2017-09-05 14:03:37 -0400542 tcp_opts_len = tcp_make_syn_options (&snd_opts, tc->rcv_wscale);
543 tcp_hdr_opts_len = tcp_opts_len + sizeof (tcp_header_t);
544
545 th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, tc->iss,
546 tc->rcv_nxt, tcp_hdr_opts_len, TCP_FLAG_SYN,
547 initial_wnd);
548 vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
549 tcp_options_write ((u8 *) (th + 1), &snd_opts);
Dave Barach68b0fb02017-02-28 15:15:56 -0500550}
551
552/**
553 * Convert buffer to SYN-ACK
554 */
555void
556tcp_make_synack (tcp_connection_t * tc, vlib_buffer_t * b)
557{
Dave Barach68b0fb02017-02-28 15:15:56 -0500558 tcp_options_t _snd_opts, *snd_opts = &_snd_opts;
559 u8 tcp_opts_len, tcp_hdr_opts_len;
560 tcp_header_t *th;
561 u16 initial_wnd;
Dave Barach68b0fb02017-02-28 15:15:56 -0500562
Dave Barachb7b92992018-10-17 10:38:51 -0400563 clib_memset (snd_opts, 0, sizeof (*snd_opts));
Dave Barach68b0fb02017-02-28 15:15:56 -0500564 initial_wnd = tcp_initial_window_to_advertise (tc);
Dave Barach68b0fb02017-02-28 15:15:56 -0500565 tcp_opts_len = tcp_make_synack_options (tc, snd_opts);
566 tcp_hdr_opts_len = tcp_opts_len + sizeof (tcp_header_t);
567
568 th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, tc->iss,
569 tc->rcv_nxt, tcp_hdr_opts_len,
570 TCP_FLAG_SYN | TCP_FLAG_ACK, initial_wnd);
Dave Barach68b0fb02017-02-28 15:15:56 -0500571 tcp_options_write ((u8 *) (th + 1), snd_opts);
572
573 vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
Dave Barach68b0fb02017-02-28 15:15:56 -0500574}
575
576always_inline void
Florin Corasbe72ae62018-11-01 11:23:03 -0700577tcp_enqueue_to_ip_lookup_i (tcp_worker_ctx_t * wrk, vlib_buffer_t * b, u32 bi,
Florin Coras56b39f62018-03-27 17:29:32 -0700578 u8 is_ip4, u32 fib_index, u8 flush)
Dave Barach68b0fb02017-02-28 15:15:56 -0500579{
Florin Corasbe72ae62018-11-01 11:23:03 -0700580 vlib_main_t *vm = wrk->vm;
Dave Barach68b0fb02017-02-28 15:15:56 -0500581 u32 *to_next, next_index;
582 vlib_frame_t *f;
583
Damjan Marion213b5aa2017-07-13 21:19:27 +0200584 b->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
Dave Barach68b0fb02017-02-28 15:15:56 -0500585 b->error = 0;
586
Florin Coras56b39f62018-03-27 17:29:32 -0700587 vnet_buffer (b)->sw_if_index[VLIB_TX] = fib_index;
588 vnet_buffer (b)->sw_if_index[VLIB_RX] = 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500589
590 /* Send to IP lookup */
591 next_index = is_ip4 ? ip4_lookup_node.index : ip6_lookup_node.index;
Florin Corasf988e692017-11-27 04:34:14 -0500592 tcp_trajectory_add_start (b, 1);
Dave Barach68b0fb02017-02-28 15:15:56 -0500593
Florin Corasbe72ae62018-11-01 11:23:03 -0700594 f = wrk->ip_lookup_tx_frames[!is_ip4];
Florin Coras9d063042017-09-14 03:08:00 -0400595 if (!f)
596 {
597 f = vlib_get_frame_to_node (vm, next_index);
598 ASSERT (f);
Florin Corasbe72ae62018-11-01 11:23:03 -0700599 wrk->ip_lookup_tx_frames[!is_ip4] = f;
Florin Coras9d063042017-09-14 03:08:00 -0400600 }
601
Dave Barach68b0fb02017-02-28 15:15:56 -0500602 to_next = vlib_frame_vector_args (f);
Florin Coras9d063042017-09-14 03:08:00 -0400603 to_next[f->n_vectors] = bi;
604 f->n_vectors += 1;
605 if (flush || f->n_vectors == VLIB_FRAME_SIZE)
606 {
607 vlib_put_frame_to_node (vm, next_index, f);
Florin Corasbe72ae62018-11-01 11:23:03 -0700608 wrk->ip_lookup_tx_frames[!is_ip4] = 0;
Florin Coras9d063042017-09-14 03:08:00 -0400609 }
610}
611
Florin Coras0dbd5172018-06-25 16:19:34 -0700612static void
Florin Corasbe72ae62018-11-01 11:23:03 -0700613tcp_enqueue_to_ip_lookup_now (tcp_worker_ctx_t * wrk, vlib_buffer_t * b,
614 u32 bi, u8 is_ip4, u32 fib_index)
Florin Coras9d063042017-09-14 03:08:00 -0400615{
Florin Corasbe72ae62018-11-01 11:23:03 -0700616 tcp_enqueue_to_ip_lookup_i (wrk, b, bi, is_ip4, fib_index, 1);
Florin Coras9d063042017-09-14 03:08:00 -0400617}
618
Florin Coras0dbd5172018-06-25 16:19:34 -0700619static void
Florin Corasbe72ae62018-11-01 11:23:03 -0700620tcp_enqueue_to_ip_lookup (tcp_worker_ctx_t * wrk, vlib_buffer_t * b, u32 bi,
Florin Coras56b39f62018-03-27 17:29:32 -0700621 u8 is_ip4, u32 fib_index)
Florin Coras9d063042017-09-14 03:08:00 -0400622{
Florin Corasbe72ae62018-11-01 11:23:03 -0700623 tcp_enqueue_to_ip_lookup_i (wrk, b, bi, is_ip4, fib_index, 0);
624 if (wrk->vm->thread_index == 0 && vlib_num_workers ())
625 session_flush_frames_main_thread (wrk->vm);
Dave Barach68b0fb02017-02-28 15:15:56 -0500626}
627
Florin Coras1f152cd2017-08-18 19:28:03 -0700628always_inline void
Florin Corasbe72ae62018-11-01 11:23:03 -0700629tcp_enqueue_to_output_i (tcp_worker_ctx_t * wrk, vlib_buffer_t * b, u32 bi,
Florin Coras1f152cd2017-08-18 19:28:03 -0700630 u8 is_ip4, u8 flush)
631{
Florin Coras1f152cd2017-08-18 19:28:03 -0700632 u32 *to_next, next_index;
633 vlib_frame_t *f;
634
635 b->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
636 b->error = 0;
637
638 /* Decide where to send the packet */
639 next_index = is_ip4 ? tcp4_output_node.index : tcp6_output_node.index;
Florin Corasf988e692017-11-27 04:34:14 -0500640 tcp_trajectory_add_start (b, 2);
Florin Coras1f152cd2017-08-18 19:28:03 -0700641
642 /* Get frame to v4/6 output node */
Florin Corasbe72ae62018-11-01 11:23:03 -0700643 f = wrk->tx_frames[!is_ip4];
Florin Coras1f152cd2017-08-18 19:28:03 -0700644 if (!f)
645 {
Florin Corasbe72ae62018-11-01 11:23:03 -0700646 f = vlib_get_frame_to_node (wrk->vm, next_index);
Florin Coras1f152cd2017-08-18 19:28:03 -0700647 ASSERT (f);
Florin Corasbe72ae62018-11-01 11:23:03 -0700648 wrk->tx_frames[!is_ip4] = f;
Florin Coras1f152cd2017-08-18 19:28:03 -0700649 }
650 to_next = vlib_frame_vector_args (f);
651 to_next[f->n_vectors] = bi;
652 f->n_vectors += 1;
653 if (flush || f->n_vectors == VLIB_FRAME_SIZE)
654 {
Florin Corasbe72ae62018-11-01 11:23:03 -0700655 vlib_put_frame_to_node (wrk->vm, next_index, f);
656 wrk->tx_frames[!is_ip4] = 0;
Florin Coras1f152cd2017-08-18 19:28:03 -0700657 }
658}
659
Florin Coras0dbd5172018-06-25 16:19:34 -0700660static void
Florin Corasbe72ae62018-11-01 11:23:03 -0700661tcp_enqueue_to_output (tcp_worker_ctx_t * wrk, vlib_buffer_t * b, u32 bi,
662 u8 is_ip4)
Florin Coras1f152cd2017-08-18 19:28:03 -0700663{
Florin Corasbe72ae62018-11-01 11:23:03 -0700664 tcp_enqueue_to_output_i (wrk, b, bi, is_ip4, 0);
Florin Coras1f152cd2017-08-18 19:28:03 -0700665}
666
Florin Coras0dbd5172018-06-25 16:19:34 -0700667static void
Florin Corasbe72ae62018-11-01 11:23:03 -0700668tcp_enqueue_to_output_now (tcp_worker_ctx_t * wrk, vlib_buffer_t * b, u32 bi,
Florin Coras1f152cd2017-08-18 19:28:03 -0700669 u8 is_ip4)
670{
Florin Corasbe72ae62018-11-01 11:23:03 -0700671 tcp_enqueue_to_output_i (wrk, b, bi, is_ip4, 1);
Florin Coras1f152cd2017-08-18 19:28:03 -0700672}
Filip Tehlare275bed2019-03-06 00:06:56 -0800673#endif /* CLIB_MARCH_VARIANT */
Florin Coras1f152cd2017-08-18 19:28:03 -0700674
Florin Coras0dbd5172018-06-25 16:19:34 -0700675static int
Dave Barach68b0fb02017-02-28 15:15:56 -0500676tcp_make_reset_in_place (vlib_main_t * vm, vlib_buffer_t * b0,
Florin Corasdc629cd2017-05-09 00:52:37 -0700677 tcp_state_t state, u8 thread_index, u8 is_ip4)
Dave Barach68b0fb02017-02-28 15:15:56 -0500678{
Dave Barach68b0fb02017-02-28 15:15:56 -0500679 ip4_header_t *ih4;
680 ip6_header_t *ih6;
681 tcp_header_t *th0;
Florin Corasdc629cd2017-05-09 00:52:37 -0700682 ip4_address_t src_ip40, dst_ip40;
683 ip6_address_t src_ip60, dst_ip60;
684 u16 src_port, dst_port;
Dave Barach68b0fb02017-02-28 15:15:56 -0500685 u32 tmp;
Florin Corasdc629cd2017-05-09 00:52:37 -0700686 u32 seq, ack;
687 u8 flags;
Dave Barach68b0fb02017-02-28 15:15:56 -0500688
689 /* Find IP and TCP headers */
Florin Corasdc629cd2017-05-09 00:52:37 -0700690 th0 = tcp_buffer_hdr (b0);
691
692 /* Save src and dst ip */
Dave Barach68b0fb02017-02-28 15:15:56 -0500693 if (is_ip4)
694 {
695 ih4 = vlib_buffer_get_current (b0);
Florin Corasdc629cd2017-05-09 00:52:37 -0700696 ASSERT ((ih4->ip_version_and_header_length & 0xF0) == 0x40);
697 src_ip40.as_u32 = ih4->src_address.as_u32;
698 dst_ip40.as_u32 = ih4->dst_address.as_u32;
Dave Barach68b0fb02017-02-28 15:15:56 -0500699 }
700 else
701 {
702 ih6 = vlib_buffer_get_current (b0);
Dave Barach68b0fb02017-02-28 15:15:56 -0500703 ASSERT ((ih6->ip_version_traffic_class_and_flow_label & 0xF0) == 0x60);
Dave Barach178cf492018-11-13 16:34:13 -0500704 clib_memcpy_fast (&src_ip60, &ih6->src_address, sizeof (ip6_address_t));
705 clib_memcpy_fast (&dst_ip60, &ih6->dst_address, sizeof (ip6_address_t));
Dave Barach68b0fb02017-02-28 15:15:56 -0500706 }
707
Florin Corasdc629cd2017-05-09 00:52:37 -0700708 src_port = th0->src_port;
709 dst_port = th0->dst_port;
710
711 /* Try to determine what/why we're actually resetting */
Dave Barach68b0fb02017-02-28 15:15:56 -0500712 if (state == TCP_STATE_CLOSED)
713 {
714 if (!tcp_syn (th0))
715 return -1;
716
717 tmp = clib_net_to_host_u32 (th0->seq_number);
718
719 /* Got a SYN for no listener. */
Florin Corasdc629cd2017-05-09 00:52:37 -0700720 flags = TCP_FLAG_RST | TCP_FLAG_ACK;
721 ack = clib_host_to_net_u32 (tmp + 1);
722 seq = 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500723 }
Florin Corasdc629cd2017-05-09 00:52:37 -0700724 else
Dave Barach68b0fb02017-02-28 15:15:56 -0500725 {
Florin Corasdc629cd2017-05-09 00:52:37 -0700726 flags = TCP_FLAG_RST;
727 seq = th0->ack_number;
728 ack = 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500729 }
730
Florin Corasdc629cd2017-05-09 00:52:37 -0700731 tcp_reuse_buffer (vm, b0);
Florin Corasf988e692017-11-27 04:34:14 -0500732 tcp_trajectory_add_start (b0, 4);
Florin Corasdc629cd2017-05-09 00:52:37 -0700733 th0 = vlib_buffer_push_tcp_net_order (b0, dst_port, src_port, seq, ack,
734 sizeof (tcp_header_t), flags, 0);
Dave Barach68b0fb02017-02-28 15:15:56 -0500735
Dave Barach68b0fb02017-02-28 15:15:56 -0500736 if (is_ip4)
737 {
Florin Corasdc629cd2017-05-09 00:52:37 -0700738 ih4 = vlib_buffer_push_ip4 (vm, b0, &dst_ip40, &src_ip40,
Florin Corasfdbc3822017-07-27 00:34:12 -0700739 IP_PROTOCOL_TCP, 1);
Dave Barach68b0fb02017-02-28 15:15:56 -0500740 th0->checksum = ip4_tcp_udp_compute_checksum (vm, b0, ih4);
741 }
742 else
743 {
744 int bogus = ~0;
Florin Corasdc629cd2017-05-09 00:52:37 -0700745 ih6 = vlib_buffer_push_ip6 (vm, b0, &dst_ip60, &src_ip60,
746 IP_PROTOCOL_TCP);
Dave Barach68b0fb02017-02-28 15:15:56 -0500747 th0->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b0, ih6, &bogus);
748 ASSERT (!bogus);
749 }
750
751 return 0;
752}
753
Filip Tehlare275bed2019-03-06 00:06:56 -0800754#ifndef CLIB_MARCH_VARIANT
Dave Barach68b0fb02017-02-28 15:15:56 -0500755/**
756 * Send reset without reusing existing buffer
Florin Coras1f152cd2017-08-18 19:28:03 -0700757 *
758 * It extracts connection info out of original packet
Dave Barach68b0fb02017-02-28 15:15:56 -0500759 */
760void
Florin Corasd4c49be2019-02-07 00:15:53 -0800761tcp_send_reset_w_pkt (tcp_connection_t * tc, vlib_buffer_t * pkt,
762 u32 thread_index, u8 is_ip4)
Dave Barach68b0fb02017-02-28 15:15:56 -0500763{
Florin Corasd4c49be2019-02-07 00:15:53 -0800764 tcp_worker_ctx_t *wrk = tcp_get_worker (thread_index);
Florin Corasbe72ae62018-11-01 11:23:03 -0700765 vlib_main_t *vm = wrk->vm;
Dave Barach68b0fb02017-02-28 15:15:56 -0500766 vlib_buffer_t *b;
Florin Coras56b39f62018-03-27 17:29:32 -0700767 u32 bi, sw_if_index, fib_index;
Dave Barach68b0fb02017-02-28 15:15:56 -0500768 u8 tcp_hdr_len, flags = 0;
769 tcp_header_t *th, *pkt_th;
770 u32 seq, ack;
771 ip4_header_t *ih4, *pkt_ih4;
772 ip6_header_t *ih6, *pkt_ih6;
Florin Coras56b39f62018-03-27 17:29:32 -0700773 fib_protocol_t fib_proto;
Dave Barach68b0fb02017-02-28 15:15:56 -0500774
Florin Corasbdf7fd62019-01-31 17:31:01 -0800775 if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
Florin Coras66b11312017-07-31 17:18:03 -0700776 return;
777
Dave Barach68b0fb02017-02-28 15:15:56 -0500778 b = vlib_get_buffer (vm, bi);
Florin Coras56b39f62018-03-27 17:29:32 -0700779 sw_if_index = vnet_buffer (pkt)->sw_if_index[VLIB_RX];
780 fib_proto = is_ip4 ? FIB_PROTOCOL_IP4 : FIB_PROTOCOL_IP6;
781 fib_index = fib_table_get_index_for_sw_if_index (fib_proto, sw_if_index);
Florin Coras1f152cd2017-08-18 19:28:03 -0700782 tcp_init_buffer (vm, b);
Dave Barach68b0fb02017-02-28 15:15:56 -0500783
784 /* Make and write options */
785 tcp_hdr_len = sizeof (tcp_header_t);
786
787 if (is_ip4)
788 {
789 pkt_ih4 = vlib_buffer_get_current (pkt);
790 pkt_th = ip4_next_header (pkt_ih4);
791 }
792 else
793 {
794 pkt_ih6 = vlib_buffer_get_current (pkt);
795 pkt_th = ip6_next_header (pkt_ih6);
796 }
797
798 if (tcp_ack (pkt_th))
799 {
800 flags = TCP_FLAG_RST;
801 seq = pkt_th->ack_number;
Florin Coras776f3d82018-11-02 08:23:58 -0700802 ack = (tc->state >= TCP_STATE_SYN_RCVD) ? tc->rcv_nxt : 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500803 }
804 else
805 {
806 flags = TCP_FLAG_RST | TCP_FLAG_ACK;
807 seq = 0;
808 ack = clib_host_to_net_u32 (vnet_buffer (pkt)->tcp.seq_end);
809 }
810
811 th = vlib_buffer_push_tcp_net_order (b, pkt_th->dst_port, pkt_th->src_port,
812 seq, ack, tcp_hdr_len, flags, 0);
813
814 /* Swap src and dst ip */
815 if (is_ip4)
816 {
817 ASSERT ((pkt_ih4->ip_version_and_header_length & 0xF0) == 0x40);
818 ih4 = vlib_buffer_push_ip4 (vm, b, &pkt_ih4->dst_address,
Florin Corasfdbc3822017-07-27 00:34:12 -0700819 &pkt_ih4->src_address, IP_PROTOCOL_TCP, 1);
Dave Barach68b0fb02017-02-28 15:15:56 -0500820 th->checksum = ip4_tcp_udp_compute_checksum (vm, b, ih4);
821 }
822 else
823 {
824 int bogus = ~0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500825 ASSERT ((pkt_ih6->ip_version_traffic_class_and_flow_label & 0xF0) ==
826 0x60);
rootc9d1c5b2017-08-15 12:58:31 -0400827 ih6 = vlib_buffer_push_ip6 (vm, b, &pkt_ih6->dst_address,
828 &pkt_ih6->src_address, IP_PROTOCOL_TCP);
Dave Barach68b0fb02017-02-28 15:15:56 -0500829 th->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ih6, &bogus);
830 ASSERT (!bogus);
831 }
832
Florin Corasbe72ae62018-11-01 11:23:03 -0700833 tcp_enqueue_to_ip_lookup_now (wrk, b, bi, is_ip4, fib_index);
Florin Coras6534b7a2017-07-18 05:38:03 -0400834 TCP_EVT_DBG (TCP_EVT_RST_SENT, tc);
Dave Barach68b0fb02017-02-28 15:15:56 -0500835}
836
Florin Coras1f152cd2017-08-18 19:28:03 -0700837/**
838 * Build and set reset packet for connection
839 */
840void
841tcp_send_reset (tcp_connection_t * tc)
842{
Florin Corasbe72ae62018-11-01 11:23:03 -0700843 tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
844 vlib_main_t *vm = wrk->vm;
Florin Coras1f152cd2017-08-18 19:28:03 -0700845 vlib_buffer_t *b;
846 u32 bi;
847 tcp_header_t *th;
848 u16 tcp_hdr_opts_len, advertise_wnd, opts_write_len;
849 u8 flags;
850
Florin Corasbdf7fd62019-01-31 17:31:01 -0800851 if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
Florin Coras1f152cd2017-08-18 19:28:03 -0700852 return;
853 b = vlib_get_buffer (vm, bi);
854 tcp_init_buffer (vm, b);
855
856 tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts, tc->state);
857 tcp_hdr_opts_len = tc->snd_opts_len + sizeof (tcp_header_t);
858 advertise_wnd = tcp_window_to_advertise (tc, TCP_STATE_ESTABLISHED);
859 flags = TCP_FLAG_RST;
860 th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, tc->snd_nxt,
861 tc->rcv_nxt, tcp_hdr_opts_len, flags,
862 advertise_wnd);
863 opts_write_len = tcp_options_write ((u8 *) (th + 1), &tc->snd_opts);
864 ASSERT (opts_write_len == tc->snd_opts_len);
865 vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
Florin Corasf1762d62017-09-24 19:43:08 -0400866 if (tc->c_is_ip4)
867 {
868 ip4_header_t *ih4;
869 ih4 = vlib_buffer_push_ip4 (vm, b, &tc->c_lcl_ip.ip4,
870 &tc->c_rmt_ip.ip4, IP_PROTOCOL_TCP, 0);
871 th->checksum = ip4_tcp_udp_compute_checksum (vm, b, ih4);
872 }
873 else
874 {
875 int bogus = ~0;
876 ip6_header_t *ih6;
877 ih6 = vlib_buffer_push_ip6 (vm, b, &tc->c_lcl_ip.ip6,
878 &tc->c_rmt_ip.ip6, IP_PROTOCOL_TCP);
879 th->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ih6, &bogus);
880 ASSERT (!bogus);
881 }
Florin Corasbe72ae62018-11-01 11:23:03 -0700882 tcp_enqueue_to_ip_lookup_now (wrk, b, bi, tc->c_is_ip4, tc->c_fib_index);
Florin Corasf1762d62017-09-24 19:43:08 -0400883 TCP_EVT_DBG (TCP_EVT_RST_SENT, tc);
Florin Coras1f152cd2017-08-18 19:28:03 -0700884}
885
Florin Coras0dbd5172018-06-25 16:19:34 -0700886static void
Florin Corasbe72ae62018-11-01 11:23:03 -0700887tcp_push_ip_hdr (tcp_worker_ctx_t * wrk, tcp_connection_t * tc,
888 vlib_buffer_t * b)
Dave Barach68b0fb02017-02-28 15:15:56 -0500889{
890 tcp_header_t *th = vlib_buffer_get_current (b);
Florin Corasbe72ae62018-11-01 11:23:03 -0700891 vlib_main_t *vm = wrk->vm;
Dave Barach68b0fb02017-02-28 15:15:56 -0500892 if (tc->c_is_ip4)
893 {
894 ip4_header_t *ih;
Dave Barach2c25a622017-06-26 11:35:07 -0400895 ih = vlib_buffer_push_ip4 (vm, b, &tc->c_lcl_ip4,
Florin Corasfdbc3822017-07-27 00:34:12 -0700896 &tc->c_rmt_ip4, IP_PROTOCOL_TCP, 1);
Dave Barach2c25a622017-06-26 11:35:07 -0400897 th->checksum = ip4_tcp_udp_compute_checksum (vm, b, ih);
Dave Barach68b0fb02017-02-28 15:15:56 -0500898 }
899 else
900 {
901 ip6_header_t *ih;
902 int bogus = ~0;
903
Dave Barach2c25a622017-06-26 11:35:07 -0400904 ih = vlib_buffer_push_ip6 (vm, b, &tc->c_lcl_ip6,
Dave Barach68b0fb02017-02-28 15:15:56 -0500905 &tc->c_rmt_ip6, IP_PROTOCOL_TCP);
Dave Barach2c25a622017-06-26 11:35:07 -0400906 th->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ih, &bogus);
Dave Barach68b0fb02017-02-28 15:15:56 -0500907 ASSERT (!bogus);
908 }
909}
910
911/**
912 * Send SYN
913 *
914 * Builds a SYN packet for a half-open connection and sends it to ipx_lookup.
915 * The packet is not forwarded through tcpx_output to avoid doing lookups
916 * in the half_open pool.
917 */
918void
919tcp_send_syn (tcp_connection_t * tc)
920{
Florin Corasbe72ae62018-11-01 11:23:03 -0700921 tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
922 vlib_main_t *vm = wrk->vm;
Dave Barach68b0fb02017-02-28 15:15:56 -0500923 vlib_buffer_t *b;
924 u32 bi;
Dave Barach68b0fb02017-02-28 15:15:56 -0500925
Florin Corasf988e692017-11-27 04:34:14 -0500926 /*
927 * Setup retransmit and establish timers before requesting buffer
928 * such that we can return if we've ran out.
929 */
Florin Coras85a3ddd2018-12-24 16:54:34 -0800930 tcp_timer_set (tc, TCP_TIMER_ESTABLISH_AO, TCP_ESTABLISH_TIME);
Florin Corasf988e692017-11-27 04:34:14 -0500931 tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN,
932 tc->rto * TCP_TO_TIMER_TICK);
933
Florin Corasbdf7fd62019-01-31 17:31:01 -0800934 if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
Florin Coras222e1f412019-02-16 20:47:32 -0800935 {
936 tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN, 1);
937 return;
938 }
Florin Coras66b11312017-07-31 17:18:03 -0700939
Dave Barach68b0fb02017-02-28 15:15:56 -0500940 b = vlib_get_buffer (vm, bi);
Florin Coras1f152cd2017-08-18 19:28:03 -0700941 tcp_init_buffer (vm, b);
Florin Coras4eeeaaf2017-09-05 14:03:37 -0400942 tcp_make_syn (tc, b);
Dave Barach68b0fb02017-02-28 15:15:56 -0500943
944 /* Measure RTT with this */
Florin Corasefefc6b2018-11-07 12:49:19 -0800945 tc->rtt_ts = tcp_time_now_us (vlib_num_workers ()? 1 : 0);
Dave Barach68b0fb02017-02-28 15:15:56 -0500946 tc->rtt_seq = tc->snd_nxt;
Dave Barach68b0fb02017-02-28 15:15:56 -0500947 tc->rto_boff = 0;
948
Florin Corasbe72ae62018-11-01 11:23:03 -0700949 tcp_push_ip_hdr (wrk, tc, b);
950 tcp_enqueue_to_ip_lookup (wrk, b, bi, tc->c_is_ip4, tc->c_fib_index);
Florin Coras6534b7a2017-07-18 05:38:03 -0400951 TCP_EVT_DBG (TCP_EVT_SYN_SENT, tc);
Dave Barach68b0fb02017-02-28 15:15:56 -0500952}
953
Florin Coras7ac053b2018-11-05 15:57:21 -0800954void
955tcp_send_synack (tcp_connection_t * tc)
956{
957 tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
958 vlib_main_t *vm = wrk->vm;
959 vlib_buffer_t *b;
960 u32 bi;
961
Florin Coras222e1f412019-02-16 20:47:32 -0800962 tcp_retransmit_timer_force_update (tc);
963
Florin Corasbdf7fd62019-01-31 17:31:01 -0800964 if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
Florin Coras222e1f412019-02-16 20:47:32 -0800965 {
966 tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1);
967 return;
968 }
Florin Coras7ac053b2018-11-05 15:57:21 -0800969
Florin Corasdf084782018-12-06 17:41:10 -0800970 tc->rtt_ts = tcp_time_now_us (tc->c_thread_index);
Florin Coras7ac053b2018-11-05 15:57:21 -0800971 b = vlib_get_buffer (vm, bi);
Florin Corasbdf7fd62019-01-31 17:31:01 -0800972 tcp_init_buffer (vm, b);
Florin Coras7ac053b2018-11-05 15:57:21 -0800973 tcp_make_synack (tc, b);
974 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Florin Coras222e1f412019-02-16 20:47:32 -0800975 TCP_EVT_DBG (TCP_EVT_SYNACK_SENT, tc);
Florin Coras7ac053b2018-11-05 15:57:21 -0800976}
977
Florin Coras66b11312017-07-31 17:18:03 -0700978/**
979 * Flush tx frame populated by retransmits and timer pops
980 */
981void
Florin Corasbe72ae62018-11-01 11:23:03 -0700982tcp_flush_frame_to_output (tcp_worker_ctx_t * wrk, u8 is_ip4)
Florin Coras66b11312017-07-31 17:18:03 -0700983{
Florin Corasbe72ae62018-11-01 11:23:03 -0700984 if (wrk->tx_frames[!is_ip4])
Florin Coras66b11312017-07-31 17:18:03 -0700985 {
986 u32 next_index;
987 next_index = is_ip4 ? tcp4_output_node.index : tcp6_output_node.index;
Florin Corasbe72ae62018-11-01 11:23:03 -0700988 vlib_put_frame_to_node (wrk->vm, next_index, wrk->tx_frames[!is_ip4]);
989 wrk->tx_frames[!is_ip4] = 0;
Florin Coras66b11312017-07-31 17:18:03 -0700990 }
991}
992
993/**
Florin Coras9d063042017-09-14 03:08:00 -0400994 * Flush ip lookup tx frames populated by timer pops
995 */
Florin Coras0dbd5172018-06-25 16:19:34 -0700996static void
Florin Corasbe72ae62018-11-01 11:23:03 -0700997tcp_flush_frame_to_ip_lookup (tcp_worker_ctx_t * wrk, u8 is_ip4)
Florin Coras9d063042017-09-14 03:08:00 -0400998{
Florin Corasbe72ae62018-11-01 11:23:03 -0700999 if (wrk->ip_lookup_tx_frames[!is_ip4])
Florin Coras9d063042017-09-14 03:08:00 -04001000 {
1001 u32 next_index;
1002 next_index = is_ip4 ? ip4_lookup_node.index : ip6_lookup_node.index;
Florin Corasbe72ae62018-11-01 11:23:03 -07001003 vlib_put_frame_to_node (wrk->vm, next_index,
1004 wrk->ip_lookup_tx_frames[!is_ip4]);
1005 wrk->ip_lookup_tx_frames[!is_ip4] = 0;
Florin Coras9d063042017-09-14 03:08:00 -04001006 }
1007}
1008
1009/**
1010 * Flush v4 and v6 tcp and ip-lookup tx frames for thread index
Florin Coras66b11312017-07-31 17:18:03 -07001011 */
1012void
Florin Corasbe72ae62018-11-01 11:23:03 -07001013tcp_flush_frames_to_output (tcp_worker_ctx_t * wrk)
Florin Coras66b11312017-07-31 17:18:03 -07001014{
Florin Corasbe72ae62018-11-01 11:23:03 -07001015 tcp_flush_frame_to_output (wrk, 1);
1016 tcp_flush_frame_to_output (wrk, 0);
1017 tcp_flush_frame_to_ip_lookup (wrk, 1);
1018 tcp_flush_frame_to_ip_lookup (wrk, 0);
Dave Barach68b0fb02017-02-28 15:15:56 -05001019}
1020
1021/**
1022 * Send FIN
1023 */
1024void
1025tcp_send_fin (tcp_connection_t * tc)
1026{
Florin Corasbe72ae62018-11-01 11:23:03 -07001027 tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
1028 vlib_main_t *vm = wrk->vm;
Florin Coras9d063042017-09-14 03:08:00 -04001029 vlib_buffer_t *b;
1030 u32 bi;
1031 u8 fin_snt = 0;
1032
Florin Corasc977e7c2018-10-16 20:30:31 -07001033 fin_snt = tc->flags & TCP_CONN_FINSNT;
1034 if (fin_snt)
Florin Coras47596832019-03-12 18:58:54 -07001035 tc->snd_nxt -= 1;
Florin Corasc977e7c2018-10-16 20:30:31 -07001036
Florin Corasbdf7fd62019-01-31 17:31:01 -08001037 if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
Florin Coraseb97e5f2018-10-15 21:35:42 -07001038 {
1039 /* Out of buffers so program fin retransmit ASAP */
1040 tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1);
Florin Coras85a3ddd2018-12-24 16:54:34 -08001041 if (fin_snt)
Florin Coras47596832019-03-12 18:58:54 -07001042 tc->snd_nxt += 1;
Florin Coras222e1f412019-02-16 20:47:32 -08001043 else
1044 /* Make sure retransmit retries a fin not data */
1045 tc->flags |= TCP_CONN_FINSNT;
Florin Coras85a3ddd2018-12-24 16:54:34 -08001046 return;
Florin Coraseb97e5f2018-10-15 21:35:42 -07001047 }
1048
Florin Corasc977e7c2018-10-16 20:30:31 -07001049 tcp_retransmit_timer_force_update (tc);
Dave Barach68b0fb02017-02-28 15:15:56 -05001050 b = vlib_get_buffer (vm, bi);
Florin Coras24793e32018-05-09 11:34:25 -07001051 tcp_init_buffer (vm, b);
Florin Corasd79b41e2017-03-04 05:37:52 -08001052 tcp_make_fin (tc, b);
Florin Corasbe72ae62018-11-01 11:23:03 -07001053 tcp_enqueue_to_output_now (wrk, b, bi, tc->c_is_ip4);
Florin Corasc977e7c2018-10-16 20:30:31 -07001054 TCP_EVT_DBG (TCP_EVT_FIN_SENT, tc);
Florin Coras47596832019-03-12 18:58:54 -07001055 /* Account for the FIN */
1056 tc->snd_nxt += 1;
Florin Coras9d063042017-09-14 03:08:00 -04001057 if (!fin_snt)
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001058 {
1059 tc->flags |= TCP_CONN_FINSNT;
1060 tc->flags &= ~TCP_CONN_FINPNDG;
Florin Coras47596832019-03-12 18:58:54 -07001061 tc->snd_una_max = seq_max (tc->snd_una_max, tc->snd_nxt);
Florin Corasa096f2d2017-09-28 23:49:42 -04001062 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001063}
1064
Dave Barach68b0fb02017-02-28 15:15:56 -05001065/**
Florin Corasfd247442019-03-12 16:56:26 -07001066 * Push TCP header and update connection variables. Should only be called
1067 * for segments with data, not for 'control' packets.
Dave Barach68b0fb02017-02-28 15:15:56 -05001068 */
Florin Coras0dbd5172018-06-25 16:19:34 -07001069always_inline void
Florin Coras47596832019-03-12 18:58:54 -07001070tcp_push_hdr_i (tcp_connection_t * tc, vlib_buffer_t * b, u32 snd_nxt,
1071 u8 compute_opts, u8 maybe_burst, u8 update_snd_nxt)
Dave Barach68b0fb02017-02-28 15:15:56 -05001072{
Florin Corasfd247442019-03-12 16:56:26 -07001073 u8 tcp_hdr_opts_len, flags = TCP_FLAG_ACK;
Dave Barach68b0fb02017-02-28 15:15:56 -05001074 u32 advertise_wnd, data_len;
Florin Corasb26743d2018-06-26 09:31:04 -07001075 tcp_main_t *tm = &tcp_main;
Dave Barach68b0fb02017-02-28 15:15:56 -05001076 tcp_header_t *th;
1077
Florin Corasb26743d2018-06-26 09:31:04 -07001078 data_len = b->current_length;
1079 if (PREDICT_FALSE (b->flags & VLIB_BUFFER_NEXT_PRESENT))
1080 data_len += b->total_length_not_including_first_buffer;
1081
Dave Barach68b0fb02017-02-28 15:15:56 -05001082 vnet_buffer (b)->tcp.flags = 0;
Florin Corasb26743d2018-06-26 09:31:04 -07001083 vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
Dave Barach68b0fb02017-02-28 15:15:56 -05001084
Florin Corasc8343412017-05-04 14:25:50 -07001085 if (compute_opts)
1086 tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts, tc->state);
1087
Florin Corasc8343412017-05-04 14:25:50 -07001088 tcp_hdr_opts_len = tc->snd_opts_len + sizeof (tcp_header_t);
Florin Corasb26743d2018-06-26 09:31:04 -07001089
1090 if (maybe_burst)
1091 advertise_wnd = tc->rcv_wnd >> tc->rcv_wscale;
1092 else
Florin Coras47596832019-03-12 18:58:54 -07001093 advertise_wnd = tcp_window_to_advertise (tc, TCP_STATE_ESTABLISHED);
Florin Corasb26743d2018-06-26 09:31:04 -07001094
Florin Coras42ceddb2018-12-12 10:56:01 -08001095 if (PREDICT_FALSE (tc->flags & TCP_CONN_PSH_PENDING))
1096 {
Florin Coras47596832019-03-12 18:58:54 -07001097 if (seq_geq (tc->psh_seq, snd_nxt)
1098 && seq_lt (tc->psh_seq, snd_nxt + data_len))
Florin Coras42ceddb2018-12-12 10:56:01 -08001099 flags |= TCP_FLAG_PSH;
1100 }
Florin Coras47596832019-03-12 18:58:54 -07001101 th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, snd_nxt,
Dave Barach68b0fb02017-02-28 15:15:56 -05001102 tc->rcv_nxt, tcp_hdr_opts_len, flags,
1103 advertise_wnd);
Dave Barach68b0fb02017-02-28 15:15:56 -05001104
Florin Corasb26743d2018-06-26 09:31:04 -07001105 if (maybe_burst)
1106 {
Dave Barach178cf492018-11-13 16:34:13 -05001107 clib_memcpy_fast ((u8 *) (th + 1),
1108 tm->wrk_ctx[tc->c_thread_index].cached_opts,
1109 tc->snd_opts_len);
Florin Corasb26743d2018-06-26 09:31:04 -07001110 }
1111 else
1112 {
1113 u8 len = tcp_options_write ((u8 *) (th + 1), &tc->snd_opts);
1114 ASSERT (len == tc->snd_opts_len);
1115 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001116
Florin Coras93992a92017-05-24 18:03:56 -07001117 /*
1118 * Update connection variables
1119 */
1120
Florin Coras47596832019-03-12 18:58:54 -07001121 if (update_snd_nxt)
1122 tc->snd_nxt += data_len;
Florin Corasc28764f2017-04-26 00:08:42 -07001123 tc->rcv_las = tc->rcv_nxt;
Florin Coras3e350af2017-03-30 02:54:28 -07001124
Florin Corase69f4952017-03-07 10:06:24 -08001125 TCP_EVT_DBG (TCP_EVT_PKTIZE, tc);
Dave Barach68b0fb02017-02-28 15:15:56 -05001126}
1127
Florin Coras0dbd5172018-06-25 16:19:34 -07001128u32
Florin Coras14ed6df2019-03-06 21:13:42 -08001129tcp_session_push_header (transport_connection_t * tconn, vlib_buffer_t * b)
Florin Coras0dbd5172018-06-25 16:19:34 -07001130{
Florin Coras14ed6df2019-03-06 21:13:42 -08001131 tcp_connection_t *tc = (tcp_connection_t *) tconn;
Florin Coras47596832019-03-12 18:58:54 -07001132 tcp_push_hdr_i (tc, b, tc->snd_nxt, /* compute opts */ 0, /* burst */ 1,
1133 /* update_snd_nxt */ 1);
1134 tc->snd_una_max = seq_max (tc->snd_nxt, tc->snd_una_max);
Florin Coras0dbd5172018-06-25 16:19:34 -07001135 tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una);
1136 /* If not tracking an ACK, start tracking */
1137 if (tc->rtt_ts == 0 && !tcp_in_cong_recovery (tc))
1138 {
Florin Corasd67f1122018-05-21 17:47:40 -07001139 tc->rtt_ts = tcp_time_now_us (tc->c_thread_index);
Florin Coras0dbd5172018-06-25 16:19:34 -07001140 tc->rtt_seq = tc->snd_nxt;
1141 }
1142 if (PREDICT_FALSE (!tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT)))
1143 {
1144 tcp_retransmit_timer_set (tc);
1145 tc->rto_boff = 0;
1146 }
1147 tcp_trajectory_add_start (b, 3);
1148 return 0;
1149}
1150
Dave Barach68b0fb02017-02-28 15:15:56 -05001151void
Florin Coras6792ec02017-03-13 03:49:51 -07001152tcp_send_ack (tcp_connection_t * tc)
Dave Barach68b0fb02017-02-28 15:15:56 -05001153{
Florin Corasbe72ae62018-11-01 11:23:03 -07001154 tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
1155 vlib_main_t *vm = wrk->vm;
Dave Barach68b0fb02017-02-28 15:15:56 -05001156 vlib_buffer_t *b;
1157 u32 bi;
1158
Florin Corasbdf7fd62019-01-31 17:31:01 -08001159 if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
Florin Coras222e1f412019-02-16 20:47:32 -08001160 {
1161 tcp_update_rcv_wnd (tc);
1162 return;
1163 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001164 b = vlib_get_buffer (vm, bi);
Florin Coras24793e32018-05-09 11:34:25 -07001165 tcp_init_buffer (vm, b);
Dave Barach68b0fb02017-02-28 15:15:56 -05001166 tcp_make_ack (tc, b);
Florin Corasbe72ae62018-11-01 11:23:03 -07001167 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Dave Barach68b0fb02017-02-28 15:15:56 -05001168}
1169
Florin Coras7ac053b2018-11-05 15:57:21 -08001170void
1171tcp_program_ack (tcp_worker_ctx_t * wrk, tcp_connection_t * tc)
1172{
1173 if (!(tc->flags & TCP_CONN_SNDACK))
1174 {
1175 vec_add1 (wrk->pending_acks, tc->c_c_index);
1176 tc->flags |= TCP_CONN_SNDACK;
1177 }
1178}
1179
1180void
1181tcp_program_dupack (tcp_worker_ctx_t * wrk, tcp_connection_t * tc)
1182{
1183 if (!(tc->flags & TCP_CONN_SNDACK))
1184 {
1185 vec_add1 (wrk->pending_acks, tc->c_c_index);
1186 tc->flags |= TCP_CONN_SNDACK;
1187 }
1188 if (tc->pending_dupacks < 255)
1189 tc->pending_dupacks += 1;
1190}
1191
1192void
1193tcp_send_acks (tcp_worker_ctx_t * wrk)
1194{
1195 u32 thread_index, *pending_acks;
1196 tcp_connection_t *tc;
1197 int i, j, n_acks;
1198
1199 if (!vec_len (wrk->pending_acks))
1200 return;
1201
1202 thread_index = wrk->vm->thread_index;
1203 pending_acks = wrk->pending_acks;
1204 for (i = 0; i < vec_len (pending_acks); i++)
1205 {
1206 tc = tcp_connection_get (pending_acks[i], thread_index);
1207 tc->flags &= ~TCP_CONN_SNDACK;
Florin Corase5b17912019-02-21 16:46:24 -08001208 if (!tc->pending_dupacks)
1209 {
1210 tcp_send_ack (tc);
1211 continue;
1212 }
1213
Florin Coras7ac053b2018-11-05 15:57:21 -08001214 /* If we're supposed to send dupacks but have no ooo data
1215 * send only one ack */
Florin Corase5b17912019-02-21 16:46:24 -08001216 if (!vec_len (tc->snd_sacks))
1217 {
1218 tcp_send_ack (tc);
1219 continue;
1220 }
1221
1222 /* Start with first sack block */
1223 tc->snd_sack_pos = 0;
1224
1225 /* Generate enough dupacks to cover all sack blocks. Do not generate
1226 * more sacks than the number of packets received. But do generate at
1227 * least 3, i.e., the number needed to signal congestion, if needed. */
1228 n_acks = vec_len (tc->snd_sacks) / TCP_OPTS_MAX_SACK_BLOCKS;
1229 n_acks = clib_min (n_acks, tc->pending_dupacks);
1230 n_acks = clib_max (n_acks, clib_min (tc->pending_dupacks, 3));
Florin Coras7ac053b2018-11-05 15:57:21 -08001231 for (j = 0; j < n_acks; j++)
1232 tcp_send_ack (tc);
Florin Corase5b17912019-02-21 16:46:24 -08001233
Florin Coras7ac053b2018-11-05 15:57:21 -08001234 tc->pending_dupacks = 0;
Florin Corase5b17912019-02-21 16:46:24 -08001235 tc->snd_sack_pos = 0;
Florin Coras7ac053b2018-11-05 15:57:21 -08001236 }
1237 _vec_len (wrk->pending_acks) = 0;
1238}
1239
Florin Corasb2215d62017-08-01 16:56:58 -07001240/**
1241 * Delayed ack timer handler
1242 *
1243 * Sends delayed ACK when timer expires
1244 */
Florin Coras6792ec02017-03-13 03:49:51 -07001245void
1246tcp_timer_delack_handler (u32 index)
1247{
Damjan Marion586afd72017-04-05 19:18:20 +02001248 u32 thread_index = vlib_get_thread_index ();
Florin Coras6792ec02017-03-13 03:49:51 -07001249 tcp_connection_t *tc;
1250
1251 tc = tcp_connection_get (index, thread_index);
1252 tc->timers[TCP_TIMER_DELACK] = TCP_TIMER_HANDLE_INVALID;
Florin Coras6792ec02017-03-13 03:49:51 -07001253 tcp_send_ack (tc);
1254}
1255
Florin Corasb2215d62017-08-01 16:56:58 -07001256/**
Florin Coras36ee9f12018-11-02 12:52:10 -07001257 * Allocate a new buffer and build a new tcp segment
Dave Barach68b0fb02017-02-28 15:15:56 -05001258 *
Florin Coras36ee9f12018-11-02 12:52:10 -07001259 * @param wrk tcp worker
1260 * @param tc connection for which the segment will be allocated
1261 * @param offset offset of the first byte in the tx fifo
1262 * @param max_deq_byte segment size
1263 * @param[out] b pointer to buffer allocated
1264 *
1265 * @return the number of bytes in the segment or 0 if buffer cannot be
1266 * allocated or no data available
Florin Coras93992a92017-05-24 18:03:56 -07001267 */
Florin Coras36ee9f12018-11-02 12:52:10 -07001268static int
1269tcp_prepare_segment (tcp_worker_ctx_t * wrk, tcp_connection_t * tc,
1270 u32 offset, u32 max_deq_bytes, vlib_buffer_t ** b)
Dave Barach68b0fb02017-02-28 15:15:56 -05001271{
Florin Corasbe72ae62018-11-01 11:23:03 -07001272 u32 bytes_per_buffer = vnet_get_tcp_main ()->bytes_per_buffer;
1273 vlib_main_t *vm = wrk->vm;
Florin Corasbdf7fd62019-01-31 17:31:01 -08001274 u32 bi, seg_size;
Florin Coras93992a92017-05-24 18:03:56 -07001275 int n_bytes = 0;
Florin Corase87216f2017-08-17 16:59:22 -07001276 u8 *data;
Dave Barach68b0fb02017-02-28 15:15:56 -05001277
Florin Coras1ee78302019-02-05 15:51:15 -08001278 seg_size = max_deq_bytes + TRANSPORT_MAX_HDRS_LEN;
Florin Coras1f152cd2017-08-18 19:28:03 -07001279
Florin Corasb2215d62017-08-01 16:56:58 -07001280 /*
1281 * Prepare options
1282 */
Florin Corasc8343412017-05-04 14:25:50 -07001283 tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts, tc->state);
1284
Florin Corasb2215d62017-08-01 16:56:58 -07001285 /*
1286 * Allocate and fill in buffer(s)
1287 */
Dave Barach68b0fb02017-02-28 15:15:56 -05001288
Florin Corasb2215d62017-08-01 16:56:58 -07001289 /* Easy case, buffer size greater than mss */
Florin Corasbe72ae62018-11-01 11:23:03 -07001290 if (PREDICT_TRUE (seg_size <= bytes_per_buffer))
Florin Corasb2215d62017-08-01 16:56:58 -07001291 {
Florin Corasbdf7fd62019-01-31 17:31:01 -08001292 if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
Florin Coras24793e32018-05-09 11:34:25 -07001293 return 0;
1294 *b = vlib_get_buffer (vm, bi);
1295 data = tcp_init_buffer (vm, *b);
Florin Coras31c99552019-03-01 13:00:58 -08001296 n_bytes = session_tx_fifo_peek_bytes (&tc->connection, data, offset,
1297 max_deq_bytes);
Florin Corasb2215d62017-08-01 16:56:58 -07001298 ASSERT (n_bytes == max_deq_bytes);
1299 b[0]->current_length = n_bytes;
Florin Coras47596832019-03-12 18:58:54 -07001300 tcp_push_hdr_i (tc, *b, tc->snd_una + offset, /* compute opts */ 0,
1301 /* burst */ 0, /* update_snd_nxt */ 0);
Florin Corasb2215d62017-08-01 16:56:58 -07001302 }
1303 /* Split mss into multiple buffers */
1304 else
1305 {
Florin Corasbdf7fd62019-01-31 17:31:01 -08001306 u32 chain_bi = ~0, n_bufs_per_seg, n_bufs;
1307 u16 n_peeked, len_to_deq;
Florin Corasb2215d62017-08-01 16:56:58 -07001308 vlib_buffer_t *chain_b, *prev_b;
Florin Corasb2215d62017-08-01 16:56:58 -07001309 int i;
1310
Florin Corasb2215d62017-08-01 16:56:58 -07001311 /* Make sure we have enough buffers */
Florin Corasbe72ae62018-11-01 11:23:03 -07001312 n_bufs_per_seg = ceil ((double) seg_size / bytes_per_buffer);
Florin Corasbdf7fd62019-01-31 17:31:01 -08001313 vec_validate_aligned (wrk->tx_buffers, n_bufs_per_seg - 1,
1314 CLIB_CACHE_LINE_BYTES);
1315 n_bufs = vlib_buffer_alloc (vm, wrk->tx_buffers, n_bufs_per_seg);
1316 if (PREDICT_FALSE (n_bufs != n_bufs_per_seg))
Florin Corasb2215d62017-08-01 16:56:58 -07001317 {
Florin Corasbdf7fd62019-01-31 17:31:01 -08001318 if (n_bufs)
1319 vlib_buffer_free (vm, wrk->tx_buffers, n_bufs);
1320 return 0;
Florin Corasb2215d62017-08-01 16:56:58 -07001321 }
1322
Florin Corasbdf7fd62019-01-31 17:31:01 -08001323 *b = vlib_get_buffer (vm, wrk->tx_buffers[--n_bufs]);
Florin Coras24793e32018-05-09 11:34:25 -07001324 data = tcp_init_buffer (vm, *b);
Florin Coras31c99552019-03-01 13:00:58 -08001325 n_bytes = session_tx_fifo_peek_bytes (&tc->connection, data, offset,
1326 bytes_per_buffer -
1327 TRANSPORT_MAX_HDRS_LEN);
Florin Corasb2215d62017-08-01 16:56:58 -07001328 b[0]->current_length = n_bytes;
1329 b[0]->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
1330 b[0]->total_length_not_including_first_buffer = 0;
Florin Corasb2215d62017-08-01 16:56:58 -07001331 max_deq_bytes -= n_bytes;
1332
1333 chain_b = *b;
1334 for (i = 1; i < n_bufs_per_seg; i++)
1335 {
1336 prev_b = chain_b;
Florin Corasbe72ae62018-11-01 11:23:03 -07001337 len_to_deq = clib_min (max_deq_bytes, bytes_per_buffer);
Florin Corasbdf7fd62019-01-31 17:31:01 -08001338 chain_bi = wrk->tx_buffers[--n_bufs];
Florin Corasb2215d62017-08-01 16:56:58 -07001339 chain_b = vlib_get_buffer (vm, chain_bi);
1340 chain_b->current_data = 0;
Florin Corase87216f2017-08-17 16:59:22 -07001341 data = vlib_buffer_get_current (chain_b);
Florin Coras31c99552019-03-01 13:00:58 -08001342 n_peeked = session_tx_fifo_peek_bytes (&tc->connection, data,
1343 offset + n_bytes,
1344 len_to_deq);
Florin Corasb2215d62017-08-01 16:56:58 -07001345 ASSERT (n_peeked == len_to_deq);
Florin Coras1f152cd2017-08-18 19:28:03 -07001346 n_bytes += n_peeked;
Florin Corasb2215d62017-08-01 16:56:58 -07001347 chain_b->current_length = n_peeked;
Florin Coras1f152cd2017-08-18 19:28:03 -07001348 chain_b->next_buffer = 0;
Florin Corasb2215d62017-08-01 16:56:58 -07001349
1350 /* update previous buffer */
1351 prev_b->next_buffer = chain_bi;
1352 prev_b->flags |= VLIB_BUFFER_NEXT_PRESENT;
1353
Florin Corasb2215d62017-08-01 16:56:58 -07001354 max_deq_bytes -= n_peeked;
Florin Coras1f152cd2017-08-18 19:28:03 -07001355 b[0]->total_length_not_including_first_buffer += n_peeked;
Florin Corasb2215d62017-08-01 16:56:58 -07001356 }
Florin Coras1f152cd2017-08-18 19:28:03 -07001357
Florin Coras47596832019-03-12 18:58:54 -07001358 tcp_push_hdr_i (tc, *b, tc->snd_una + offset, /* compute opts */ 0,
1359 /* burst */ 0, /* update_snd_nxt */ 0);
Florin Corasbdf7fd62019-01-31 17:31:01 -08001360
1361 if (PREDICT_FALSE (n_bufs))
1362 {
1363 clib_warning ("not all buffers consumed");
1364 vlib_buffer_free (vm, wrk->tx_buffers, n_bufs);
1365 }
Florin Corasb2215d62017-08-01 16:56:58 -07001366 }
1367
Florin Coras93992a92017-05-24 18:03:56 -07001368 ASSERT (n_bytes > 0);
Florin Corasbe72ae62018-11-01 11:23:03 -07001369 ASSERT (((*b)->current_data + (*b)->current_length) <= bytes_per_buffer);
Florin Corasbb292f42017-05-19 09:49:19 -07001370
Florin Coras36ee9f12018-11-02 12:52:10 -07001371 return n_bytes;
1372}
1373
1374/**
1375 * Build a retransmit segment
1376 *
1377 * @return the number of bytes in the segment or 0 if there's nothing to
1378 * retransmit
1379 */
1380static u32
1381tcp_prepare_retransmit_segment (tcp_worker_ctx_t * wrk,
1382 tcp_connection_t * tc, u32 offset,
1383 u32 max_deq_bytes, vlib_buffer_t ** b)
1384{
1385 u32 start, available_bytes;
1386 int n_bytes = 0;
1387
1388 ASSERT (tc->state >= TCP_STATE_ESTABLISHED);
1389 ASSERT (max_deq_bytes != 0);
1390
1391 /*
1392 * Make sure we can retransmit something
1393 */
Florin Coras31c99552019-03-01 13:00:58 -08001394 available_bytes = transport_max_tx_dequeue (&tc->connection);
Florin Coras36ee9f12018-11-02 12:52:10 -07001395 ASSERT (available_bytes >= offset);
1396 available_bytes -= offset;
1397 if (!available_bytes)
1398 return 0;
1399
1400 max_deq_bytes = clib_min (tc->snd_mss, max_deq_bytes);
1401 max_deq_bytes = clib_min (available_bytes, max_deq_bytes);
1402
1403 /* Start is beyond snd_congestion */
1404 start = tc->snd_una + offset;
1405 if (seq_geq (start, tc->snd_congestion))
1406 goto done;
1407
1408 /* Don't overshoot snd_congestion */
1409 if (seq_gt (start + max_deq_bytes, tc->snd_congestion))
1410 {
1411 max_deq_bytes = tc->snd_congestion - start;
1412 if (max_deq_bytes == 0)
1413 goto done;
1414 }
1415
1416 n_bytes = tcp_prepare_segment (wrk, tc, offset, max_deq_bytes, b);
1417 if (!n_bytes)
1418 return 0;
1419
Florin Coras93992a92017-05-24 18:03:56 -07001420 if (tcp_in_fastrecovery (tc))
1421 tc->snd_rxt_bytes += n_bytes;
Dave Barach68b0fb02017-02-28 15:15:56 -05001422
Florin Coras6792ec02017-03-13 03:49:51 -07001423done:
1424 TCP_EVT_DBG (TCP_EVT_CC_RTX, tc, offset, n_bytes);
Dave Barach68b0fb02017-02-28 15:15:56 -05001425 return n_bytes;
1426}
1427
Florin Coras6792ec02017-03-13 03:49:51 -07001428/**
1429 * Reset congestion control, switch cwnd to loss window and try again.
1430 */
1431static void
Florin Corasca1c8f32018-05-23 21:01:30 -07001432tcp_rxt_timeout_cc (tcp_connection_t * tc)
Florin Coras6792ec02017-03-13 03:49:51 -07001433{
Florin Corasca1c8f32018-05-23 21:01:30 -07001434 TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 6);
Florin Coras93992a92017-05-24 18:03:56 -07001435 tc->prev_ssthresh = tc->ssthresh;
1436 tc->prev_cwnd = tc->cwnd;
1437
Florin Coras6792ec02017-03-13 03:49:51 -07001438 /* Cleanly recover cc (also clears up fast retransmit) */
1439 if (tcp_in_fastrecovery (tc))
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001440 {
1441 /* TODO be less aggressive about this */
1442 scoreboard_clear (&tc->sack_sb);
1443 tcp_cc_fastrecovery_exit (tc);
1444 }
Florin Coras222e1f412019-02-16 20:47:32 -08001445 else
1446 tc->rcv_dupacks = 0;
Florin Coras6792ec02017-03-13 03:49:51 -07001447
1448 /* Start again from the beginning */
Florin Corasd2aab832018-05-22 11:39:59 -07001449 tc->cc_algo->congestion (tc);
Florin Coras6792ec02017-03-13 03:49:51 -07001450 tc->cwnd = tcp_loss_wnd (tc);
Florin Coras47596832019-03-12 18:58:54 -07001451 tc->snd_congestion = tc->snd_nxt;
Florin Corasf1762d62017-09-24 19:43:08 -04001452 tc->rtt_ts = 0;
Florin Corasd2aab832018-05-22 11:39:59 -07001453 tc->cwnd_acc_bytes = 0;
Florin Corasc44a5582018-11-01 16:30:54 -07001454 tcp_connection_tx_pacer_reset (tc, tc->cwnd, 2 * tc->snd_mss);
Florin Coras3af90fc2017-05-03 21:09:42 -07001455 tcp_recovery_on (tc);
Florin Coras6792ec02017-03-13 03:49:51 -07001456}
1457
Florin Coras0dbd5172018-06-25 16:19:34 -07001458static inline void
Dave Barach68b0fb02017-02-28 15:15:56 -05001459tcp_timer_retransmit_handler_i (u32 index, u8 is_syn)
1460{
Damjan Marion586afd72017-04-05 19:18:20 +02001461 u32 thread_index = vlib_get_thread_index ();
Florin Corasbe72ae62018-11-01 11:23:03 -07001462 tcp_worker_ctx_t *wrk = tcp_get_worker (thread_index);
1463 vlib_main_t *vm = wrk->vm;
Dave Barach68b0fb02017-02-28 15:15:56 -05001464 tcp_connection_t *tc;
Florin Corasb2215d62017-08-01 16:56:58 -07001465 vlib_buffer_t *b = 0;
Florin Coras3af90fc2017-05-03 21:09:42 -07001466 u32 bi, n_bytes;
Dave Barach68b0fb02017-02-28 15:15:56 -05001467
1468 if (is_syn)
1469 {
1470 tc = tcp_half_open_connection_get (index);
Dave Barachb7f1faa2017-08-29 11:43:37 -04001471 /* Note: the connection may have transitioned to ESTABLISHED... */
Florin Coras311e11b2018-12-05 11:19:14 -08001472 if (PREDICT_FALSE (tc == 0 || tc->state != TCP_STATE_SYN_SENT))
Dave Barachb7f1faa2017-08-29 11:43:37 -04001473 return;
Florin Coras68810622017-07-24 17:40:28 -07001474 tc->timers[TCP_TIMER_RETRANSMIT_SYN] = TCP_TIMER_HANDLE_INVALID;
Dave Barach68b0fb02017-02-28 15:15:56 -05001475 }
1476 else
1477 {
1478 tc = tcp_connection_get (index, thread_index);
Dave Barachb7f1faa2017-08-29 11:43:37 -04001479 /* Note: the connection may have been closed and pool_put */
Florin Coras85a3ddd2018-12-24 16:54:34 -08001480 if (PREDICT_FALSE (tc == 0 || tc->state == TCP_STATE_SYN_SENT))
Dave Barachb7f1faa2017-08-29 11:43:37 -04001481 return;
Florin Coras68810622017-07-24 17:40:28 -07001482 tc->timers[TCP_TIMER_RETRANSMIT] = TCP_TIMER_HANDLE_INVALID;
Florin Coras85a3ddd2018-12-24 16:54:34 -08001483 /* Wait-close and retransmit could pop at the same time */
1484 if (tc->state == TCP_STATE_CLOSED)
1485 return;
Dave Barach68b0fb02017-02-28 15:15:56 -05001486 }
1487
Florin Corase04c2992017-03-01 08:17:34 -08001488 if (tc->state >= TCP_STATE_ESTABLISHED)
Dave Barach68b0fb02017-02-28 15:15:56 -05001489 {
Florin Corase55a6d72018-10-31 23:09:22 -07001490 TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 2);
1491
Florin Coras93992a92017-05-24 18:03:56 -07001492 /* Lost FIN, retransmit and return */
Florin Coras222e1f412019-02-16 20:47:32 -08001493 if (tc->flags & TCP_CONN_FINSNT)
Florin Coras93992a92017-05-24 18:03:56 -07001494 {
1495 tcp_send_fin (tc);
Florin Corasf988e692017-11-27 04:34:14 -05001496 tc->rto_boff += 1;
1497 tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
Florin Coras93992a92017-05-24 18:03:56 -07001498 return;
1499 }
1500
Florin Coras3ec66b02018-08-23 16:27:05 -07001501 /* Shouldn't be here. This condition is tricky because it has to take
1502 * into account boff > 0 due to persist timeout. */
Florin Coras47596832019-03-12 18:58:54 -07001503 if ((tc->rto_boff == 0 && tc->snd_una == tc->snd_nxt)
Florin Coras3ec66b02018-08-23 16:27:05 -07001504 || (tc->rto_boff > 0 && seq_geq (tc->snd_una, tc->snd_congestion)
1505 && !tcp_flight_size (tc)))
Florin Corasa096f2d2017-09-28 23:49:42 -04001506 {
Florin Coras3ec66b02018-08-23 16:27:05 -07001507 ASSERT (!tcp_in_recovery (tc));
1508 tc->rto_boff = 0;
Florin Corasa096f2d2017-09-28 23:49:42 -04001509 return;
1510 }
1511
Florin Coras3ec66b02018-08-23 16:27:05 -07001512 /* We're not in recovery so make sure rto_boff is 0. Can be non 0 due
1513 * to persist timer timeout */
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001514 if (!tcp_in_recovery (tc) && tc->rto_boff > 0)
1515 {
1516 tc->rto_boff = 0;
1517 tcp_update_rto (tc);
1518 }
1519
Florin Coras548f7572019-06-07 15:31:06 -07001520 /* Peer is dead or network connectivity is lost. Close connection.
1521 * RFC 1122 section 4.2.3.5 recommends a value of at least 100s. For
1522 * a min rto of 0.2s we need to retry about 8 times. */
1523 if (tc->rto_boff >= TCP_RTO_BOFF_MAX)
1524 {
1525 tcp_send_reset (tc);
1526 tcp_connection_set_state (tc, TCP_STATE_CLOSED);
1527 session_transport_closing_notify (&tc->connection);
1528 tcp_connection_timers_reset (tc);
1529 tcp_timer_update (tc, TCP_TIMER_WAITCLOSE, TCP_CLOSEWAIT_TIME);
1530 return;
1531 }
1532
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001533 /* Increment RTO backoff (also equal to number of retries) and go back
1534 * to first un-acked byte */
1535 tc->rto_boff += 1;
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001536
Florin Coras6792ec02017-03-13 03:49:51 -07001537 /* First retransmit timeout */
1538 if (tc->rto_boff == 1)
Florin Corasca1c8f32018-05-23 21:01:30 -07001539 tcp_rxt_timeout_cc (tc);
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001540 else
1541 scoreboard_clear (&tc->sack_sb);
Dave Barach68b0fb02017-02-28 15:15:56 -05001542
Florin Coras3ec66b02018-08-23 16:27:05 -07001543 /* If we've sent beyond snd_congestion, update it */
Florin Coras47596832019-03-12 18:58:54 -07001544 tc->snd_congestion = seq_max (tc->snd_nxt, tc->snd_congestion);
Florin Coras3ec66b02018-08-23 16:27:05 -07001545
Florin Coras47596832019-03-12 18:58:54 -07001546 tc->snd_nxt = tc->snd_una;
Dave Barach68b0fb02017-02-28 15:15:56 -05001547 tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
1548
Florin Coras3ec66b02018-08-23 16:27:05 -07001549 /* Send one segment. Note that n_bytes may be zero due to buffer
1550 * shortfall */
Florin Corasbe72ae62018-11-01 11:23:03 -07001551 n_bytes = tcp_prepare_retransmit_segment (wrk, tc, 0, tc->snd_mss, &b);
Florin Coras85a3ddd2018-12-24 16:54:34 -08001552 if (!n_bytes)
Florin Corase04c2992017-03-01 08:17:34 -08001553 {
Florin Coras222e1f412019-02-16 20:47:32 -08001554 tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1);
Florin Corase04c2992017-03-01 08:17:34 -08001555 return;
1556 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001557
Dave Barachb7f1faa2017-08-29 11:43:37 -04001558 bi = vlib_get_buffer_index (vm, b);
1559
Florin Coras93992a92017-05-24 18:03:56 -07001560 /* For first retransmit, record timestamp (Eifel detection RFC3522) */
1561 if (tc->rto_boff == 1)
Florin Corasbe72ae62018-11-01 11:23:03 -07001562 tc->snd_rxt_ts = tcp_time_now_w_thread (tc->c_thread_index);
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001563
Florin Corasbe72ae62018-11-01 11:23:03 -07001564 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Florin Coras3ec66b02018-08-23 16:27:05 -07001565 tcp_retransmit_timer_force_update (tc);
Florin Coras93992a92017-05-24 18:03:56 -07001566 }
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001567 /* Retransmit for SYN */
1568 else if (tc->state == TCP_STATE_SYN_SENT)
Florin Coras93992a92017-05-24 18:03:56 -07001569 {
Florin Coras68810622017-07-24 17:40:28 -07001570 /* Half-open connection actually moved to established but we were
1571 * waiting for syn retransmit to pop to call cleanup from the right
1572 * thread. */
1573 if (tc->flags & TCP_CONN_HALF_OPEN_DONE)
1574 {
Florin Coras68810622017-07-24 17:40:28 -07001575 if (tcp_half_open_connection_cleanup (tc))
Florin Coras85a3ddd2018-12-24 16:54:34 -08001576 TCP_DBG ("could not remove half-open connection");
Florin Coras68810622017-07-24 17:40:28 -07001577 return;
1578 }
1579
Florin Corase55a6d72018-10-31 23:09:22 -07001580 TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 2);
1581
Dave Barach68b0fb02017-02-28 15:15:56 -05001582 /* Try without increasing RTO a number of times. If this fails,
1583 * start growing RTO exponentially */
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001584 tc->rto_boff += 1;
Dave Barach68b0fb02017-02-28 15:15:56 -05001585 if (tc->rto_boff > TCP_RTO_SYN_RETRIES)
1586 tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
1587
Florin Corasf988e692017-11-27 04:34:14 -05001588 tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN,
1589 tc->rto * TCP_TO_TIMER_TICK);
1590
Florin Corasbdf7fd62019-01-31 17:31:01 -08001591 if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
Florin Coras222e1f412019-02-16 20:47:32 -08001592 {
1593 tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN, 1);
1594 return;
1595 }
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001596
Florin Corasb2215d62017-08-01 16:56:58 -07001597 b = vlib_get_buffer (vm, bi);
Florin Coras1f152cd2017-08-18 19:28:03 -07001598 tcp_init_buffer (vm, b);
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001599 tcp_make_syn (tc, b);
Florin Corase04c2992017-03-01 08:17:34 -08001600
Dave Barach2c25a622017-06-26 11:35:07 -04001601 tc->rtt_ts = 0;
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001602 TCP_EVT_DBG (TCP_EVT_SYN_RXT, tc, 0);
1603
1604 /* This goes straight to ipx_lookup. Retransmit timer set already */
Florin Corasbe72ae62018-11-01 11:23:03 -07001605 tcp_push_ip_hdr (wrk, tc, b);
1606 tcp_enqueue_to_ip_lookup (wrk, b, bi, tc->c_is_ip4, tc->c_fib_index);
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001607 }
1608 /* Retransmit SYN-ACK */
1609 else if (tc->state == TCP_STATE_SYN_RCVD)
1610 {
Florin Corase55a6d72018-10-31 23:09:22 -07001611 TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 2);
1612
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001613 tc->rto_boff += 1;
Florin Coras9d063042017-09-14 03:08:00 -04001614 if (tc->rto_boff > TCP_RTO_SYN_RETRIES)
1615 tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001616 tc->rtt_ts = 0;
1617
Florin Coras222e1f412019-02-16 20:47:32 -08001618 tcp_retransmit_timer_force_update (tc);
1619
Florin Corasbdf7fd62019-01-31 17:31:01 -08001620 if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
Florin Corasf988e692017-11-27 04:34:14 -05001621 {
Florin Coras222e1f412019-02-16 20:47:32 -08001622 tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1);
Florin Corasf988e692017-11-27 04:34:14 -05001623 return;
1624 }
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001625
1626 b = vlib_get_buffer (vm, bi);
Florin Coras24793e32018-05-09 11:34:25 -07001627 tcp_init_buffer (vm, b);
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001628 tcp_make_synack (tc, b);
1629 TCP_EVT_DBG (TCP_EVT_SYN_RXT, tc, 1);
1630
1631 /* Retransmit timer already updated, just enqueue to output */
Florin Corasbe72ae62018-11-01 11:23:03 -07001632 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Dave Barach68b0fb02017-02-28 15:15:56 -05001633 }
Florin Coras93992a92017-05-24 18:03:56 -07001634 else
1635 {
1636 ASSERT (tc->state == TCP_STATE_CLOSED);
Florin Coras93992a92017-05-24 18:03:56 -07001637 return;
1638 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001639}
1640
1641void
1642tcp_timer_retransmit_handler (u32 index)
1643{
1644 tcp_timer_retransmit_handler_i (index, 0);
1645}
1646
1647void
1648tcp_timer_retransmit_syn_handler (u32 index)
1649{
1650 tcp_timer_retransmit_handler_i (index, 1);
1651}
1652
1653/**
Florin Coras3e350af2017-03-30 02:54:28 -07001654 * Got 0 snd_wnd from peer, try to do something about it.
1655 *
1656 */
1657void
1658tcp_timer_persist_handler (u32 index)
1659{
Damjan Marion586afd72017-04-05 19:18:20 +02001660 u32 thread_index = vlib_get_thread_index ();
Florin Corasbe72ae62018-11-01 11:23:03 -07001661 tcp_worker_ctx_t *wrk = tcp_get_worker (thread_index);
1662 u32 bi, max_snd_bytes, available_bytes, offset;
1663 tcp_main_t *tm = vnet_get_tcp_main ();
1664 vlib_main_t *vm = wrk->vm;
Florin Coras3e350af2017-03-30 02:54:28 -07001665 tcp_connection_t *tc;
1666 vlib_buffer_t *b;
Florin Coras93992a92017-05-24 18:03:56 -07001667 int n_bytes = 0;
Florin Coras1f152cd2017-08-18 19:28:03 -07001668 u8 *data;
Florin Coras3e350af2017-03-30 02:54:28 -07001669
Florin Corasdb84e572017-05-09 18:54:52 -07001670 tc = tcp_connection_get_if_valid (index, thread_index);
Florin Corasdb84e572017-05-09 18:54:52 -07001671 if (!tc)
1672 return;
Florin Coras3e350af2017-03-30 02:54:28 -07001673
1674 /* Make sure timer handle is set to invalid */
1675 tc->timers[TCP_TIMER_PERSIST] = TCP_TIMER_HANDLE_INVALID;
1676
1677 /* Problem already solved or worse */
Florin Coras7e74bf32019-03-06 16:51:58 -08001678 if (tc->state == TCP_STATE_CLOSED || tc->snd_wnd > tc->snd_mss
1679 || (tc->flags & TCP_CONN_FINSNT))
Florin Coras3e350af2017-03-30 02:54:28 -07001680 return;
1681
Florin Coras31c99552019-03-01 13:00:58 -08001682 available_bytes = transport_max_tx_dequeue (&tc->connection);
Florin Coras47596832019-03-12 18:58:54 -07001683 offset = tc->snd_nxt - tc->snd_una;
Florin Coras50958952017-08-29 14:50:13 -07001684
1685 /* Reprogram persist if no new bytes available to send. We may have data
1686 * next time */
1687 if (!available_bytes)
1688 {
1689 tcp_persist_timer_set (tc);
1690 return;
1691 }
1692
1693 if (available_bytes <= offset)
1694 {
1695 ASSERT (tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT));
1696 return;
1697 }
1698
Florin Coras3e350af2017-03-30 02:54:28 -07001699 /* Increment RTO backoff */
1700 tc->rto_boff += 1;
1701 tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
1702
Florin Corasb2215d62017-08-01 16:56:58 -07001703 /*
1704 * Try to force the first unsent segment (or buffer)
1705 */
Florin Corasbdf7fd62019-01-31 17:31:01 -08001706 if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
Florin Coras3ec66b02018-08-23 16:27:05 -07001707 {
1708 tcp_persist_timer_set (tc);
1709 return;
1710 }
Florin Coras3e350af2017-03-30 02:54:28 -07001711 b = vlib_get_buffer (vm, bi);
Florin Coras1f152cd2017-08-18 19:28:03 -07001712 data = tcp_init_buffer (vm, b);
Florin Coras93992a92017-05-24 18:03:56 -07001713
Florin Coras1f152cd2017-08-18 19:28:03 -07001714 tcp_validate_txf_size (tc, offset);
Florin Corasc8343412017-05-04 14:25:50 -07001715 tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts, tc->state);
Florin Coras7e74bf32019-03-06 16:51:58 -08001716 max_snd_bytes = clib_min (tc->snd_mss,
1717 tm->bytes_per_buffer - TRANSPORT_MAX_HDRS_LEN);
1718 n_bytes = session_tx_fifo_peek_bytes (&tc->connection, data, offset,
1719 max_snd_bytes);
Florin Coras3e350af2017-03-30 02:54:28 -07001720 b->current_length = n_bytes;
Florin Coras84275e92017-09-26 12:30:40 -04001721 ASSERT (n_bytes != 0 && (tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT)
1722 || tc->snd_nxt == tc->snd_una_max
1723 || tc->rto_boff > 1));
Florin Coras93992a92017-05-24 18:03:56 -07001724
Florin Coras47596832019-03-12 18:58:54 -07001725 tcp_push_hdr_i (tc, b, tc->snd_nxt, /* compute opts */ 0,
1726 /* burst */ 0, /* update_snd_nxt */ 1);
1727 tc->snd_una_max = seq_max (tc->snd_nxt, tc->snd_una_max);
Florin Coras8b20bf52018-06-14 14:55:50 -07001728 tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una);
Florin Corasbe72ae62018-11-01 11:23:03 -07001729 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Florin Coras3e350af2017-03-30 02:54:28 -07001730
Florin Coras1f152cd2017-08-18 19:28:03 -07001731 /* Just sent new data, enable retransmit */
1732 tcp_retransmit_timer_update (tc);
Florin Coras3e350af2017-03-30 02:54:28 -07001733}
1734
1735/**
Florin Coras6792ec02017-03-13 03:49:51 -07001736 * Retransmit first unacked segment
1737 */
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001738int
Florin Corasbe72ae62018-11-01 11:23:03 -07001739tcp_retransmit_first_unacked (tcp_worker_ctx_t * wrk, tcp_connection_t * tc)
Dave Barach68b0fb02017-02-28 15:15:56 -05001740{
Florin Corasbe72ae62018-11-01 11:23:03 -07001741 vlib_main_t *vm = wrk->vm;
1742 vlib_buffer_t *b;
Florin Coras47596832019-03-12 18:58:54 -07001743 u32 bi, n_bytes;
Dave Barach68b0fb02017-02-28 15:15:56 -05001744
Florin Corase55a6d72018-10-31 23:09:22 -07001745 TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 1);
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001746
Florin Corasbe72ae62018-11-01 11:23:03 -07001747 n_bytes = tcp_prepare_retransmit_segment (wrk, tc, 0, tc->snd_mss, &b);
Florin Corasb2215d62017-08-01 16:56:58 -07001748 if (!n_bytes)
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001749 return -1;
1750
Florin Corasb2215d62017-08-01 16:56:58 -07001751 bi = vlib_get_buffer_index (vm, b);
Florin Corasbe72ae62018-11-01 11:23:03 -07001752 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001753
1754 return 0;
Florin Coras6792ec02017-03-13 03:49:51 -07001755}
1756
Florin Coras36ee9f12018-11-02 12:52:10 -07001757static int
1758tcp_fast_retransmit_unsent (tcp_worker_ctx_t * wrk, tcp_connection_t * tc,
1759 u32 burst_size)
1760{
1761 u32 offset, n_segs = 0, n_written, bi;
1762 vlib_main_t *vm = wrk->vm;
1763 vlib_buffer_t *b = 0;
1764
Florin Coras47596832019-03-12 18:58:54 -07001765 offset = tc->snd_nxt - tc->snd_una;
Florin Coras36ee9f12018-11-02 12:52:10 -07001766 while (n_segs < burst_size)
1767 {
1768 n_written = tcp_prepare_segment (wrk, tc, offset, tc->snd_mss, &b);
1769 if (!n_written)
1770 goto done;
1771
1772 bi = vlib_get_buffer_index (vm, b);
1773 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
1774 offset += n_written;
1775 n_segs += 1;
Florin Coras47596832019-03-12 18:58:54 -07001776
1777 tc->snd_nxt += n_written;
1778 tc->snd_una_max = seq_max (tc->snd_nxt, tc->snd_una_max);
Florin Coras36ee9f12018-11-02 12:52:10 -07001779 }
1780
1781done:
1782 return n_segs;
1783}
1784
1785#define scoreboard_rescue_rxt_valid(_sb, _tc) \
1786 (seq_geq (_sb->rescue_rxt, _tc->snd_una) \
1787 && seq_leq (_sb->rescue_rxt, _tc->snd_congestion))
1788
Florin Coras6792ec02017-03-13 03:49:51 -07001789/**
Florin Coras93992a92017-05-24 18:03:56 -07001790 * Do fast retransmit with SACKs
Florin Coras6792ec02017-03-13 03:49:51 -07001791 */
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001792int
Florin Corasbe72ae62018-11-01 11:23:03 -07001793tcp_fast_retransmit_sack (tcp_worker_ctx_t * wrk, tcp_connection_t * tc,
1794 u32 burst_size)
Dave Barach68b0fb02017-02-28 15:15:56 -05001795{
Florin Coras36ee9f12018-11-02 12:52:10 -07001796 u32 n_written = 0, offset, max_bytes, n_segs = 0, n_segs_now;
Florin Coras93992a92017-05-24 18:03:56 -07001797 sack_scoreboard_hole_t *hole;
Florin Coras36ee9f12018-11-02 12:52:10 -07001798 vlib_main_t *vm = wrk->vm;
1799 vlib_buffer_t *b = 0;
Florin Coras93992a92017-05-24 18:03:56 -07001800 sack_scoreboard_t *sb;
Florin Coras47596832019-03-12 18:58:54 -07001801 u32 bi, max_deq;
Florin Coras93992a92017-05-24 18:03:56 -07001802 int snd_space;
1803 u8 snd_limited = 0, can_rescue = 0;
Dave Barach68b0fb02017-02-28 15:15:56 -05001804
1805 ASSERT (tcp_in_fastrecovery (tc));
Dave Barach68b0fb02017-02-28 15:15:56 -05001806
Florin Corasca1c8f32018-05-23 21:01:30 -07001807 snd_space = tcp_available_cc_snd_space (tc);
Florin Corasca1c8f32018-05-23 21:01:30 -07001808 if (snd_space < tc->snd_mss)
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001809 {
Florin Corasbe72ae62018-11-01 11:23:03 -07001810 tcp_program_fastretransmit (wrk, tc);
Florin Coras36ee9f12018-11-02 12:52:10 -07001811 return 0;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001812 }
Florin Corasca1c8f32018-05-23 21:01:30 -07001813
1814 TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 0);
Florin Coras36ee9f12018-11-02 12:52:10 -07001815 sb = &tc->sack_sb;
1816 hole = scoreboard_get_hole (sb, sb->cur_rxt_hole);
1817
Florin Coras31c99552019-03-01 13:00:58 -08001818 max_deq = transport_max_tx_dequeue (&tc->connection);
Florin Coras47596832019-03-12 18:58:54 -07001819 max_deq -= tc->snd_nxt - tc->snd_una;
Florin Coras36ee9f12018-11-02 12:52:10 -07001820
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001821 while (snd_space > 0 && n_segs < burst_size)
Florin Coras93992a92017-05-24 18:03:56 -07001822 {
Florin Coras36ee9f12018-11-02 12:52:10 -07001823 hole = scoreboard_next_rxt_hole (sb, hole, max_deq, &can_rescue,
1824 &snd_limited);
Florin Coras93992a92017-05-24 18:03:56 -07001825 if (!hole)
1826 {
Florin Coras36ee9f12018-11-02 12:52:10 -07001827 if (max_deq)
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001828 {
Florin Coras36ee9f12018-11-02 12:52:10 -07001829 snd_space = clib_min (max_deq, snd_space);
1830 burst_size = clib_min (burst_size - n_segs,
1831 snd_space / tc->snd_mss);
1832 n_segs_now = tcp_fast_retransmit_unsent (wrk, tc, burst_size);
1833 if (max_deq > n_segs_now * tc->snd_mss)
1834 tcp_program_fastretransmit (wrk, tc);
1835 n_segs += n_segs_now;
1836 goto done;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001837 }
Florin Coras93992a92017-05-24 18:03:56 -07001838
Florin Coras36ee9f12018-11-02 12:52:10 -07001839 if (!can_rescue || scoreboard_rescue_rxt_valid (sb, tc))
1840 break;
1841
Florin Coras93992a92017-05-24 18:03:56 -07001842 /* If rescue rxt undefined or less than snd_una then one segment of
1843 * up to SMSS octets that MUST include the highest outstanding
1844 * unSACKed sequence number SHOULD be returned, and RescueRxt set to
1845 * RecoveryPoint. HighRxt MUST NOT be updated.
1846 */
Florin Coras1f152cd2017-08-18 19:28:03 -07001847 max_bytes = clib_min (tc->snd_mss,
1848 tc->snd_congestion - tc->snd_una);
1849 max_bytes = clib_min (max_bytes, snd_space);
Florin Coras93992a92017-05-24 18:03:56 -07001850 offset = tc->snd_congestion - tc->snd_una - max_bytes;
1851 sb->rescue_rxt = tc->snd_congestion;
Florin Corasbe72ae62018-11-01 11:23:03 -07001852 n_written = tcp_prepare_retransmit_segment (wrk, tc, offset,
1853 max_bytes, &b);
Florin Coras24793e32018-05-09 11:34:25 -07001854 if (!n_written)
1855 goto done;
1856
Florin Corasb2215d62017-08-01 16:56:58 -07001857 bi = vlib_get_buffer_index (vm, b);
Florin Corasbe72ae62018-11-01 11:23:03 -07001858 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001859 n_segs += 1;
Florin Coras93992a92017-05-24 18:03:56 -07001860 break;
1861 }
1862
Florin Coras1f152cd2017-08-18 19:28:03 -07001863 max_bytes = clib_min (hole->end - sb->high_rxt, snd_space);
1864 max_bytes = snd_limited ? clib_min (max_bytes, tc->snd_mss) : max_bytes;
1865 if (max_bytes == 0)
1866 break;
Florin Coras36ee9f12018-11-02 12:52:10 -07001867
Florin Coras93992a92017-05-24 18:03:56 -07001868 offset = sb->high_rxt - tc->snd_una;
Florin Corasbe72ae62018-11-01 11:23:03 -07001869 n_written = tcp_prepare_retransmit_segment (wrk, tc, offset, max_bytes,
1870 &b);
Florin Coras36ee9f12018-11-02 12:52:10 -07001871 ASSERT (n_written <= snd_space);
Florin Coras93992a92017-05-24 18:03:56 -07001872
1873 /* Nothing left to retransmit */
1874 if (n_written == 0)
Florin Corasb2215d62017-08-01 16:56:58 -07001875 break;
Florin Coras93992a92017-05-24 18:03:56 -07001876
Florin Corasb2215d62017-08-01 16:56:58 -07001877 bi = vlib_get_buffer_index (vm, b);
Florin Corasbe72ae62018-11-01 11:23:03 -07001878 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Florin Coras36ee9f12018-11-02 12:52:10 -07001879
1880 sb->high_rxt += n_written;
Florin Coras93992a92017-05-24 18:03:56 -07001881 snd_space -= n_written;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001882 n_segs += 1;
Florin Coras93992a92017-05-24 18:03:56 -07001883 }
1884
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001885 if (hole)
Florin Corasbe72ae62018-11-01 11:23:03 -07001886 tcp_program_fastretransmit (wrk, tc);
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001887
Florin Coras24793e32018-05-09 11:34:25 -07001888done:
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001889 return n_segs;
Florin Coras93992a92017-05-24 18:03:56 -07001890}
1891
1892/**
1893 * Fast retransmit without SACK info
1894 */
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001895int
Florin Corasbe72ae62018-11-01 11:23:03 -07001896tcp_fast_retransmit_no_sack (tcp_worker_ctx_t * wrk, tcp_connection_t * tc,
1897 u32 burst_size)
Florin Coras93992a92017-05-24 18:03:56 -07001898{
Florin Coras47596832019-03-12 18:58:54 -07001899 u32 n_written = 0, offset = 0, bi, max_deq, n_segs_now;
Florin Corasbe72ae62018-11-01 11:23:03 -07001900 vlib_main_t *vm = wrk->vm;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001901 int snd_space, n_segs = 0;
Florin Coras93992a92017-05-24 18:03:56 -07001902 vlib_buffer_t *b;
1903
1904 ASSERT (tcp_in_fastrecovery (tc));
1905 TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 0);
Florin Coras6792ec02017-03-13 03:49:51 -07001906
Florin Coras87a9bf82019-06-12 11:26:35 -07001907 snd_space = tcp_available_cc_snd_space (tc);
1908
Florin Coras36ee9f12018-11-02 12:52:10 -07001909 if (!tcp_fastrecovery_first (tc))
1910 goto send_unsent;
1911
1912 /* RFC 6582: [If a partial ack], retransmit the first unacknowledged
1913 * segment. */
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001914 while (snd_space > 0 && n_segs < burst_size)
Dave Barach68b0fb02017-02-28 15:15:56 -05001915 {
Florin Coras36ee9f12018-11-02 12:52:10 -07001916 n_written = tcp_prepare_retransmit_segment (wrk, tc, offset,
1917 tc->snd_mss, &b);
Dave Barach68b0fb02017-02-28 15:15:56 -05001918
1919 /* Nothing left to retransmit */
Florin Coras6792ec02017-03-13 03:49:51 -07001920 if (n_written == 0)
Florin Corasb2215d62017-08-01 16:56:58 -07001921 break;
Dave Barach68b0fb02017-02-28 15:15:56 -05001922
Florin Corasb2215d62017-08-01 16:56:58 -07001923 bi = vlib_get_buffer_index (vm, b);
Florin Corasbe72ae62018-11-01 11:23:03 -07001924 tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
Florin Coras6792ec02017-03-13 03:49:51 -07001925 snd_space -= n_written;
Florin Coras36ee9f12018-11-02 12:52:10 -07001926 offset += n_written;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001927 n_segs += 1;
Dave Barach68b0fb02017-02-28 15:15:56 -05001928 }
1929
Florin Coras36ee9f12018-11-02 12:52:10 -07001930 if (n_segs == burst_size)
1931 goto done;
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001932
Florin Coras36ee9f12018-11-02 12:52:10 -07001933send_unsent:
1934
1935 /* RFC 6582: Send a new segment if permitted by the new value of cwnd. */
Florin Coras9ece3c02018-11-05 11:06:53 -08001936 if (snd_space < tc->snd_mss || tc->snd_mss == 0)
Florin Coras36ee9f12018-11-02 12:52:10 -07001937 goto done;
1938
Florin Coras31c99552019-03-01 13:00:58 -08001939 max_deq = transport_max_tx_dequeue (&tc->connection);
Florin Coras47596832019-03-12 18:58:54 -07001940 max_deq -= tc->snd_nxt - tc->snd_una;
Florin Coras36ee9f12018-11-02 12:52:10 -07001941 if (max_deq)
1942 {
1943 snd_space = clib_min (max_deq, snd_space);
1944 burst_size = clib_min (burst_size - n_segs, snd_space / tc->snd_mss);
1945 n_segs_now = tcp_fast_retransmit_unsent (wrk, tc, burst_size);
1946 if (max_deq > n_segs_now * tc->snd_mss)
1947 tcp_program_fastretransmit (wrk, tc);
1948 n_segs += n_segs_now;
1949 }
1950
Florin Coras36ee9f12018-11-02 12:52:10 -07001951done:
1952 tcp_fastrecovery_first_off (tc);
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001953 return n_segs;
Florin Coras93992a92017-05-24 18:03:56 -07001954}
1955
1956/**
1957 * Do fast retransmit
1958 */
Florin Corasbf4d5ce2018-10-19 16:26:24 -07001959int
Florin Corasbe72ae62018-11-01 11:23:03 -07001960tcp_fast_retransmit (tcp_worker_ctx_t * wrk, tcp_connection_t * tc,
1961 u32 burst_size)
Florin Coras93992a92017-05-24 18:03:56 -07001962{
Florin Corasca1c8f32018-05-23 21:01:30 -07001963 if (tcp_opts_sack_permitted (&tc->rcv_opts))
Florin Corasbe72ae62018-11-01 11:23:03 -07001964 return tcp_fast_retransmit_sack (wrk, tc, burst_size);
Florin Coras93992a92017-05-24 18:03:56 -07001965 else
Florin Corasbe72ae62018-11-01 11:23:03 -07001966 return tcp_fast_retransmit_no_sack (wrk, tc, burst_size);
Dave Barach68b0fb02017-02-28 15:15:56 -05001967}
Filip Tehlare275bed2019-03-06 00:06:56 -08001968#endif /* CLIB_MARCH_VARIANT */
Dave Barach68b0fb02017-02-28 15:15:56 -05001969
Florin Corasf9d05682018-04-26 08:26:52 -07001970static void
1971tcp_output_handle_link_local (tcp_connection_t * tc0, vlib_buffer_t * b0,
Florin Coras8b20bf52018-06-14 14:55:50 -07001972 u16 * next0, u32 * error0)
Florin Corasf9d05682018-04-26 08:26:52 -07001973{
1974 ip_adjacency_t *adj;
1975 adj_index_t ai;
1976
Florin Coras1c8ff632018-05-17 13:28:34 -07001977 /* Not thread safe but as long as the connection exists the adj should
1978 * not be removed */
Florin Corasf9d05682018-04-26 08:26:52 -07001979 ai = adj_nbr_find (FIB_PROTOCOL_IP6, VNET_LINK_IP6, &tc0->c_rmt_ip,
1980 tc0->sw_if_index);
1981 if (ai == ADJ_INDEX_INVALID)
1982 {
Florin Corasf9d05682018-04-26 08:26:52 -07001983 vnet_buffer (b0)->sw_if_index[VLIB_TX] = ~0;
1984 *next0 = TCP_OUTPUT_NEXT_DROP;
1985 *error0 = TCP_ERROR_LINK_LOCAL_RW;
1986 return;
1987 }
1988
1989 adj = adj_get (ai);
Florin Coras1c8ff632018-05-17 13:28:34 -07001990 if (PREDICT_TRUE (adj->lookup_next_index == IP_LOOKUP_NEXT_REWRITE))
Florin Corasf9d05682018-04-26 08:26:52 -07001991 *next0 = TCP_OUTPUT_NEXT_IP_REWRITE;
1992 else if (adj->lookup_next_index == IP_LOOKUP_NEXT_ARP)
1993 *next0 = TCP_OUTPUT_NEXT_IP_ARP;
Florin Coras1c8ff632018-05-17 13:28:34 -07001994 else
1995 {
1996 *next0 = TCP_OUTPUT_NEXT_DROP;
1997 *error0 = TCP_ERROR_LINK_LOCAL_RW;
1998 }
1999 vnet_buffer (b0)->ip.adj_index[VLIB_TX] = ai;
Florin Corasf9d05682018-04-26 08:26:52 -07002000}
2001
Florin Coras8b20bf52018-06-14 14:55:50 -07002002static void
2003tcp46_output_trace_frame (vlib_main_t * vm, vlib_node_runtime_t * node,
2004 u32 * to_next, u32 n_bufs)
Dave Barach68b0fb02017-02-28 15:15:56 -05002005{
Florin Coras8b20bf52018-06-14 14:55:50 -07002006 u32 n_trace = vlib_get_trace_count (vm, node);
2007 tcp_connection_t *tc;
2008 tcp_tx_trace_t *t;
2009 vlib_buffer_t *b;
2010 tcp_header_t *th;
2011 int i;
Dave Barach68b0fb02017-02-28 15:15:56 -05002012
Florin Coras8b20bf52018-06-14 14:55:50 -07002013 for (i = 0; i < clib_min (n_trace, n_bufs); i++)
Dave Barach68b0fb02017-02-28 15:15:56 -05002014 {
Florin Coras8b20bf52018-06-14 14:55:50 -07002015 b = vlib_get_buffer (vm, to_next[i]);
2016 th = vlib_buffer_get_current (b);
2017 tc = tcp_connection_get (vnet_buffer (b)->tcp.connection_index,
2018 vm->thread_index);
2019 t = vlib_add_trace (vm, node, b, sizeof (*t));
Dave Barach178cf492018-11-13 16:34:13 -05002020 clib_memcpy_fast (&t->tcp_header, th, sizeof (t->tcp_header));
2021 clib_memcpy_fast (&t->tcp_connection, tc, sizeof (t->tcp_connection));
Florin Coras8b20bf52018-06-14 14:55:50 -07002022 }
2023}
Dave Barach68b0fb02017-02-28 15:15:56 -05002024
Florin Coras0dbd5172018-06-25 16:19:34 -07002025always_inline void
Florin Coras8b20bf52018-06-14 14:55:50 -07002026tcp_output_push_ip (vlib_main_t * vm, vlib_buffer_t * b0,
2027 tcp_connection_t * tc0, u8 is_ip4)
2028{
2029 tcp_header_t *th0 = 0;
Dave Barach68b0fb02017-02-28 15:15:56 -05002030
Florin Coras8b20bf52018-06-14 14:55:50 -07002031 th0 = vlib_buffer_get_current (b0);
2032 TCP_EVT_DBG (TCP_EVT_OUTPUT, tc0, th0->flags, b0->current_length);
2033 if (is_ip4)
2034 {
2035 vlib_buffer_push_ip4 (vm, b0, &tc0->c_lcl_ip4, &tc0->c_rmt_ip4,
2036 IP_PROTOCOL_TCP, 1);
2037 b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
2038 vnet_buffer (b0)->l4_hdr_offset = (u8 *) th0 - b0->data;
2039 th0->checksum = 0;
2040 }
2041 else
2042 {
2043 ip6_header_t *ih0;
2044 ih0 = vlib_buffer_push_ip6 (vm, b0, &tc0->c_lcl_ip6,
2045 &tc0->c_rmt_ip6, IP_PROTOCOL_TCP);
2046 b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
2047 vnet_buffer (b0)->l3_hdr_offset = (u8 *) ih0 - b0->data;
2048 vnet_buffer (b0)->l4_hdr_offset = (u8 *) th0 - b0->data;
2049 th0->checksum = 0;
2050 }
2051}
Dave Barach68b0fb02017-02-28 15:15:56 -05002052
Florin Coras0dbd5172018-06-25 16:19:34 -07002053always_inline void
Florin Coras8b20bf52018-06-14 14:55:50 -07002054tcp_output_handle_packet (tcp_connection_t * tc0, vlib_buffer_t * b0,
2055 u32 * error0, u16 * next0, u8 is_ip4)
2056{
Florin Coras81a13db2018-03-16 08:48:31 -07002057
Florin Coras8b20bf52018-06-14 14:55:50 -07002058 if (PREDICT_FALSE (tc0->state == TCP_STATE_CLOSED))
2059 {
2060 *error0 = TCP_ERROR_INVALID_CONNECTION;
2061 *next0 = TCP_OUTPUT_NEXT_DROP;
2062 return;
Dave Barach68b0fb02017-02-28 15:15:56 -05002063 }
2064
Florin Coras540a8da2019-06-18 08:55:14 -07002065 /* If next_index is not drop use it */
2066 *next0 = tc0->out_next_index ? tc0->out_next_index : *next0;
2067
Florin Coras8b20bf52018-06-14 14:55:50 -07002068 vnet_buffer (b0)->sw_if_index[VLIB_TX] = tc0->c_fib_index;
2069 vnet_buffer (b0)->sw_if_index[VLIB_RX] = 0;
2070
2071 if (!is_ip4)
2072 {
2073 if (PREDICT_FALSE (ip6_address_is_link_local_unicast (&tc0->c_rmt_ip6)))
2074 tcp_output_handle_link_local (tc0, b0, next0, error0);
2075 }
2076
Florin Coras8b20bf52018-06-14 14:55:50 -07002077 if (!TCP_ALWAYS_ACK)
2078 tcp_timer_reset (tc0, TCP_TIMER_DELACK);
2079}
2080
2081always_inline uword
2082tcp46_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
2083 vlib_frame_t * frame, int is_ip4)
2084{
2085 u32 n_left_from, *from, thread_index = vm->thread_index;
2086 vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
2087 u16 nexts[VLIB_FRAME_SIZE], *next;
2088
2089 from = vlib_frame_vector_args (frame);
2090 n_left_from = frame->n_vectors;
Florin Corasbe72ae62018-11-01 11:23:03 -07002091 tcp_set_time_now (tcp_get_worker (thread_index));
Florin Coras8b20bf52018-06-14 14:55:50 -07002092
2093 if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
2094 tcp46_output_trace_frame (vm, node, from, n_left_from);
2095
2096 vlib_get_buffers (vm, from, bufs, n_left_from);
2097 b = bufs;
2098 next = nexts;
2099
2100 while (n_left_from >= 4)
2101 {
2102 u32 error0 = TCP_ERROR_PKTS_SENT, error1 = TCP_ERROR_PKTS_SENT;
2103 tcp_connection_t *tc0, *tc1;
2104
2105 {
2106 vlib_prefetch_buffer_header (b[2], STORE);
2107 CLIB_PREFETCH (b[2]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE);
2108
2109 vlib_prefetch_buffer_header (b[3], STORE);
2110 CLIB_PREFETCH (b[3]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE);
2111 }
2112
2113 next[0] = next[1] = TCP_OUTPUT_NEXT_IP_LOOKUP;
2114
2115 tc0 = tcp_connection_get (vnet_buffer (b[0])->tcp.connection_index,
2116 thread_index);
2117 tc1 = tcp_connection_get (vnet_buffer (b[1])->tcp.connection_index,
2118 thread_index);
2119
2120 tcp_output_push_ip (vm, b[0], tc0, is_ip4);
2121 tcp_output_push_ip (vm, b[1], tc1, is_ip4);
2122
2123 tcp_output_handle_packet (tc0, b[0], &error0, &next[0], is_ip4);
2124 tcp_output_handle_packet (tc1, b[1], &error1, &next[1], is_ip4);
2125
2126 b += 2;
2127 next += 2;
2128 n_left_from -= 2;
2129 }
2130 while (n_left_from > 0)
2131 {
2132 u32 error0 = TCP_ERROR_PKTS_SENT;
2133 tcp_connection_t *tc0;
2134
2135 if (n_left_from > 1)
2136 {
Florin Coras91ca4622018-06-30 11:27:59 -07002137 vlib_prefetch_buffer_header (b[1], STORE);
2138 CLIB_PREFETCH (b[1]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE);
Florin Coras8b20bf52018-06-14 14:55:50 -07002139 }
2140
2141 next[0] = TCP_OUTPUT_NEXT_IP_LOOKUP;
2142 tc0 = tcp_connection_get (vnet_buffer (b[0])->tcp.connection_index,
2143 thread_index);
2144
2145 tcp_output_push_ip (vm, b[0], tc0, is_ip4);
2146 tcp_output_handle_packet (tc0, b[0], &error0, &next[0], is_ip4);
2147
2148 b += 1;
2149 next += 1;
2150 n_left_from -= 1;
2151 }
2152
2153 vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
2154 return frame->n_vectors;
Dave Barach68b0fb02017-02-28 15:15:56 -05002155}
2156
Filip Tehlare275bed2019-03-06 00:06:56 -08002157VLIB_NODE_FN (tcp4_output_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
2158 vlib_frame_t * from_frame)
Dave Barach68b0fb02017-02-28 15:15:56 -05002159{
2160 return tcp46_output_inline (vm, node, from_frame, 1 /* is_ip4 */ );
2161}
2162
Filip Tehlare275bed2019-03-06 00:06:56 -08002163VLIB_NODE_FN (tcp6_output_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
2164 vlib_frame_t * from_frame)
Dave Barach68b0fb02017-02-28 15:15:56 -05002165{
2166 return tcp46_output_inline (vm, node, from_frame, 0 /* is_ip4 */ );
2167}
2168
Florin Corase69f4952017-03-07 10:06:24 -08002169/* *INDENT-OFF* */
Dave Barach68b0fb02017-02-28 15:15:56 -05002170VLIB_REGISTER_NODE (tcp4_output_node) =
2171{
Dave Barachf970d2f2018-11-26 18:34:33 -05002172 .name = "tcp4-output",
2173 /* Takes a vector of packets. */
2174 .vector_size = sizeof (u32),
2175 .n_errors = TCP_N_ERROR,
Dave Barach7fff3d22018-11-27 16:52:59 -05002176 .protocol_hint = VLIB_NODE_PROTO_HINT_TCP,
Dave Barachf970d2f2018-11-26 18:34:33 -05002177 .error_strings = tcp_error_strings,
2178 .n_next_nodes = TCP_OUTPUT_N_NEXT,
2179 .next_nodes = {
Dave Barach68b0fb02017-02-28 15:15:56 -05002180#define _(s,n) [TCP_OUTPUT_NEXT_##s] = n,
2181 foreach_tcp4_output_next
2182#undef _
Dave Barachf970d2f2018-11-26 18:34:33 -05002183 },
2184 .format_buffer = format_tcp_header,
2185 .format_trace = format_tcp_tx_trace,
Florin Corase69f4952017-03-07 10:06:24 -08002186};
2187/* *INDENT-ON* */
Dave Barach68b0fb02017-02-28 15:15:56 -05002188
Florin Corase69f4952017-03-07 10:06:24 -08002189/* *INDENT-OFF* */
Dave Barach68b0fb02017-02-28 15:15:56 -05002190VLIB_REGISTER_NODE (tcp6_output_node) =
2191{
Florin Corase69f4952017-03-07 10:06:24 -08002192 .name = "tcp6-output",
Dave Barach68b0fb02017-02-28 15:15:56 -05002193 /* Takes a vector of packets. */
Florin Corase69f4952017-03-07 10:06:24 -08002194 .vector_size = sizeof (u32),
2195 .n_errors = TCP_N_ERROR,
Dave Barach7fff3d22018-11-27 16:52:59 -05002196 .protocol_hint = VLIB_NODE_PROTO_HINT_TCP,
Florin Corase69f4952017-03-07 10:06:24 -08002197 .error_strings = tcp_error_strings,
2198 .n_next_nodes = TCP_OUTPUT_N_NEXT,
2199 .next_nodes = {
Dave Barach68b0fb02017-02-28 15:15:56 -05002200#define _(s,n) [TCP_OUTPUT_NEXT_##s] = n,
2201 foreach_tcp6_output_next
2202#undef _
Florin Corase69f4952017-03-07 10:06:24 -08002203 },
2204 .format_buffer = format_tcp_header,
2205 .format_trace = format_tcp_tx_trace,
2206};
2207/* *INDENT-ON* */
Dave Barach68b0fb02017-02-28 15:15:56 -05002208
Dave Barach68b0fb02017-02-28 15:15:56 -05002209typedef enum _tcp_reset_next
2210{
2211 TCP_RESET_NEXT_DROP,
2212 TCP_RESET_NEXT_IP_LOOKUP,
2213 TCP_RESET_N_NEXT
2214} tcp_reset_next_t;
2215
2216#define foreach_tcp4_reset_next \
2217 _(DROP, "error-drop") \
2218 _(IP_LOOKUP, "ip4-lookup")
2219
2220#define foreach_tcp6_reset_next \
2221 _(DROP, "error-drop") \
2222 _(IP_LOOKUP, "ip6-lookup")
2223
2224static uword
2225tcp46_send_reset_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
2226 vlib_frame_t * from_frame, u8 is_ip4)
2227{
2228 u32 n_left_from, next_index, *from, *to_next;
Damjan Marion586afd72017-04-05 19:18:20 +02002229 u32 my_thread_index = vm->thread_index;
Dave Barach68b0fb02017-02-28 15:15:56 -05002230
2231 from = vlib_frame_vector_args (from_frame);
2232 n_left_from = from_frame->n_vectors;
2233
2234 next_index = node->cached_next_index;
2235
2236 while (n_left_from > 0)
2237 {
2238 u32 n_left_to_next;
2239
2240 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2241
2242 while (n_left_from > 0 && n_left_to_next > 0)
2243 {
2244 u32 bi0;
2245 vlib_buffer_t *b0;
Clement Durand6cf260c2017-04-13 13:27:04 +02002246 tcp_tx_trace_t *t0;
2247 tcp_header_t *th0;
Dave Barach68b0fb02017-02-28 15:15:56 -05002248 u32 error0 = TCP_ERROR_RST_SENT, next0 = TCP_RESET_NEXT_IP_LOOKUP;
2249
2250 bi0 = from[0];
2251 to_next[0] = bi0;
2252 from += 1;
2253 to_next += 1;
2254 n_left_from -= 1;
2255 n_left_to_next -= 1;
2256
2257 b0 = vlib_get_buffer (vm, bi0);
2258
2259 if (tcp_make_reset_in_place (vm, b0, vnet_buffer (b0)->tcp.flags,
2260 my_thread_index, is_ip4))
2261 {
2262 error0 = TCP_ERROR_LOOKUP_DROPS;
2263 next0 = TCP_RESET_NEXT_DROP;
2264 goto done;
2265 }
2266
2267 /* Prepare to send to IP lookup */
Florin Corasf988e692017-11-27 04:34:14 -05002268 vnet_buffer (b0)->sw_if_index[VLIB_TX] = ~0;
Dave Barach68b0fb02017-02-28 15:15:56 -05002269 next0 = TCP_RESET_NEXT_IP_LOOKUP;
2270
2271 done:
Florin Corase69f4952017-03-07 10:06:24 -08002272 b0->error = node->errors[error0];
Damjan Marion213b5aa2017-07-13 21:19:27 +02002273 b0->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
Dave Barach68b0fb02017-02-28 15:15:56 -05002274 if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
2275 {
Clement Durand6cf260c2017-04-13 13:27:04 +02002276 th0 = vlib_buffer_get_current (b0);
2277 if (is_ip4)
2278 th0 = ip4_next_header ((ip4_header_t *) th0);
2279 else
2280 th0 = ip6_next_header ((ip6_header_t *) th0);
Clement Durand6cf260c2017-04-13 13:27:04 +02002281 t0 = vlib_add_trace (vm, node, b0, sizeof (*t0));
Dave Barach178cf492018-11-13 16:34:13 -05002282 clib_memcpy_fast (&t0->tcp_header, th0,
2283 sizeof (t0->tcp_header));
Dave Barach68b0fb02017-02-28 15:15:56 -05002284 }
2285
2286 vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
2287 n_left_to_next, bi0, next0);
2288 }
2289 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2290 }
2291 return from_frame->n_vectors;
2292}
2293
Filip Tehlare275bed2019-03-06 00:06:56 -08002294VLIB_NODE_FN (tcp4_reset_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
2295 vlib_frame_t * from_frame)
Dave Barach68b0fb02017-02-28 15:15:56 -05002296{
2297 return tcp46_send_reset_inline (vm, node, from_frame, 1);
2298}
2299
Filip Tehlare275bed2019-03-06 00:06:56 -08002300VLIB_NODE_FN (tcp6_reset_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
2301 vlib_frame_t * from_frame)
Dave Barach68b0fb02017-02-28 15:15:56 -05002302{
2303 return tcp46_send_reset_inline (vm, node, from_frame, 0);
2304}
2305
2306/* *INDENT-OFF* */
2307VLIB_REGISTER_NODE (tcp4_reset_node) = {
Dave Barach68b0fb02017-02-28 15:15:56 -05002308 .name = "tcp4-reset",
2309 .vector_size = sizeof (u32),
2310 .n_errors = TCP_N_ERROR,
2311 .error_strings = tcp_error_strings,
2312 .n_next_nodes = TCP_RESET_N_NEXT,
2313 .next_nodes = {
2314#define _(s,n) [TCP_RESET_NEXT_##s] = n,
2315 foreach_tcp4_reset_next
2316#undef _
2317 },
Clement Durand6cf260c2017-04-13 13:27:04 +02002318 .format_trace = format_tcp_tx_trace,
Dave Barach68b0fb02017-02-28 15:15:56 -05002319};
2320/* *INDENT-ON* */
2321
2322/* *INDENT-OFF* */
2323VLIB_REGISTER_NODE (tcp6_reset_node) = {
Dave Barach68b0fb02017-02-28 15:15:56 -05002324 .name = "tcp6-reset",
2325 .vector_size = sizeof (u32),
2326 .n_errors = TCP_N_ERROR,
2327 .error_strings = tcp_error_strings,
2328 .n_next_nodes = TCP_RESET_N_NEXT,
2329 .next_nodes = {
2330#define _(s,n) [TCP_RESET_NEXT_##s] = n,
2331 foreach_tcp6_reset_next
2332#undef _
2333 },
Clement Durand6cf260c2017-04-13 13:27:04 +02002334 .format_trace = format_tcp_tx_trace,
Dave Barach68b0fb02017-02-28 15:15:56 -05002335};
2336/* *INDENT-ON* */
2337
Dave Barach68b0fb02017-02-28 15:15:56 -05002338/*
2339 * fd.io coding-style-patch-verification: ON
2340 *
2341 * Local Variables:
2342 * eval: (c-set-style "gnu")
2343 * End:
2344 */