Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2016 Cisco and/or its affiliates. |
| 3 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | * you may not use this file except in compliance with the License. |
| 5 | * You may obtain a copy of the License at: |
| 6 | * |
| 7 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | * |
| 9 | * Unless required by applicable law or agreed to in writing, software |
| 10 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | * See the License for the specific language governing permissions and |
| 13 | * limitations under the License. |
| 14 | */ |
| 15 | |
| 16 | #ifndef _vnet_tcp_h_ |
| 17 | #define _vnet_tcp_h_ |
| 18 | |
| 19 | #include <vnet/vnet.h> |
| 20 | #include <vnet/ip/ip.h> |
| 21 | #include <vnet/tcp/tcp_packet.h> |
| 22 | #include <vnet/tcp/tcp_timer.h> |
| 23 | #include <vnet/session/transport.h> |
| 24 | #include <vnet/session/session.h> |
Florin Coras | e69f495 | 2017-03-07 10:06:24 -0800 | [diff] [blame] | 25 | #include <vnet/tcp/tcp_debug.h> |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 26 | |
Florin Coras | 3af90fc | 2017-05-03 21:09:42 -0700 | [diff] [blame] | 27 | #define TCP_TICK 0.001 /**< TCP tick period (s) */ |
| 28 | #define THZ (u32) (1/TCP_TICK) /**< TCP tick frequency */ |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 29 | #define TCP_TSTAMP_RESOLUTION TCP_TICK /**< Time stamp resolution */ |
| 30 | #define TCP_PAWS_IDLE 24 * 24 * 60 * 60 * THZ /**< 24 days */ |
| 31 | #define TCP_MAX_OPTION_SPACE 40 |
| 32 | |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 33 | #define TCP_DUPACK_THRESHOLD 3 |
Florin Coras | f03a59a | 2017-06-09 21:07:32 -0700 | [diff] [blame^] | 34 | #define TCP_MAX_RX_FIFO_SIZE 4 << 20 |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 35 | #define TCP_IW_N_SEGMENTS 10 |
Florin Coras | f03a59a | 2017-06-09 21:07:32 -0700 | [diff] [blame^] | 36 | #define TCP_ALWAYS_ACK 1 /**< On/off delayed acks */ |
Florin Coras | 93992a9 | 2017-05-24 18:03:56 -0700 | [diff] [blame] | 37 | #define TCP_USE_SACKS 1 /**< Disable only for testing */ |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 38 | |
| 39 | /** TCP FSM state definitions as per RFC793. */ |
| 40 | #define foreach_tcp_fsm_state \ |
| 41 | _(CLOSED, "CLOSED") \ |
| 42 | _(LISTEN, "LISTEN") \ |
| 43 | _(SYN_SENT, "SYN_SENT") \ |
| 44 | _(SYN_RCVD, "SYN_RCVD") \ |
| 45 | _(ESTABLISHED, "ESTABLISHED") \ |
| 46 | _(CLOSE_WAIT, "CLOSE_WAIT") \ |
| 47 | _(FIN_WAIT_1, "FIN_WAIT_1") \ |
| 48 | _(LAST_ACK, "LAST_ACK") \ |
| 49 | _(CLOSING, "CLOSING") \ |
| 50 | _(FIN_WAIT_2, "FIN_WAIT_2") \ |
| 51 | _(TIME_WAIT, "TIME_WAIT") |
| 52 | |
| 53 | typedef enum _tcp_state |
| 54 | { |
| 55 | #define _(sym, str) TCP_STATE_##sym, |
| 56 | foreach_tcp_fsm_state |
| 57 | #undef _ |
| 58 | TCP_N_STATES |
| 59 | } tcp_state_t; |
| 60 | |
| 61 | format_function_t format_tcp_state; |
Dave Barach | 1f75cfd | 2017-04-14 16:46:44 -0400 | [diff] [blame] | 62 | format_function_t format_tcp_flags; |
Florin Coras | 45d3496 | 2017-04-25 00:05:27 -0700 | [diff] [blame] | 63 | format_function_t format_tcp_sacks; |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 64 | |
| 65 | /** TCP timers */ |
| 66 | #define foreach_tcp_timer \ |
| 67 | _(RETRANSMIT, "RETRANSMIT") \ |
| 68 | _(DELACK, "DELAYED ACK") \ |
| 69 | _(PERSIST, "PERSIST") \ |
| 70 | _(KEEP, "KEEP") \ |
Florin Coras | d79b41e | 2017-03-04 05:37:52 -0800 | [diff] [blame] | 71 | _(WAITCLOSE, "WAIT CLOSE") \ |
| 72 | _(RETRANSMIT_SYN, "RETRANSMIT SYN") \ |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 73 | _(ESTABLISH, "ESTABLISH") |
| 74 | |
| 75 | typedef enum _tcp_timers |
| 76 | { |
| 77 | #define _(sym, str) TCP_TIMER_##sym, |
| 78 | foreach_tcp_timer |
| 79 | #undef _ |
| 80 | TCP_N_TIMERS |
| 81 | } tcp_timers_e; |
| 82 | |
| 83 | typedef void (timer_expiration_handler) (u32 index); |
| 84 | |
| 85 | extern timer_expiration_handler tcp_timer_delack_handler; |
| 86 | extern timer_expiration_handler tcp_timer_retransmit_handler; |
Florin Coras | 3e350af | 2017-03-30 02:54:28 -0700 | [diff] [blame] | 87 | extern timer_expiration_handler tcp_timer_persist_handler; |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 88 | extern timer_expiration_handler tcp_timer_retransmit_syn_handler; |
| 89 | |
| 90 | #define TCP_TIMER_HANDLE_INVALID ((u32) ~0) |
| 91 | |
| 92 | /* Timer delays as multiples of 100ms */ |
| 93 | #define TCP_TO_TIMER_TICK TCP_TICK*10 /* Period for converting from TCP |
| 94 | * ticks to timer units */ |
| 95 | #define TCP_DELACK_TIME 1 /* 0.1s */ |
| 96 | #define TCP_ESTABLISH_TIME 750 /* 75s */ |
| 97 | #define TCP_2MSL_TIME 300 /* 30s */ |
Florin Coras | 93992a9 | 2017-05-24 18:03:56 -0700 | [diff] [blame] | 98 | #define TCP_CLOSEWAIT_TIME 20 /* 0.1s */ |
Florin Coras | d79b41e | 2017-03-04 05:37:52 -0800 | [diff] [blame] | 99 | #define TCP_CLEANUP_TIME 5 /* 0.5s Time to wait before cleanup */ |
Florin Coras | 5921f98 | 2017-04-03 18:00:00 -0700 | [diff] [blame] | 100 | #define TCP_TIMER_PERSIST_MIN 2 /* 0.2s */ |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 101 | |
| 102 | #define TCP_RTO_MAX 60 * THZ /* Min max RTO (60s) as per RFC6298 */ |
Florin Coras | f03a59a | 2017-06-09 21:07:32 -0700 | [diff] [blame^] | 103 | #define TCP_RTO_MIN 0.2 * THZ /* Min RTO (200ms) - lower than standard */ |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 104 | #define TCP_RTT_MAX 30 * THZ /* 30s (probably too much) */ |
| 105 | #define TCP_RTO_SYN_RETRIES 3 /* SYN retries without doubling RTO */ |
| 106 | #define TCP_RTO_INIT 1 * THZ /* Initial retransmit timer */ |
| 107 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 108 | /** TCP connection flags */ |
| 109 | #define foreach_tcp_connection_flag \ |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 110 | _(SNDACK, "Send ACK") \ |
Florin Coras | d79b41e | 2017-03-04 05:37:52 -0800 | [diff] [blame] | 111 | _(FINSNT, "FIN sent") \ |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 112 | _(SENT_RCV_WND0, "Sent 0 receive window") \ |
| 113 | _(RECOVERY, "Recovery on") \ |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 114 | _(FAST_RECOVERY, "Fast Recovery on") \ |
| 115 | _(FR_1_SMSS, "Sent 1 SMSS") |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 116 | |
| 117 | typedef enum _tcp_connection_flag_bits |
| 118 | { |
| 119 | #define _(sym, str) TCP_CONN_##sym##_BIT, |
| 120 | foreach_tcp_connection_flag |
| 121 | #undef _ |
| 122 | TCP_CONN_N_FLAG_BITS |
| 123 | } tcp_connection_flag_bits_e; |
| 124 | |
| 125 | typedef enum _tcp_connection_flag |
| 126 | { |
| 127 | #define _(sym, str) TCP_CONN_##sym = 1 << TCP_CONN_##sym##_BIT, |
| 128 | foreach_tcp_connection_flag |
| 129 | #undef _ |
| 130 | TCP_CONN_N_FLAGS |
| 131 | } tcp_connection_flags_e; |
| 132 | |
| 133 | /** TCP buffer flags */ |
| 134 | #define foreach_tcp_buf_flag \ |
| 135 | _ (ACK) /**< Sending ACK. */ \ |
| 136 | _ (DUPACK) /**< Sending DUPACK. */ \ |
| 137 | |
| 138 | enum |
| 139 | { |
| 140 | #define _(f) TCP_BUF_BIT_##f, |
| 141 | foreach_tcp_buf_flag |
| 142 | #undef _ |
| 143 | TCP_N_BUF_BITS, |
| 144 | }; |
| 145 | |
| 146 | enum |
| 147 | { |
| 148 | #define _(f) TCP_BUF_FLAG_##f = 1 << TCP_BUF_BIT_##f, |
| 149 | foreach_tcp_buf_flag |
| 150 | #undef _ |
| 151 | }; |
| 152 | |
Florin Coras | f03a59a | 2017-06-09 21:07:32 -0700 | [diff] [blame^] | 153 | #define TCP_MAX_SACK_BLOCKS 15 /**< Max number of SACK blocks stored */ |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 154 | #define TCP_INVALID_SACK_HOLE_INDEX ((u32)~0) |
| 155 | |
| 156 | typedef struct _sack_scoreboard_hole |
| 157 | { |
| 158 | u32 next; /**< Index for next entry in linked list */ |
| 159 | u32 prev; /**< Index for previous entry in linked list */ |
| 160 | u32 start; /**< Start sequence number */ |
| 161 | u32 end; /**< End sequence number */ |
Florin Coras | 93992a9 | 2017-05-24 18:03:56 -0700 | [diff] [blame] | 162 | u8 is_lost; /**< Mark hole as lost */ |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 163 | } sack_scoreboard_hole_t; |
| 164 | |
| 165 | typedef struct _sack_scoreboard |
| 166 | { |
| 167 | sack_scoreboard_hole_t *holes; /**< Pool of holes */ |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 168 | u32 head; /**< Index of first entry */ |
| 169 | u32 tail; /**< Index of last entry */ |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 170 | u32 sacked_bytes; /**< Number of bytes sacked in sb */ |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 171 | u32 last_sacked_bytes; /**< Number of bytes last sacked */ |
Florin Coras | 93992a9 | 2017-05-24 18:03:56 -0700 | [diff] [blame] | 172 | u32 last_bytes_delivered; /**< Number of sack bytes delivered */ |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 173 | u32 snd_una_adv; /**< Bytes to add to snd_una */ |
Florin Coras | 93992a9 | 2017-05-24 18:03:56 -0700 | [diff] [blame] | 174 | u32 high_sacked; /**< Highest byte sacked (fack) */ |
| 175 | u32 high_rxt; /**< Highest retransmitted sequence */ |
| 176 | u32 rescue_rxt; /**< Rescue sequence number */ |
| 177 | u32 lost_bytes; /**< Bytes lost as per RFC6675 */ |
| 178 | u32 cur_rxt_hole; /**< Retransmitting from this hole */ |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 179 | } sack_scoreboard_t; |
| 180 | |
| 181 | typedef enum _tcp_cc_algorithm_type |
| 182 | { |
| 183 | TCP_CC_NEWRENO, |
| 184 | } tcp_cc_algorithm_type_e; |
| 185 | |
| 186 | typedef struct _tcp_cc_algorithm tcp_cc_algorithm_t; |
| 187 | |
| 188 | typedef enum _tcp_cc_ack_t |
| 189 | { |
| 190 | TCP_CC_ACK, |
| 191 | TCP_CC_DUPACK, |
| 192 | TCP_CC_PARTIALACK |
| 193 | } tcp_cc_ack_t; |
| 194 | |
| 195 | typedef struct _tcp_connection |
| 196 | { |
| 197 | transport_connection_t connection; /**< Common transport data. First! */ |
| 198 | |
| 199 | u8 state; /**< TCP state as per tcp_state_t */ |
| 200 | u16 flags; /**< Connection flags (see tcp_conn_flags_e) */ |
| 201 | u32 timers[TCP_N_TIMERS]; /**< Timer handles into timer wheel */ |
| 202 | |
| 203 | /* TODO RFC4898 */ |
| 204 | |
| 205 | /** Send sequence variables RFC793 */ |
| 206 | u32 snd_una; /**< oldest unacknowledged sequence number */ |
| 207 | u32 snd_una_max; /**< newest unacknowledged sequence number + 1*/ |
| 208 | u32 snd_wnd; /**< send window */ |
| 209 | u32 snd_wl1; /**< seq number used for last snd.wnd update */ |
| 210 | u32 snd_wl2; /**< ack number used for last snd.wnd update */ |
| 211 | u32 snd_nxt; /**< next seq number to be sent */ |
Florin Coras | f03a59a | 2017-06-09 21:07:32 -0700 | [diff] [blame^] | 212 | u16 snd_mss; /**< Effective send max seg (data) size */ |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 213 | |
| 214 | /** Receive sequence variables RFC793 */ |
| 215 | u32 rcv_nxt; /**< next sequence number expected */ |
| 216 | u32 rcv_wnd; /**< receive window we expect */ |
| 217 | |
| 218 | u32 rcv_las; /**< rcv_nxt at last ack sent/rcv_wnd update */ |
| 219 | u32 iss; /**< initial sent sequence */ |
| 220 | u32 irs; /**< initial remote sequence */ |
| 221 | |
| 222 | /* Options */ |
Florin Coras | 93992a9 | 2017-05-24 18:03:56 -0700 | [diff] [blame] | 223 | tcp_options_t rcv_opts; /**< Rx options for connection */ |
Florin Coras | c834341 | 2017-05-04 14:25:50 -0700 | [diff] [blame] | 224 | tcp_options_t snd_opts; /**< Tx options for connection */ |
| 225 | u8 snd_opts_len; /**< Tx options len */ |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 226 | u8 rcv_wscale; /**< Window scale to advertise to peer */ |
| 227 | u8 snd_wscale; /**< Window scale to use when sending */ |
| 228 | u32 tsval_recent; /**< Last timestamp received */ |
| 229 | u32 tsval_recent_age; /**< When last updated tstamp_recent*/ |
| 230 | |
| 231 | sack_block_t *snd_sacks; /**< Vector of SACKs to send. XXX Fixed size? */ |
| 232 | sack_scoreboard_t sack_sb; /**< SACK "scoreboard" that tracks holes */ |
| 233 | |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 234 | u16 rcv_dupacks; /**< Number of DUPACKs received */ |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 235 | u8 snt_dupacks; /**< Number of DUPACKs sent in a burst */ |
| 236 | |
| 237 | /* Congestion control */ |
| 238 | u32 cwnd; /**< Congestion window */ |
| 239 | u32 ssthresh; /**< Slow-start threshold */ |
| 240 | u32 prev_ssthresh; /**< ssthresh before congestion */ |
Florin Coras | 93992a9 | 2017-05-24 18:03:56 -0700 | [diff] [blame] | 241 | u32 prev_cwnd; /**< ssthresh before congestion */ |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 242 | u32 bytes_acked; /**< Bytes acknowledged by current segment */ |
Florin Coras | 93992a9 | 2017-05-24 18:03:56 -0700 | [diff] [blame] | 243 | u32 snd_rxt_bytes; /**< Retransmitted bytes */ |
| 244 | u32 snd_rxt_ts; /**< Timestamp when first packet is retransmitted */ |
Florin Coras | e69f495 | 2017-03-07 10:06:24 -0800 | [diff] [blame] | 245 | u32 tsecr_last_ack; /**< Timestamp echoed to us in last healthy ACK */ |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 246 | u32 snd_congestion; /**< snd_una_max when congestion is detected */ |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 247 | tcp_cc_algorithm_t *cc_algo; /**< Congestion control algorithm */ |
| 248 | |
| 249 | /* RTT and RTO */ |
| 250 | u32 rto; /**< Retransmission timeout */ |
| 251 | u32 rto_boff; /**< Index for RTO backoff */ |
| 252 | u32 srtt; /**< Smoothed RTT */ |
| 253 | u32 rttvar; /**< Smoothed mean RTT difference. Approximates variance */ |
| 254 | u32 rtt_ts; /**< Timestamp for tracked ACK */ |
| 255 | u32 rtt_seq; /**< Sequence number for tracked ACK */ |
| 256 | |
Florin Coras | c834341 | 2017-05-04 14:25:50 -0700 | [diff] [blame] | 257 | u16 mss; /**< Our max seg size that includes options */ |
Florin Coras | f03a59a | 2017-06-09 21:07:32 -0700 | [diff] [blame^] | 258 | u32 limited_transmit; /**< snd_nxt when limited transmit starts */ |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 259 | } tcp_connection_t; |
| 260 | |
| 261 | struct _tcp_cc_algorithm |
| 262 | { |
| 263 | void (*rcv_ack) (tcp_connection_t * tc); |
| 264 | void (*rcv_cong_ack) (tcp_connection_t * tc, tcp_cc_ack_t ack); |
| 265 | void (*congestion) (tcp_connection_t * tc); |
| 266 | void (*recovered) (tcp_connection_t * tc); |
| 267 | void (*init) (tcp_connection_t * tc); |
| 268 | }; |
| 269 | |
| 270 | #define tcp_fastrecovery_on(tc) (tc)->flags |= TCP_CONN_FAST_RECOVERY |
| 271 | #define tcp_fastrecovery_off(tc) (tc)->flags &= ~TCP_CONN_FAST_RECOVERY |
Florin Coras | 3e350af | 2017-03-30 02:54:28 -0700 | [diff] [blame] | 272 | #define tcp_recovery_on(tc) (tc)->flags |= TCP_CONN_RECOVERY |
| 273 | #define tcp_recovery_off(tc) (tc)->flags &= ~TCP_CONN_RECOVERY |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 274 | #define tcp_in_fastrecovery(tc) ((tc)->flags & TCP_CONN_FAST_RECOVERY) |
Florin Coras | 3e350af | 2017-03-30 02:54:28 -0700 | [diff] [blame] | 275 | #define tcp_in_recovery(tc) ((tc)->flags & (TCP_CONN_RECOVERY)) |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 276 | #define tcp_in_slowstart(tc) (tc->cwnd < tc->ssthresh) |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 277 | #define tcp_fastrecovery_sent_1_smss(tc) ((tc)->flags & TCP_CONN_FR_1_SMSS) |
| 278 | #define tcp_fastrecovery_1_smss_on(tc) ((tc)->flags |= TCP_CONN_FR_1_SMSS) |
| 279 | #define tcp_fastrecovery_1_smss_off(tc) ((tc)->flags &= ~TCP_CONN_FR_1_SMSS) |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 280 | |
Florin Coras | 3e350af | 2017-03-30 02:54:28 -0700 | [diff] [blame] | 281 | #define tcp_in_cong_recovery(tc) ((tc)->flags & \ |
| 282 | (TCP_CONN_FAST_RECOVERY | TCP_CONN_RECOVERY)) |
| 283 | |
| 284 | always_inline void |
| 285 | tcp_cong_recovery_off (tcp_connection_t * tc) |
| 286 | { |
| 287 | tc->flags &= ~(TCP_CONN_FAST_RECOVERY | TCP_CONN_RECOVERY); |
| 288 | tcp_fastrecovery_1_smss_off (tc); |
| 289 | } |
| 290 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 291 | typedef enum |
| 292 | { |
| 293 | TCP_IP4, |
| 294 | TCP_IP6, |
| 295 | TCP_N_AF, |
| 296 | } tcp_af_t; |
| 297 | |
| 298 | typedef enum _tcp_error |
| 299 | { |
| 300 | #define tcp_error(n,s) TCP_ERROR_##n, |
| 301 | #include <vnet/tcp/tcp_error.def> |
| 302 | #undef tcp_error |
| 303 | TCP_N_ERROR, |
| 304 | } tcp_error_t; |
| 305 | |
| 306 | typedef struct _tcp_lookup_dispatch |
| 307 | { |
| 308 | u8 next, error; |
| 309 | } tcp_lookup_dispatch_t; |
| 310 | |
| 311 | typedef struct _tcp_main |
| 312 | { |
| 313 | /* Per-worker thread tcp connection pools */ |
| 314 | tcp_connection_t **connections; |
| 315 | |
| 316 | /* Pool of listeners. */ |
| 317 | tcp_connection_t *listener_pool; |
| 318 | |
| 319 | /** Dispatch table by state and flags */ |
| 320 | tcp_lookup_dispatch_t dispatch_table[TCP_N_STATES][64]; |
| 321 | |
| 322 | u8 log2_tstamp_clocks_per_tick; |
| 323 | f64 tstamp_ticks_per_clock; |
| 324 | |
| 325 | /** per-worker tx buffer free lists */ |
| 326 | u32 **tx_buffers; |
| 327 | |
| 328 | /* Per worker-thread timer wheel for connections timers */ |
| 329 | tw_timer_wheel_16t_2w_512sl_t *timer_wheels; |
| 330 | |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 331 | // /* Convenience per worker-thread vector of connections to DELACK */ |
| 332 | // u32 **delack_connections; |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 333 | |
| 334 | /* Pool of half-open connections on which we've sent a SYN */ |
| 335 | tcp_connection_t *half_open_connections; |
| 336 | |
| 337 | /* Pool of local TCP endpoints */ |
| 338 | transport_endpoint_t *local_endpoints; |
| 339 | |
| 340 | /* Local endpoints lookup table */ |
| 341 | transport_endpoint_table_t local_endpoints_table; |
| 342 | |
| 343 | /* Congestion control algorithms registered */ |
| 344 | tcp_cc_algorithm_t *cc_algos; |
| 345 | |
Florin Coras | a0b34a7 | 2017-03-07 01:20:52 -0800 | [diff] [blame] | 346 | /* Flag that indicates if stack is on or off */ |
| 347 | u8 is_enabled; |
| 348 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 349 | /* convenience */ |
| 350 | vlib_main_t *vlib_main; |
| 351 | vnet_main_t *vnet_main; |
| 352 | ip4_main_t *ip4_main; |
| 353 | ip6_main_t *ip6_main; |
| 354 | } tcp_main_t; |
| 355 | |
| 356 | extern tcp_main_t tcp_main; |
| 357 | extern vlib_node_registration_t tcp4_input_node; |
| 358 | extern vlib_node_registration_t tcp6_input_node; |
| 359 | extern vlib_node_registration_t tcp4_output_node; |
| 360 | extern vlib_node_registration_t tcp6_output_node; |
| 361 | |
| 362 | always_inline tcp_main_t * |
| 363 | vnet_get_tcp_main () |
| 364 | { |
| 365 | return &tcp_main; |
| 366 | } |
| 367 | |
Florin Coras | 82b13a8 | 2017-04-25 11:58:06 -0700 | [diff] [blame] | 368 | always_inline tcp_header_t * |
| 369 | tcp_buffer_hdr (vlib_buffer_t * b) |
| 370 | { |
| 371 | ASSERT ((signed) b->current_data >= (signed) -VLIB_BUFFER_PRE_DATA_SIZE); |
| 372 | return (tcp_header_t *) (b->data + b->current_data |
| 373 | + vnet_buffer (b)->tcp.hdr_offset); |
| 374 | } |
| 375 | |
Florin Coras | a0b34a7 | 2017-03-07 01:20:52 -0800 | [diff] [blame] | 376 | clib_error_t *vnet_tcp_enable_disable (vlib_main_t * vm, u8 is_en); |
| 377 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 378 | always_inline tcp_connection_t * |
| 379 | tcp_connection_get (u32 conn_index, u32 thread_index) |
| 380 | { |
Florin Coras | d79b41e | 2017-03-04 05:37:52 -0800 | [diff] [blame] | 381 | if (pool_is_free_index (tcp_main.connections[thread_index], conn_index)) |
| 382 | return 0; |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 383 | return pool_elt_at_index (tcp_main.connections[thread_index], conn_index); |
| 384 | } |
| 385 | |
| 386 | always_inline tcp_connection_t * |
| 387 | tcp_connection_get_if_valid (u32 conn_index, u32 thread_index) |
| 388 | { |
| 389 | if (tcp_main.connections[thread_index] == 0) |
| 390 | return 0; |
| 391 | if (pool_is_free_index (tcp_main.connections[thread_index], conn_index)) |
| 392 | return 0; |
| 393 | return pool_elt_at_index (tcp_main.connections[thread_index], conn_index); |
| 394 | } |
| 395 | |
| 396 | void tcp_connection_close (tcp_connection_t * tc); |
| 397 | void tcp_connection_cleanup (tcp_connection_t * tc); |
| 398 | void tcp_connection_del (tcp_connection_t * tc); |
Florin Coras | d79b41e | 2017-03-04 05:37:52 -0800 | [diff] [blame] | 399 | void tcp_connection_reset (tcp_connection_t * tc); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 400 | |
Florin Coras | bb292f4 | 2017-05-19 09:49:19 -0700 | [diff] [blame] | 401 | u8 *format_tcp_connection_id (u8 * s, va_list * args); |
Florin Coras | e69f495 | 2017-03-07 10:06:24 -0800 | [diff] [blame] | 402 | u8 *format_tcp_connection (u8 * s, va_list * args); |
Florin Coras | 06d1101 | 2017-05-17 14:21:51 -0700 | [diff] [blame] | 403 | u8 *format_tcp_scoreboard (u8 * s, va_list * args); |
Florin Coras | e69f495 | 2017-03-07 10:06:24 -0800 | [diff] [blame] | 404 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 405 | always_inline tcp_connection_t * |
| 406 | tcp_listener_get (u32 tli) |
| 407 | { |
| 408 | return pool_elt_at_index (tcp_main.listener_pool, tli); |
| 409 | } |
| 410 | |
| 411 | always_inline tcp_connection_t * |
| 412 | tcp_half_open_connection_get (u32 conn_index) |
| 413 | { |
| 414 | return pool_elt_at_index (tcp_main.half_open_connections, conn_index); |
| 415 | } |
| 416 | |
| 417 | void tcp_make_ack (tcp_connection_t * ts, vlib_buffer_t * b); |
Florin Coras | d79b41e | 2017-03-04 05:37:52 -0800 | [diff] [blame] | 418 | void tcp_make_fin (tcp_connection_t * tc, vlib_buffer_t * b); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 419 | void tcp_make_synack (tcp_connection_t * ts, vlib_buffer_t * b); |
| 420 | void tcp_send_reset (vlib_buffer_t * pkt, u8 is_ip4); |
| 421 | void tcp_send_syn (tcp_connection_t * tc); |
| 422 | void tcp_send_fin (tcp_connection_t * tc); |
Florin Coras | c834341 | 2017-05-04 14:25:50 -0700 | [diff] [blame] | 423 | void tcp_init_mss (tcp_connection_t * tc); |
| 424 | void tcp_update_snd_mss (tcp_connection_t * tc); |
Florin Coras | 93992a9 | 2017-05-24 18:03:56 -0700 | [diff] [blame] | 425 | void tcp_update_rto (tcp_connection_t * tc); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 426 | |
| 427 | always_inline u32 |
| 428 | tcp_end_seq (tcp_header_t * th, u32 len) |
| 429 | { |
| 430 | return th->seq_number + tcp_is_syn (th) + tcp_is_fin (th) + len; |
| 431 | } |
| 432 | |
| 433 | /* Modulo arithmetic for TCP sequence numbers */ |
| 434 | #define seq_lt(_s1, _s2) ((i32)((_s1)-(_s2)) < 0) |
| 435 | #define seq_leq(_s1, _s2) ((i32)((_s1)-(_s2)) <= 0) |
| 436 | #define seq_gt(_s1, _s2) ((i32)((_s1)-(_s2)) > 0) |
| 437 | #define seq_geq(_s1, _s2) ((i32)((_s1)-(_s2)) >= 0) |
Florin Coras | f03a59a | 2017-06-09 21:07:32 -0700 | [diff] [blame^] | 438 | #define seq_max(_s1, _s2) (seq_gt((_s1), (_s2)) ? (_s1) : (_s2)) |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 439 | |
| 440 | /* Modulo arithmetic for timestamps */ |
| 441 | #define timestamp_lt(_t1, _t2) ((i32)((_t1)-(_t2)) < 0) |
| 442 | #define timestamp_leq(_t1, _t2) ((i32)((_t1)-(_t2)) <= 0) |
| 443 | |
Florin Coras | 93992a9 | 2017-05-24 18:03:56 -0700 | [diff] [blame] | 444 | /** |
| 445 | * Our estimate of the number of bytes that have left the network |
| 446 | */ |
| 447 | always_inline u32 |
| 448 | tcp_bytes_out (const tcp_connection_t * tc) |
| 449 | { |
| 450 | if (tcp_opts_sack_permitted (&tc->rcv_opts)) |
| 451 | return tc->sack_sb.sacked_bytes + tc->sack_sb.lost_bytes; |
| 452 | else |
| 453 | return tc->rcv_dupacks * tc->snd_mss; |
| 454 | } |
| 455 | |
| 456 | /** |
| 457 | * Our estimate of the number of bytes in flight (pipe size) |
| 458 | */ |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 459 | always_inline u32 |
| 460 | tcp_flight_size (const tcp_connection_t * tc) |
| 461 | { |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 462 | int flight_size; |
| 463 | |
Florin Coras | 93992a9 | 2017-05-24 18:03:56 -0700 | [diff] [blame] | 464 | flight_size = (int) (tc->snd_una_max - tc->snd_una) - tcp_bytes_out (tc) |
| 465 | + tc->snd_rxt_bytes; |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 466 | |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 467 | if (flight_size < 0) |
Florin Coras | 93992a9 | 2017-05-24 18:03:56 -0700 | [diff] [blame] | 468 | { |
| 469 | if (0) |
| 470 | clib_warning |
| 471 | ("Negative: %u %u %u dupacks %u sacked bytes %u flags %d", |
| 472 | tc->snd_una_max - tc->snd_una, tcp_bytes_out (tc), |
| 473 | tc->snd_rxt_bytes, tc->rcv_dupacks, tc->sack_sb.sacked_bytes, |
| 474 | tc->rcv_opts.flags); |
| 475 | return 0; |
| 476 | } |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 477 | |
| 478 | return flight_size; |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 479 | } |
| 480 | |
| 481 | /** |
| 482 | * Initial cwnd as per RFC5681 |
| 483 | */ |
| 484 | always_inline u32 |
| 485 | tcp_initial_cwnd (const tcp_connection_t * tc) |
| 486 | { |
| 487 | if (tc->snd_mss > 2190) |
| 488 | return 2 * tc->snd_mss; |
| 489 | else if (tc->snd_mss > 1095) |
| 490 | return 3 * tc->snd_mss; |
| 491 | else |
| 492 | return 4 * tc->snd_mss; |
| 493 | } |
| 494 | |
| 495 | always_inline u32 |
| 496 | tcp_loss_wnd (const tcp_connection_t * tc) |
| 497 | { |
| 498 | return tc->snd_mss; |
| 499 | } |
| 500 | |
| 501 | always_inline u32 |
| 502 | tcp_available_wnd (const tcp_connection_t * tc) |
| 503 | { |
| 504 | return clib_min (tc->cwnd, tc->snd_wnd); |
| 505 | } |
| 506 | |
| 507 | always_inline u32 |
| 508 | tcp_available_snd_space (const tcp_connection_t * tc) |
| 509 | { |
| 510 | u32 available_wnd = tcp_available_wnd (tc); |
| 511 | u32 flight_size = tcp_flight_size (tc); |
| 512 | |
| 513 | if (available_wnd <= flight_size) |
| 514 | return 0; |
| 515 | |
| 516 | return available_wnd - flight_size; |
| 517 | } |
| 518 | |
Florin Coras | 93992a9 | 2017-05-24 18:03:56 -0700 | [diff] [blame] | 519 | i32 tcp_rcv_wnd_available (tcp_connection_t * tc); |
Florin Coras | bb292f4 | 2017-05-19 09:49:19 -0700 | [diff] [blame] | 520 | u32 tcp_snd_space (tcp_connection_t * tc); |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 521 | void tcp_update_rcv_wnd (tcp_connection_t * tc); |
| 522 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 523 | void tcp_retransmit_first_unacked (tcp_connection_t * tc); |
Florin Coras | 93992a9 | 2017-05-24 18:03:56 -0700 | [diff] [blame] | 524 | void tcp_fast_retransmit_no_sack (tcp_connection_t * tc); |
| 525 | void tcp_fast_retransmit_sack (tcp_connection_t * tc); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 526 | void tcp_fast_retransmit (tcp_connection_t * tc); |
Florin Coras | 93992a9 | 2017-05-24 18:03:56 -0700 | [diff] [blame] | 527 | void tcp_cc_init_congestion (tcp_connection_t * tc); |
| 528 | int tcp_cc_recover (tcp_connection_t * tc); |
| 529 | void tcp_cc_fastrecovery_exit (tcp_connection_t * tc); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 530 | |
Florin Coras | 45d3496 | 2017-04-25 00:05:27 -0700 | [diff] [blame] | 531 | /* Made public for unit testing only */ |
| 532 | void tcp_update_sack_list (tcp_connection_t * tc, u32 start, u32 end); |
| 533 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 534 | always_inline u32 |
| 535 | tcp_time_now (void) |
| 536 | { |
| 537 | return clib_cpu_time_now () * tcp_main.tstamp_ticks_per_clock; |
| 538 | } |
| 539 | |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 540 | always_inline void |
| 541 | tcp_update_time (f64 now, u32 thread_index) |
| 542 | { |
| 543 | tw_timer_expire_timers_16t_2w_512sl (&tcp_main.timer_wheels[thread_index], |
| 544 | now); |
| 545 | } |
| 546 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 547 | u32 tcp_push_header (transport_connection_t * tconn, vlib_buffer_t * b); |
| 548 | |
| 549 | u32 |
| 550 | tcp_prepare_retransmit_segment (tcp_connection_t * tc, vlib_buffer_t * b, |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 551 | u32 offset, u32 max_bytes); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 552 | |
| 553 | void tcp_connection_timers_init (tcp_connection_t * tc); |
| 554 | void tcp_connection_timers_reset (tcp_connection_t * tc); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 555 | void tcp_connection_init_vars (tcp_connection_t * tc); |
| 556 | |
| 557 | always_inline void |
| 558 | tcp_connection_force_ack (tcp_connection_t * tc, vlib_buffer_t * b) |
| 559 | { |
| 560 | /* Reset flags, make sure ack is sent */ |
| 561 | tc->flags = TCP_CONN_SNDACK; |
| 562 | vnet_buffer (b)->tcp.flags &= ~TCP_BUF_FLAG_DUPACK; |
| 563 | } |
| 564 | |
| 565 | always_inline void |
| 566 | tcp_timer_set (tcp_connection_t * tc, u8 timer_id, u32 interval) |
| 567 | { |
| 568 | tc->timers[timer_id] |
| 569 | = tw_timer_start_16t_2w_512sl (&tcp_main.timer_wheels[tc->c_thread_index], |
| 570 | tc->c_c_index, timer_id, interval); |
| 571 | } |
| 572 | |
| 573 | always_inline void |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 574 | tcp_timer_reset (tcp_connection_t * tc, u8 timer_id) |
| 575 | { |
| 576 | if (tc->timers[timer_id] == TCP_TIMER_HANDLE_INVALID) |
| 577 | return; |
| 578 | |
| 579 | tw_timer_stop_16t_2w_512sl (&tcp_main.timer_wheels[tc->c_thread_index], |
| 580 | tc->timers[timer_id]); |
| 581 | tc->timers[timer_id] = TCP_TIMER_HANDLE_INVALID; |
| 582 | } |
| 583 | |
| 584 | always_inline void |
| 585 | tcp_timer_update (tcp_connection_t * tc, u8 timer_id, u32 interval) |
| 586 | { |
| 587 | if (tc->timers[timer_id] != TCP_TIMER_HANDLE_INVALID) |
| 588 | tw_timer_stop_16t_2w_512sl (&tcp_main.timer_wheels[tc->c_thread_index], |
| 589 | tc->timers[timer_id]); |
| 590 | tc->timers[timer_id] = |
| 591 | tw_timer_start_16t_2w_512sl (&tcp_main.timer_wheels[tc->c_thread_index], |
| 592 | tc->c_c_index, timer_id, interval); |
| 593 | } |
| 594 | |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 595 | /* XXX Switch retransmit to faster TW */ |
| 596 | always_inline void |
| 597 | tcp_retransmit_timer_set (tcp_connection_t * tc) |
| 598 | { |
| 599 | tcp_timer_set (tc, TCP_TIMER_RETRANSMIT, |
| 600 | clib_max (tc->rto * TCP_TO_TIMER_TICK, 1)); |
| 601 | } |
| 602 | |
| 603 | always_inline void |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 604 | tcp_retransmit_timer_reset (tcp_connection_t * tc) |
| 605 | { |
| 606 | tcp_timer_reset (tc, TCP_TIMER_RETRANSMIT); |
| 607 | } |
| 608 | |
Florin Coras | 3e350af | 2017-03-30 02:54:28 -0700 | [diff] [blame] | 609 | always_inline void |
Florin Coras | 93992a9 | 2017-05-24 18:03:56 -0700 | [diff] [blame] | 610 | tcp_retransmit_timer_force_update (tcp_connection_t * tc) |
| 611 | { |
| 612 | tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, |
| 613 | clib_max (tc->rto * TCP_TO_TIMER_TICK, 1)); |
| 614 | } |
| 615 | |
| 616 | always_inline void |
Florin Coras | 3e350af | 2017-03-30 02:54:28 -0700 | [diff] [blame] | 617 | tcp_persist_timer_set (tcp_connection_t * tc) |
| 618 | { |
| 619 | /* Reuse RTO. It's backed off in handler */ |
| 620 | tcp_timer_set (tc, TCP_TIMER_PERSIST, |
Florin Coras | 5921f98 | 2017-04-03 18:00:00 -0700 | [diff] [blame] | 621 | clib_max (tc->rto * TCP_TO_TIMER_TICK, |
| 622 | TCP_TIMER_PERSIST_MIN)); |
Florin Coras | 3e350af | 2017-03-30 02:54:28 -0700 | [diff] [blame] | 623 | } |
| 624 | |
| 625 | always_inline void |
| 626 | tcp_persist_timer_update (tcp_connection_t * tc) |
| 627 | { |
| 628 | tcp_timer_update (tc, TCP_TIMER_PERSIST, |
Florin Coras | 5921f98 | 2017-04-03 18:00:00 -0700 | [diff] [blame] | 629 | clib_max (tc->rto * TCP_TO_TIMER_TICK, |
| 630 | TCP_TIMER_PERSIST_MIN)); |
Florin Coras | 3e350af | 2017-03-30 02:54:28 -0700 | [diff] [blame] | 631 | } |
| 632 | |
| 633 | always_inline void |
| 634 | tcp_persist_timer_reset (tcp_connection_t * tc) |
| 635 | { |
| 636 | tcp_timer_reset (tc, TCP_TIMER_PERSIST); |
| 637 | } |
| 638 | |
Florin Coras | 93992a9 | 2017-05-24 18:03:56 -0700 | [diff] [blame] | 639 | always_inline void |
| 640 | tcp_retransmit_timer_update (tcp_connection_t * tc) |
| 641 | { |
| 642 | if (tc->snd_una == tc->snd_una_max) |
| 643 | { |
| 644 | tcp_retransmit_timer_reset (tc); |
| 645 | if (tc->snd_wnd < tc->snd_mss) |
| 646 | tcp_persist_timer_set (tc); |
| 647 | } |
| 648 | else |
| 649 | tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, |
| 650 | clib_max (tc->rto * TCP_TO_TIMER_TICK, 1)); |
| 651 | } |
| 652 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 653 | always_inline u8 |
| 654 | tcp_timer_is_active (tcp_connection_t * tc, tcp_timers_e timer) |
| 655 | { |
| 656 | return tc->timers[timer] != TCP_TIMER_HANDLE_INVALID; |
| 657 | } |
| 658 | |
Florin Coras | 93992a9 | 2017-05-24 18:03:56 -0700 | [diff] [blame] | 659 | #define tcp_validate_txf_size(_tc, _a) \ |
| 660 | ASSERT(_tc->state != TCP_STATE_ESTABLISHED \ |
| 661 | || stream_session_tx_fifo_max_dequeue (&_tc->connection) >= _a) |
| 662 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 663 | void |
| 664 | scoreboard_remove_hole (sack_scoreboard_t * sb, |
| 665 | sack_scoreboard_hole_t * hole); |
Florin Coras | 93992a9 | 2017-05-24 18:03:56 -0700 | [diff] [blame] | 666 | void scoreboard_update_lost (tcp_connection_t * tc, sack_scoreboard_t * sb); |
| 667 | sack_scoreboard_hole_t *scoreboard_insert_hole (sack_scoreboard_t * sb, |
| 668 | u32 prev_index, u32 start, |
| 669 | u32 end); |
| 670 | sack_scoreboard_hole_t *scoreboard_next_rxt_hole (sack_scoreboard_t * sb, |
| 671 | sack_scoreboard_hole_t * |
| 672 | start, u8 have_sent_1_smss, |
| 673 | u8 * can_rescue, |
| 674 | u8 * snd_limited); |
| 675 | void scoreboard_init_high_rxt (sack_scoreboard_t * sb); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 676 | |
| 677 | always_inline sack_scoreboard_hole_t * |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 678 | scoreboard_get_hole (sack_scoreboard_t * sb, u32 index) |
| 679 | { |
| 680 | if (index != TCP_INVALID_SACK_HOLE_INDEX) |
| 681 | return pool_elt_at_index (sb->holes, index); |
| 682 | return 0; |
| 683 | } |
| 684 | |
| 685 | always_inline sack_scoreboard_hole_t * |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 686 | scoreboard_next_hole (sack_scoreboard_t * sb, sack_scoreboard_hole_t * hole) |
| 687 | { |
| 688 | if (hole->next != TCP_INVALID_SACK_HOLE_INDEX) |
| 689 | return pool_elt_at_index (sb->holes, hole->next); |
| 690 | return 0; |
| 691 | } |
| 692 | |
| 693 | always_inline sack_scoreboard_hole_t * |
Florin Coras | 93992a9 | 2017-05-24 18:03:56 -0700 | [diff] [blame] | 694 | scoreboard_prev_hole (sack_scoreboard_t * sb, sack_scoreboard_hole_t * hole) |
| 695 | { |
| 696 | if (hole->prev != TCP_INVALID_SACK_HOLE_INDEX) |
| 697 | return pool_elt_at_index (sb->holes, hole->prev); |
| 698 | return 0; |
| 699 | } |
| 700 | |
| 701 | always_inline sack_scoreboard_hole_t * |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 702 | scoreboard_first_hole (sack_scoreboard_t * sb) |
| 703 | { |
| 704 | if (sb->head != TCP_INVALID_SACK_HOLE_INDEX) |
| 705 | return pool_elt_at_index (sb->holes, sb->head); |
| 706 | return 0; |
| 707 | } |
| 708 | |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 709 | always_inline sack_scoreboard_hole_t * |
| 710 | scoreboard_last_hole (sack_scoreboard_t * sb) |
| 711 | { |
| 712 | if (sb->tail != TCP_INVALID_SACK_HOLE_INDEX) |
| 713 | return pool_elt_at_index (sb->holes, sb->tail); |
| 714 | return 0; |
| 715 | } |
| 716 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 717 | always_inline void |
| 718 | scoreboard_clear (sack_scoreboard_t * sb) |
| 719 | { |
Florin Coras | 93992a9 | 2017-05-24 18:03:56 -0700 | [diff] [blame] | 720 | sack_scoreboard_hole_t *hole; |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 721 | while ((hole = scoreboard_first_hole (sb))) |
| 722 | { |
| 723 | scoreboard_remove_hole (sb, hole); |
| 724 | } |
Florin Coras | f03a59a | 2017-06-09 21:07:32 -0700 | [diff] [blame^] | 725 | ASSERT (sb->head == sb->tail && sb->head == TCP_INVALID_SACK_HOLE_INDEX); |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 726 | sb->sacked_bytes = 0; |
| 727 | sb->last_sacked_bytes = 0; |
Florin Coras | 93992a9 | 2017-05-24 18:03:56 -0700 | [diff] [blame] | 728 | sb->last_bytes_delivered = 0; |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 729 | sb->snd_una_adv = 0; |
Florin Coras | 93992a9 | 2017-05-24 18:03:56 -0700 | [diff] [blame] | 730 | sb->high_sacked = 0; |
| 731 | sb->high_rxt = 0; |
| 732 | sb->lost_bytes = 0; |
| 733 | sb->cur_rxt_hole = TCP_INVALID_SACK_HOLE_INDEX; |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 734 | } |
| 735 | |
| 736 | always_inline u32 |
| 737 | scoreboard_hole_bytes (sack_scoreboard_hole_t * hole) |
| 738 | { |
| 739 | return hole->end - hole->start; |
| 740 | } |
| 741 | |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 742 | always_inline u32 |
| 743 | scoreboard_hole_index (sack_scoreboard_t * sb, sack_scoreboard_hole_t * hole) |
| 744 | { |
| 745 | return hole - sb->holes; |
| 746 | } |
| 747 | |
| 748 | always_inline void |
| 749 | scoreboard_init (sack_scoreboard_t * sb) |
| 750 | { |
| 751 | sb->head = TCP_INVALID_SACK_HOLE_INDEX; |
| 752 | sb->tail = TCP_INVALID_SACK_HOLE_INDEX; |
Florin Coras | 93992a9 | 2017-05-24 18:03:56 -0700 | [diff] [blame] | 753 | sb->cur_rxt_hole = TCP_INVALID_SACK_HOLE_INDEX; |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 754 | } |
| 755 | |
| 756 | void tcp_rcv_sacks (tcp_connection_t * tc, u32 ack); |
| 757 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 758 | always_inline void |
| 759 | tcp_cc_algo_register (tcp_cc_algorithm_type_e type, |
| 760 | const tcp_cc_algorithm_t * vft) |
| 761 | { |
| 762 | tcp_main_t *tm = vnet_get_tcp_main (); |
| 763 | vec_validate (tm->cc_algos, type); |
| 764 | |
| 765 | tm->cc_algos[type] = *vft; |
| 766 | } |
| 767 | |
| 768 | always_inline tcp_cc_algorithm_t * |
| 769 | tcp_cc_algo_get (tcp_cc_algorithm_type_e type) |
| 770 | { |
| 771 | tcp_main_t *tm = vnet_get_tcp_main (); |
| 772 | return &tm->cc_algos[type]; |
| 773 | } |
| 774 | |
| 775 | void tcp_cc_init (tcp_connection_t * tc); |
| 776 | |
| 777 | /** |
| 778 | * Push TCP header to buffer |
| 779 | * |
| 780 | * @param vm - vlib_main |
| 781 | * @param b - buffer to write the header to |
| 782 | * @param sp_net - source port net order |
| 783 | * @param dp_net - destination port net order |
| 784 | * @param seq - sequence number net order |
| 785 | * @param ack - ack number net order |
| 786 | * @param tcp_hdr_opts_len - header and options length in bytes |
| 787 | * @param flags - header flags |
| 788 | * @param wnd - window size |
| 789 | * |
| 790 | * @return - pointer to start of TCP header |
| 791 | */ |
| 792 | always_inline void * |
| 793 | vlib_buffer_push_tcp_net_order (vlib_buffer_t * b, u16 sp, u16 dp, u32 seq, |
| 794 | u32 ack, u8 tcp_hdr_opts_len, u8 flags, |
| 795 | u16 wnd) |
| 796 | { |
| 797 | tcp_header_t *th; |
| 798 | |
| 799 | th = vlib_buffer_push_uninit (b, tcp_hdr_opts_len); |
| 800 | |
| 801 | th->src_port = sp; |
| 802 | th->dst_port = dp; |
| 803 | th->seq_number = seq; |
| 804 | th->ack_number = ack; |
| 805 | th->data_offset_and_reserved = (tcp_hdr_opts_len >> 2) << 4; |
| 806 | th->flags = flags; |
| 807 | th->window = wnd; |
| 808 | th->checksum = 0; |
| 809 | th->urgent_pointer = 0; |
| 810 | return th; |
| 811 | } |
| 812 | |
| 813 | /** |
| 814 | * Push TCP header to buffer |
| 815 | * |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 816 | * @param b - buffer to write the header to |
| 817 | * @param sp_net - source port net order |
| 818 | * @param dp_net - destination port net order |
| 819 | * @param seq - sequence number host order |
| 820 | * @param ack - ack number host order |
| 821 | * @param tcp_hdr_opts_len - header and options length in bytes |
| 822 | * @param flags - header flags |
| 823 | * @param wnd - window size |
| 824 | * |
| 825 | * @return - pointer to start of TCP header |
| 826 | */ |
| 827 | always_inline void * |
| 828 | vlib_buffer_push_tcp (vlib_buffer_t * b, u16 sp_net, u16 dp_net, u32 seq, |
| 829 | u32 ack, u8 tcp_hdr_opts_len, u8 flags, u16 wnd) |
| 830 | { |
| 831 | return vlib_buffer_push_tcp_net_order (b, sp_net, dp_net, |
| 832 | clib_host_to_net_u32 (seq), |
| 833 | clib_host_to_net_u32 (ack), |
| 834 | tcp_hdr_opts_len, flags, |
| 835 | clib_host_to_net_u16 (wnd)); |
| 836 | } |
| 837 | |
| 838 | #endif /* _vnet_tcp_h_ */ |
| 839 | |
| 840 | /* |
| 841 | * fd.io coding-style-patch-verification: ON |
| 842 | * |
| 843 | * Local Variables: |
| 844 | * eval: (c-set-style "gnu") |
| 845 | * End: |
| 846 | */ |