Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 1 | /* |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 2 | * Copyright (c) 2017-2019 Cisco and/or its affiliates. |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 3 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | * you may not use this file except in compliance with the License. |
| 5 | * You may obtain a copy of the License at: |
| 6 | * |
| 7 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | * |
| 9 | * Unless required by applicable law or agreed to in writing, software |
| 10 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | * See the License for the specific language governing permissions and |
| 13 | * limitations under the License. |
| 14 | */ |
| 15 | #ifndef __included_session_h__ |
| 16 | #define __included_session_h__ |
| 17 | |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 18 | #include <vnet/session/session_types.h> |
Florin Coras | 04e5344 | 2017-07-16 17:12:15 -0700 | [diff] [blame] | 19 | #include <vnet/session/session_lookup.h> |
Florin Coras | 3e350af | 2017-03-30 02:54:28 -0700 | [diff] [blame] | 20 | #include <vnet/session/session_debug.h> |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 21 | #include <svm/message_queue.h> |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 22 | #include <svm/ssvm.h> |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 23 | |
Florin Coras | 3e350af | 2017-03-30 02:54:28 -0700 | [diff] [blame] | 24 | #define foreach_session_input_error \ |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 25 | _(NO_SESSION, "No session drops") \ |
| 26 | _(NO_LISTENER, "No listener for dst port drops") \ |
| 27 | _(ENQUEUED, "Packets pushed into rx fifo") \ |
| 28 | _(NOT_READY, "Session not ready packets") \ |
| 29 | _(FIFO_FULL, "Packets dropped for lack of rx fifo space") \ |
| 30 | _(EVENT_FIFO_FULL, "Events not sent for lack of event fifo space") \ |
| 31 | _(API_QUEUE_FULL, "Sessions not created for lack of API queue space") \ |
| 32 | _(NEW_SEG_NO_SPACE, "Created segment, couldn't allocate a fifo pair") \ |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 33 | _(NO_SPACE, "Couldn't allocate a fifo pair") \ |
| 34 | _(SEG_CREATE, "Couldn't create a new segment") |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 35 | |
| 36 | typedef enum |
| 37 | { |
| 38 | #define _(sym,str) SESSION_ERROR_##sym, |
| 39 | foreach_session_input_error |
| 40 | #undef _ |
| 41 | SESSION_N_ERROR, |
| 42 | } session_error_t; |
| 43 | |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 44 | typedef struct session_tx_context_ |
| 45 | { |
| 46 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 47 | session_t *s; |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 48 | transport_proto_vft_t *transport_vft; |
| 49 | transport_connection_t *tc; |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 50 | u32 max_dequeue; |
| 51 | u32 snd_space; |
| 52 | u32 left_to_snd; |
| 53 | u32 tx_offset; |
| 54 | u32 max_len_to_snd; |
| 55 | u16 deq_per_first_buf; |
| 56 | u16 deq_per_buf; |
| 57 | u16 snd_mss; |
Florin Coras | f08f26d | 2018-05-10 13:20:47 -0700 | [diff] [blame] | 58 | u16 n_segs_per_evt; |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 59 | u8 n_bufs_per_seg; |
| 60 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline1); |
| 61 | session_dgram_hdr_t hdr; |
| 62 | } session_tx_context_t; |
| 63 | |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 64 | typedef struct session_worker_ |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 65 | { |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 66 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 67 | |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 68 | /** Worker session pool */ |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 69 | session_t *sessions; |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 70 | |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 71 | /** vpp event message queue for worker */ |
| 72 | svm_msg_q_t *vpp_event_queue; |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 73 | |
Florin Coras | d67f112 | 2018-05-21 17:47:40 -0700 | [diff] [blame] | 74 | /** Our approximation of a "complete" dispatch loop period */ |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 75 | f64 dispatch_period; |
Florin Coras | d67f112 | 2018-05-21 17:47:40 -0700 | [diff] [blame] | 76 | |
| 77 | /** vlib_time_now last time around the track */ |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 78 | f64 last_vlib_time; |
Florin Coras | d67f112 | 2018-05-21 17:47:40 -0700 | [diff] [blame] | 79 | |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 80 | /** Per-proto vector of sessions to enqueue */ |
| 81 | u32 *session_to_enqueue[TRANSPORT_N_PROTO]; |
| 82 | |
| 83 | /** Context for session tx */ |
| 84 | session_tx_context_t ctx; |
| 85 | |
| 86 | /** Vector of tx buffer free lists */ |
| 87 | u32 *tx_buffers; |
| 88 | |
| 89 | /** Vector of partially read events */ |
| 90 | session_event_t *free_event_vector; |
| 91 | |
| 92 | /** Vector of active event vectors */ |
| 93 | session_event_t *pending_event_vector; |
| 94 | |
| 95 | /** Vector of postponed disconnects */ |
| 96 | session_event_t *pending_disconnects; |
| 97 | |
Florin Coras | 5f56d73 | 2018-10-30 10:21:59 -0700 | [diff] [blame] | 98 | /** Vector of postponed events */ |
| 99 | session_event_t *postponed_event_vector; |
| 100 | |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 101 | /** Peekers rw lock */ |
| 102 | clib_rwlock_t peekers_rw_locks; |
| 103 | |
Florin Coras | c44a558 | 2018-11-01 16:30:54 -0700 | [diff] [blame] | 104 | u32 last_tx_packets; |
| 105 | |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 106 | } session_worker_t; |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 107 | |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 108 | typedef int (session_fifo_rx_fn) (vlib_main_t * vm, |
| 109 | vlib_node_runtime_t * node, |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 110 | session_worker_t * wrk, |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 111 | session_event_t * e, int *n_tx_pkts); |
| 112 | |
| 113 | extern session_fifo_rx_fn session_tx_fifo_peek_and_snd; |
| 114 | extern session_fifo_rx_fn session_tx_fifo_dequeue_and_snd; |
| 115 | extern session_fifo_rx_fn session_tx_fifo_dequeue_internal; |
| 116 | |
| 117 | u8 session_node_lookup_fifo_event (svm_fifo_t * f, session_event_t * e); |
| 118 | |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 119 | typedef struct session_main_ |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 120 | { |
| 121 | /** Worker contexts */ |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 122 | session_worker_t *wrk; |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 123 | |
Florin Coras | b384b54 | 2018-01-15 01:08:33 -0800 | [diff] [blame] | 124 | /** Event queues memfd segment initialized only if so configured */ |
| 125 | ssvm_private_t evt_qs_segment; |
| 126 | |
Florin Coras | 93e6580 | 2017-11-29 00:07:11 -0500 | [diff] [blame] | 127 | /** Unique segment name counter */ |
| 128 | u32 unique_segment_name_counter; |
| 129 | |
| 130 | /** Per transport rx function that can either dequeue or peek */ |
Florin Coras | 561af9b | 2017-12-09 10:19:43 -0800 | [diff] [blame] | 131 | session_fifo_rx_fn **session_tx_fns; |
| 132 | |
| 133 | /** Per session type output nodes. Could optimize to group nodes by |
| 134 | * fib but lookup would then require session type parsing in session node. |
| 135 | * Trade memory for speed, for now */ |
| 136 | u32 *session_type_to_next; |
Florin Coras | 93e6580 | 2017-11-29 00:07:11 -0500 | [diff] [blame] | 137 | |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 138 | /* |
| 139 | * Config parameters |
| 140 | */ |
| 141 | |
Florin Coras | 93e6580 | 2017-11-29 00:07:11 -0500 | [diff] [blame] | 142 | /** Session manager is enabled */ |
| 143 | u8 is_enabled; |
| 144 | |
Dave Barach | 10d8cc6 | 2017-05-30 09:30:07 -0400 | [diff] [blame] | 145 | /** vpp fifo event queue configured length */ |
| 146 | u32 configured_event_queue_length; |
| 147 | |
Florin Coras | b384b54 | 2018-01-15 01:08:33 -0800 | [diff] [blame] | 148 | /** Session ssvm segment configs*/ |
| 149 | uword session_baseva; |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 150 | uword session_va_space_size; |
Florin Coras | b4c1491 | 2019-02-09 13:51:25 -0800 | [diff] [blame] | 151 | uword evt_qs_segment_size; |
Florin Coras | b384b54 | 2018-01-15 01:08:33 -0800 | [diff] [blame] | 152 | u8 evt_qs_use_memfd_seg; |
| 153 | |
| 154 | /** Session table size parameters */ |
Florin Coras | 66b1131 | 2017-07-31 17:18:03 -0700 | [diff] [blame] | 155 | u32 configured_v4_session_table_buckets; |
| 156 | u32 configured_v4_session_table_memory; |
| 157 | u32 configured_v4_halfopen_table_buckets; |
| 158 | u32 configured_v4_halfopen_table_memory; |
| 159 | u32 configured_v6_session_table_buckets; |
| 160 | u32 configured_v6_session_table_memory; |
| 161 | u32 configured_v6_halfopen_table_buckets; |
| 162 | u32 configured_v6_halfopen_table_memory; |
| 163 | |
Florin Coras | 93e6580 | 2017-11-29 00:07:11 -0500 | [diff] [blame] | 164 | /** Transport table (preallocation) size parameters */ |
| 165 | u32 local_endpoints_table_memory; |
| 166 | u32 local_endpoints_table_buckets; |
Florin Coras | e04c299 | 2017-03-01 08:17:34 -0800 | [diff] [blame] | 167 | |
Dave Barach | 2c25a62 | 2017-06-26 11:35:07 -0400 | [diff] [blame] | 168 | /** Preallocate session config parameter */ |
| 169 | u32 preallocated_sessions; |
| 170 | |
Florin Coras | c1a448b | 2018-04-20 10:51:49 -0700 | [diff] [blame] | 171 | #if SESSION_DEBUG |
Florin Coras | 3e350af | 2017-03-30 02:54:28 -0700 | [diff] [blame] | 172 | /** |
| 173 | * last event poll time by thread |
| 174 | * Debug only. Will cause false cache-line sharing as-is |
| 175 | */ |
| 176 | f64 *last_event_poll_by_thread; |
| 177 | #endif |
| 178 | |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 179 | } session_main_t; |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 180 | |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 181 | extern session_main_t session_main; |
Florin Coras | e04c299 | 2017-03-01 08:17:34 -0800 | [diff] [blame] | 182 | extern vlib_node_registration_t session_queue_node; |
Florin Coras | fd542f1 | 2018-05-16 19:28:24 -0700 | [diff] [blame] | 183 | extern vlib_node_registration_t session_queue_process_node; |
Florin Coras | 653e43f | 2019-03-04 10:56:23 -0800 | [diff] [blame] | 184 | extern vlib_node_registration_t session_queue_pre_input_node; |
Florin Coras | fd542f1 | 2018-05-16 19:28:24 -0700 | [diff] [blame] | 185 | |
| 186 | #define SESSION_Q_PROCESS_FLUSH_FRAMES 1 |
| 187 | #define SESSION_Q_PROCESS_STOP 2 |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 188 | |
Dave Barach | 2c25a62 | 2017-06-26 11:35:07 -0400 | [diff] [blame] | 189 | always_inline u8 |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 190 | session_is_valid (u32 si, u8 thread_index) |
Dave Barach | 2c25a62 | 2017-06-26 11:35:07 -0400 | [diff] [blame] | 191 | { |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 192 | session_t *s; |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 193 | s = pool_elt_at_index (session_main.wrk[thread_index].sessions, si); |
Florin Coras | 653e43f | 2019-03-04 10:56:23 -0800 | [diff] [blame] | 194 | if (s->session_state == SESSION_STATE_CLOSED) |
| 195 | return 1; |
| 196 | |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 197 | if (s->thread_index != thread_index || s->session_index != si) |
Dave Barach | 2c25a62 | 2017-06-26 11:35:07 -0400 | [diff] [blame] | 198 | return 0; |
| 199 | return 1; |
| 200 | } |
| 201 | |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 202 | session_t *session_alloc (u32 thread_index); |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 203 | void session_free (session_t * s); |
| 204 | void session_free_w_fifos (session_t * s); |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 205 | |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 206 | always_inline session_t * |
Florin Coras | cea194d | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 207 | session_get (u32 si, u32 thread_index) |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 208 | { |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 209 | ASSERT (session_is_valid (si, thread_index)); |
| 210 | return pool_elt_at_index (session_main.wrk[thread_index].sessions, si); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 211 | } |
| 212 | |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 213 | always_inline session_t * |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 214 | session_get_if_valid (u64 si, u32 thread_index) |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 215 | { |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 216 | if (thread_index >= vec_len (session_main.wrk)) |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 217 | return 0; |
| 218 | |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 219 | if (pool_is_free_index (session_main.wrk[thread_index].sessions, si)) |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 220 | return 0; |
| 221 | |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 222 | ASSERT (session_is_valid (si, thread_index)); |
| 223 | return pool_elt_at_index (session_main.wrk[thread_index].sessions, si); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 224 | } |
| 225 | |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 226 | always_inline session_t * |
Florin Coras | f8f516a | 2018-02-08 15:10:09 -0800 | [diff] [blame] | 227 | session_get_from_handle (session_handle_t handle) |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 228 | { |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 229 | session_main_t *smm = &session_main; |
Florin Coras | f8f516a | 2018-02-08 15:10:09 -0800 | [diff] [blame] | 230 | u32 session_index, thread_index; |
| 231 | session_parse_handle (handle, &session_index, &thread_index); |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 232 | return pool_elt_at_index (smm->wrk[thread_index].sessions, session_index); |
Florin Coras | f8f516a | 2018-02-08 15:10:09 -0800 | [diff] [blame] | 233 | } |
| 234 | |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 235 | always_inline session_t * |
Florin Coras | f8f516a | 2018-02-08 15:10:09 -0800 | [diff] [blame] | 236 | session_get_from_handle_if_valid (session_handle_t handle) |
| 237 | { |
| 238 | u32 session_index, thread_index; |
| 239 | session_parse_handle (handle, &session_index, &thread_index); |
| 240 | return session_get_if_valid (session_index, thread_index); |
| 241 | } |
| 242 | |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 243 | u64 session_segment_handle (session_t * s); |
Florin Coras | fa76a76 | 2018-11-29 12:40:10 -0800 | [diff] [blame] | 244 | |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 245 | /** |
| 246 | * Acquires a lock that blocks a session pool from expanding. |
| 247 | * |
| 248 | * This is typically used for safely peeking into other threads' |
| 249 | * pools in order to clone elements. Lock should be dropped as soon |
| 250 | * as possible by calling @ref session_pool_remove_peeker. |
| 251 | * |
| 252 | * NOTE: Avoid using pool_elt_at_index while the lock is held because |
| 253 | * it may lead to free elt bitmap expansion/contraction! |
| 254 | */ |
| 255 | always_inline void |
| 256 | session_pool_add_peeker (u32 thread_index) |
| 257 | { |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 258 | session_worker_t *wrk = &session_main.wrk[thread_index]; |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 259 | if (thread_index == vlib_get_thread_index ()) |
| 260 | return; |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 261 | clib_rwlock_reader_lock (&wrk->peekers_rw_locks); |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 262 | } |
| 263 | |
| 264 | always_inline void |
| 265 | session_pool_remove_peeker (u32 thread_index) |
| 266 | { |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 267 | session_worker_t *wrk = &session_main.wrk[thread_index]; |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 268 | if (thread_index == vlib_get_thread_index ()) |
| 269 | return; |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 270 | clib_rwlock_reader_unlock (&wrk->peekers_rw_locks); |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 271 | } |
| 272 | |
| 273 | /** |
| 274 | * Get session from handle and 'lock' pool resize if not in same thread |
| 275 | * |
| 276 | * Caller should drop the peek 'lock' as soon as possible. |
| 277 | */ |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 278 | always_inline session_t * |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 279 | session_get_from_handle_safe (u64 handle) |
| 280 | { |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 281 | u32 thread_index = session_thread_from_handle (handle); |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 282 | session_worker_t *wrk = &session_main.wrk[thread_index]; |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 283 | |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 284 | if (thread_index == vlib_get_thread_index ()) |
| 285 | { |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 286 | return pool_elt_at_index (wrk->sessions, |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 287 | session_index_from_handle (handle)); |
| 288 | } |
| 289 | else |
| 290 | { |
| 291 | session_pool_add_peeker (thread_index); |
| 292 | /* Don't use pool_elt_at index. See @ref session_pool_add_peeker */ |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 293 | return wrk->sessions + session_index_from_handle (handle); |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 294 | } |
| 295 | } |
| 296 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 297 | always_inline u32 |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 298 | session_get_index (session_t * s) |
| 299 | { |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 300 | return (s - session_main.wrk[s->thread_index].sessions); |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 301 | } |
| 302 | |
| 303 | always_inline session_t * |
| 304 | session_clone_safe (u32 session_index, u32 thread_index) |
| 305 | { |
| 306 | session_t *old_s, *new_s; |
| 307 | u32 current_thread_index = vlib_get_thread_index (); |
| 308 | |
| 309 | /* If during the memcpy pool is reallocated AND the memory allocator |
| 310 | * decides to give the old chunk of memory to somebody in a hurry to |
| 311 | * scribble something on it, we have a problem. So add this thread as |
| 312 | * a session pool peeker. |
| 313 | */ |
| 314 | session_pool_add_peeker (thread_index); |
| 315 | new_s = session_alloc (current_thread_index); |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 316 | old_s = session_main.wrk[thread_index].sessions + session_index; |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 317 | clib_memcpy_fast (new_s, old_s, sizeof (*new_s)); |
| 318 | session_pool_remove_peeker (thread_index); |
| 319 | new_s->thread_index = current_thread_index; |
| 320 | new_s->session_index = session_get_index (new_s); |
| 321 | return new_s; |
| 322 | } |
| 323 | |
| 324 | int session_open (u32 app_index, session_endpoint_t * tep, u32 opaque); |
| 325 | int session_listen (session_t * s, session_endpoint_cfg_t * sep); |
| 326 | int session_stop_listen (session_t * s); |
| 327 | void session_close (session_t * s); |
| 328 | void session_transport_close (session_t * s); |
| 329 | void session_transport_cleanup (session_t * s); |
| 330 | int session_send_io_evt_to_thread (svm_fifo_t * f, |
| 331 | session_evt_type_t evt_type); |
Florin Coras | 653e43f | 2019-03-04 10:56:23 -0800 | [diff] [blame] | 332 | int session_enqueue_notify (session_t * s); |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 333 | int session_dequeue_notify (session_t * s); |
| 334 | int session_send_io_evt_to_thread_custom (void *data, u32 thread_index, |
| 335 | session_evt_type_t evt_type); |
| 336 | void session_send_rpc_evt_to_thread (u32 thread_index, void *fp, |
| 337 | void *rpc_args); |
Nathan Skrzypczak | 60f3e65 | 2019-03-19 13:57:31 +0100 | [diff] [blame] | 338 | void session_send_rpc_evt_to_thread_force (u32 thread_index, void *fp, |
| 339 | void *rpc_args); |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 340 | transport_connection_t *session_get_transport (session_t * s); |
Florin Coras | 09d18c2 | 2019-04-24 11:10:02 -0700 | [diff] [blame] | 341 | void session_get_endpoint (session_t * s, transport_endpoint_t * tep, |
| 342 | u8 is_lcl); |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 343 | |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 344 | u8 *format_session (u8 * s, va_list * args); |
| 345 | uword unformat_session (unformat_input_t * input, va_list * args); |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 346 | uword unformat_transport_connection (unformat_input_t * input, |
| 347 | va_list * args); |
| 348 | |
| 349 | /* |
| 350 | * Interface to transport protos |
| 351 | */ |
| 352 | |
| 353 | int session_enqueue_stream_connection (transport_connection_t * tc, |
| 354 | vlib_buffer_t * b, u32 offset, |
| 355 | u8 queue_event, u8 is_in_order); |
| 356 | int session_enqueue_dgram_connection (session_t * s, |
| 357 | session_dgram_hdr_t * hdr, |
| 358 | vlib_buffer_t * b, u8 proto, |
| 359 | u8 queue_event); |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 360 | int session_stream_connect_notify (transport_connection_t * tc, u8 is_fail); |
| 361 | int session_dgram_connect_notify (transport_connection_t * tc, |
| 362 | u32 old_thread_index, |
| 363 | session_t ** new_session); |
Florin Coras | a27a46e | 2019-02-18 13:02:28 -0800 | [diff] [blame] | 364 | int session_stream_accept_notify (transport_connection_t * tc); |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 365 | void session_transport_closing_notify (transport_connection_t * tc); |
| 366 | void session_transport_delete_notify (transport_connection_t * tc); |
| 367 | void session_transport_closed_notify (transport_connection_t * tc); |
| 368 | void session_transport_reset_notify (transport_connection_t * tc); |
Florin Coras | c9940fc | 2019-02-05 20:55:11 -0800 | [diff] [blame] | 369 | int session_stream_accept (transport_connection_t * tc, u32 listener_index, |
Nathan Skrzypczak | 2f0f96b | 2019-06-13 10:14:28 +0200 | [diff] [blame] | 370 | u32 thread_index, u8 notify); |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 371 | void session_register_transport (transport_proto_t transport_proto, |
| 372 | const transport_proto_vft_t * vft, u8 is_ip4, |
| 373 | u32 output_node); |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 374 | int session_tx_fifo_peek_bytes (transport_connection_t * tc, u8 * buffer, |
| 375 | u32 offset, u32 max_bytes); |
| 376 | u32 session_tx_fifo_dequeue_drop (transport_connection_t * tc, u32 max_bytes); |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 377 | |
| 378 | always_inline u32 |
Florin Coras | d2aab83 | 2018-05-22 11:39:59 -0700 | [diff] [blame] | 379 | transport_max_rx_enqueue (transport_connection_t * tc) |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 380 | { |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 381 | session_t *s = session_get (tc->s_index, tc->thread_index); |
Sirshak Das | 28aa539 | 2019-02-05 01:33:33 -0600 | [diff] [blame] | 382 | return svm_fifo_max_enqueue_prod (s->rx_fifo); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 383 | } |
| 384 | |
Florin Coras | e04c299 | 2017-03-01 08:17:34 -0800 | [diff] [blame] | 385 | always_inline u32 |
Florin Coras | 42ceddb | 2018-12-12 10:56:01 -0800 | [diff] [blame] | 386 | transport_max_tx_dequeue (transport_connection_t * tc) |
| 387 | { |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 388 | session_t *s = session_get (tc->s_index, tc->thread_index); |
Sirshak Das | 28aa539 | 2019-02-05 01:33:33 -0600 | [diff] [blame] | 389 | return svm_fifo_max_dequeue_cons (s->tx_fifo); |
Florin Coras | 42ceddb | 2018-12-12 10:56:01 -0800 | [diff] [blame] | 390 | } |
| 391 | |
| 392 | always_inline u32 |
Florin Coras | f65074e | 2019-03-31 17:17:11 -0700 | [diff] [blame] | 393 | transport_max_rx_dequeue (transport_connection_t * tc) |
| 394 | { |
| 395 | session_t *s = session_get (tc->s_index, tc->thread_index); |
| 396 | return svm_fifo_max_dequeue (s->rx_fifo); |
| 397 | } |
| 398 | |
| 399 | always_inline u32 |
Florin Coras | d2aab83 | 2018-05-22 11:39:59 -0700 | [diff] [blame] | 400 | transport_rx_fifo_size (transport_connection_t * tc) |
Florin Coras | e04c299 | 2017-03-01 08:17:34 -0800 | [diff] [blame] | 401 | { |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 402 | session_t *s = session_get (tc->s_index, tc->thread_index); |
| 403 | return s->rx_fifo->nitems; |
Florin Coras | e04c299 | 2017-03-01 08:17:34 -0800 | [diff] [blame] | 404 | } |
| 405 | |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 406 | always_inline u32 |
Florin Coras | d2aab83 | 2018-05-22 11:39:59 -0700 | [diff] [blame] | 407 | transport_tx_fifo_size (transport_connection_t * tc) |
| 408 | { |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 409 | session_t *s = session_get (tc->s_index, tc->thread_index); |
| 410 | return s->tx_fifo->nitems; |
Florin Coras | d2aab83 | 2018-05-22 11:39:59 -0700 | [diff] [blame] | 411 | } |
| 412 | |
Florin Coras | 7ac053b | 2018-11-05 15:57:21 -0800 | [diff] [blame] | 413 | always_inline u8 |
| 414 | transport_rx_fifo_has_ooo_data (transport_connection_t * tc) |
| 415 | { |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 416 | session_t *s = session_get (tc->c_index, tc->thread_index); |
| 417 | return svm_fifo_has_ooo_data (s->rx_fifo); |
Florin Coras | 7ac053b | 2018-11-05 15:57:21 -0800 | [diff] [blame] | 418 | } |
| 419 | |
Florin Coras | d67f112 | 2018-05-21 17:47:40 -0700 | [diff] [blame] | 420 | always_inline f64 |
| 421 | transport_dispatch_period (u32 thread_index) |
| 422 | { |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 423 | return session_main.wrk[thread_index].dispatch_period; |
Florin Coras | d67f112 | 2018-05-21 17:47:40 -0700 | [diff] [blame] | 424 | } |
| 425 | |
| 426 | always_inline f64 |
| 427 | transport_time_now (u32 thread_index) |
| 428 | { |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 429 | return session_main.wrk[thread_index].last_vlib_time; |
Florin Coras | d67f112 | 2018-05-21 17:47:40 -0700 | [diff] [blame] | 430 | } |
| 431 | |
Florin Coras | 3ec66b0 | 2018-08-23 16:27:05 -0700 | [diff] [blame] | 432 | always_inline void |
| 433 | transport_add_tx_event (transport_connection_t * tc) |
| 434 | { |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 435 | session_t *s = session_get (tc->s_index, tc->thread_index); |
| 436 | if (svm_fifo_has_event (s->tx_fifo)) |
Florin Coras | 3ec66b0 | 2018-08-23 16:27:05 -0700 | [diff] [blame] | 437 | return; |
Florin Coras | f6c4313 | 2019-03-01 12:41:21 -0800 | [diff] [blame] | 438 | session_send_io_evt_to_thread (s->tx_fifo, SESSION_IO_EVT_TX); |
Florin Coras | 3ec66b0 | 2018-08-23 16:27:05 -0700 | [diff] [blame] | 439 | } |
| 440 | |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 441 | /* |
| 442 | * Listen sessions |
| 443 | */ |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 444 | |
| 445 | always_inline u64 |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 446 | listen_session_get_handle (session_t * s) |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 447 | { |
Aloys Augustin | cdb7170 | 2019-04-08 17:54:39 +0200 | [diff] [blame] | 448 | ASSERT (s->session_state == SESSION_STATE_LISTENING || |
| 449 | session_get_transport_proto (s) == TRANSPORT_PROTO_QUIC); |
Florin Coras | 5c9083d | 2018-04-13 06:39:07 -0700 | [diff] [blame] | 450 | return session_handle (s); |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 451 | } |
| 452 | |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 453 | always_inline session_t * |
Florin Coras | f8f516a | 2018-02-08 15:10:09 -0800 | [diff] [blame] | 454 | listen_session_get_from_handle (session_handle_t handle) |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 455 | { |
Florin Coras | 5c9083d | 2018-04-13 06:39:07 -0700 | [diff] [blame] | 456 | return session_get_from_handle (handle); |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 457 | } |
| 458 | |
Florin Coras | 371ca50 | 2018-02-21 12:07:41 -0800 | [diff] [blame] | 459 | always_inline void |
Florin Coras | 5c9083d | 2018-04-13 06:39:07 -0700 | [diff] [blame] | 460 | listen_session_parse_handle (session_handle_t handle, u32 * index, |
| 461 | u32 * thread_index) |
Florin Coras | 371ca50 | 2018-02-21 12:07:41 -0800 | [diff] [blame] | 462 | { |
Florin Coras | 5c9083d | 2018-04-13 06:39:07 -0700 | [diff] [blame] | 463 | session_parse_handle (handle, index, thread_index); |
Florin Coras | 371ca50 | 2018-02-21 12:07:41 -0800 | [diff] [blame] | 464 | } |
| 465 | |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 466 | always_inline session_t * |
Florin Coras | ba7d8f5 | 2019-02-22 13:11:38 -0800 | [diff] [blame] | 467 | listen_session_alloc (u8 thread_index, session_type_t type) |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 468 | { |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 469 | session_t *s; |
Florin Coras | 5c9083d | 2018-04-13 06:39:07 -0700 | [diff] [blame] | 470 | s = session_alloc (thread_index); |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 471 | s->session_type = type; |
| 472 | s->session_state = SESSION_STATE_LISTENING; |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 473 | return s; |
| 474 | } |
| 475 | |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 476 | always_inline session_t * |
Florin Coras | ba7d8f5 | 2019-02-22 13:11:38 -0800 | [diff] [blame] | 477 | listen_session_get (u32 ls_index) |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 478 | { |
Florin Coras | ba7d8f5 | 2019-02-22 13:11:38 -0800 | [diff] [blame] | 479 | return session_get (ls_index, 0); |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 480 | } |
| 481 | |
| 482 | always_inline void |
Florin Coras | ba7d8f5 | 2019-02-22 13:11:38 -0800 | [diff] [blame] | 483 | listen_session_free (session_t * s) |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 484 | { |
Florin Coras | 5c9083d | 2018-04-13 06:39:07 -0700 | [diff] [blame] | 485 | session_free (s); |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 486 | } |
| 487 | |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 488 | transport_connection_t *listen_session_get_transport (session_t * s); |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 489 | |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 490 | /* |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 491 | * Session layer functions |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 492 | */ |
| 493 | |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 494 | always_inline session_main_t * |
| 495 | vnet_get_session_main () |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 496 | { |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 497 | return &session_main; |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 498 | } |
| 499 | |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 500 | always_inline session_worker_t * |
| 501 | session_main_get_worker (u32 thread_index) |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 502 | { |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 503 | return &session_main.wrk[thread_index]; |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 504 | } |
| 505 | |
| 506 | always_inline svm_msg_q_t * |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 507 | session_main_get_vpp_event_queue (u32 thread_index) |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 508 | { |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 509 | return session_main.wrk[thread_index].vpp_event_queue; |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 510 | } |
Florin Coras | fd542f1 | 2018-05-16 19:28:24 -0700 | [diff] [blame] | 511 | |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 512 | always_inline u8 |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 513 | session_main_is_enabled () |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 514 | { |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 515 | return session_main.is_enabled == 1; |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 516 | } |
| 517 | |
Florin Coras | cea194d | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 518 | #define session_cli_return_if_not_enabled() \ |
| 519 | do { \ |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 520 | if (!session_main.is_enabled) \ |
Florin Coras | cea194d | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 521 | return clib_error_return(0, "session layer is not enabled"); \ |
| 522 | } while (0) |
| 523 | |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 524 | int session_main_flush_enqueue_events (u8 proto, u32 thread_index); |
| 525 | int session_main_flush_all_enqueue_events (u8 transport_proto); |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 526 | void session_flush_frames_main_thread (vlib_main_t * vm); |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 527 | ssvm_private_t *session_main_get_evt_q_segment (void); |
Dave Barach | 2a86391 | 2017-11-28 10:11:42 -0500 | [diff] [blame] | 528 | void session_node_enable_disable (u8 is_en); |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 529 | clib_error_t *vnet_session_enable_disable (vlib_main_t * vm, u8 is_en); |
Dave Barach | 2a86391 | 2017-11-28 10:11:42 -0500 | [diff] [blame] | 530 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 531 | #endif /* __included_session_h__ */ |
| 532 | |
| 533 | /* |
| 534 | * fd.io coding-style-patch-verification: ON |
| 535 | * |
| 536 | * Local Variables: |
| 537 | * eval: (c-set-style "gnu") |
| 538 | * End: |
| 539 | */ |