Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2017 Cisco and/or its affiliates. |
| 3 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | * you may not use this file except in compliance with the License. |
| 5 | * You may obtain a copy of the License at: |
| 6 | * |
| 7 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | * |
| 9 | * Unless required by applicable law or agreed to in writing, software |
| 10 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | * See the License for the specific language governing permissions and |
| 13 | * limitations under the License. |
| 14 | */ |
| 15 | #ifndef __included_session_h__ |
| 16 | #define __included_session_h__ |
| 17 | |
Florin Coras | 04e5344 | 2017-07-16 17:12:15 -0700 | [diff] [blame] | 18 | #include <vnet/session/stream_session.h> |
| 19 | #include <vnet/session/session_lookup.h> |
| 20 | #include <vnet/session/transport_interface.h> |
Florin Coras | 3e350af | 2017-03-30 02:54:28 -0700 | [diff] [blame] | 21 | #include <vnet/session/session_debug.h> |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 22 | #include <vnet/session/segment_manager.h> |
Florin Coras | e86a8ed | 2018-01-05 03:20:25 -0800 | [diff] [blame] | 23 | #include <svm/queue.h> |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 24 | |
| 25 | #define HALF_OPEN_LOOKUP_INVALID_VALUE ((u64)~0) |
| 26 | #define INVALID_INDEX ((u32)~0) |
Florin Coras | c3ddea8 | 2017-11-27 03:12:00 -0800 | [diff] [blame] | 27 | #define SESSION_PROXY_LISTENER_INDEX ((u32)~0 - 1) |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 28 | |
| 29 | /* TODO decide how much since we have pre-data as well */ |
| 30 | #define MAX_HDRS_LEN 100 /* Max number of bytes for headers */ |
| 31 | |
| 32 | typedef enum |
| 33 | { |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 34 | FIFO_EVENT_APP_RX, |
| 35 | FIFO_EVENT_APP_TX, |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 36 | FIFO_EVENT_TIMEOUT, |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 37 | FIFO_EVENT_DISCONNECT, |
Dave Barach | e5f1d27 | 2017-05-10 13:34:04 -0400 | [diff] [blame] | 38 | FIFO_EVENT_BUILTIN_RX, |
| 39 | FIFO_EVENT_RPC, |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 40 | } fifo_event_type_t; |
| 41 | |
Dave Wallace | 33e002b | 2017-09-06 01:20:02 -0400 | [diff] [blame] | 42 | static inline const char * |
| 43 | fifo_event_type_str (fifo_event_type_t et) |
| 44 | { |
| 45 | switch (et) |
| 46 | { |
| 47 | case FIFO_EVENT_APP_RX: |
| 48 | return "FIFO_EVENT_APP_RX"; |
| 49 | case FIFO_EVENT_APP_TX: |
| 50 | return "FIFO_EVENT_APP_TX"; |
| 51 | case FIFO_EVENT_TIMEOUT: |
| 52 | return "FIFO_EVENT_TIMEOUT"; |
| 53 | case FIFO_EVENT_DISCONNECT: |
| 54 | return "FIFO_EVENT_DISCONNECT"; |
| 55 | case FIFO_EVENT_BUILTIN_RX: |
| 56 | return "FIFO_EVENT_BUILTIN_RX"; |
| 57 | case FIFO_EVENT_RPC: |
| 58 | return "FIFO_EVENT_RPC"; |
| 59 | default: |
| 60 | return "UNKNOWN FIFO EVENT"; |
| 61 | } |
| 62 | } |
| 63 | |
Florin Coras | 3e350af | 2017-03-30 02:54:28 -0700 | [diff] [blame] | 64 | #define foreach_session_input_error \ |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 65 | _(NO_SESSION, "No session drops") \ |
| 66 | _(NO_LISTENER, "No listener for dst port drops") \ |
| 67 | _(ENQUEUED, "Packets pushed into rx fifo") \ |
| 68 | _(NOT_READY, "Session not ready packets") \ |
| 69 | _(FIFO_FULL, "Packets dropped for lack of rx fifo space") \ |
| 70 | _(EVENT_FIFO_FULL, "Events not sent for lack of event fifo space") \ |
| 71 | _(API_QUEUE_FULL, "Sessions not created for lack of API queue space") \ |
| 72 | _(NEW_SEG_NO_SPACE, "Created segment, couldn't allocate a fifo pair") \ |
| 73 | _(NO_SPACE, "Couldn't allocate a fifo pair") |
| 74 | |
| 75 | typedef enum |
| 76 | { |
| 77 | #define _(sym,str) SESSION_ERROR_##sym, |
| 78 | foreach_session_input_error |
| 79 | #undef _ |
| 80 | SESSION_N_ERROR, |
| 81 | } session_error_t; |
| 82 | |
Dave Barach | e5f1d27 | 2017-05-10 13:34:04 -0400 | [diff] [blame] | 83 | typedef struct |
| 84 | { |
| 85 | void *fp; |
| 86 | void *arg; |
| 87 | } rpc_args_t; |
| 88 | |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 89 | /* *INDENT-OFF* */ |
| 90 | typedef CLIB_PACKED (struct { |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 91 | union |
| 92 | { |
| 93 | svm_fifo_t * fifo; |
| 94 | u64 session_handle; |
Dave Barach | e5f1d27 | 2017-05-10 13:34:04 -0400 | [diff] [blame] | 95 | rpc_args_t rpc_args; |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 96 | }; |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 97 | u8 event_type; |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 98 | u8 postponed; |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 99 | }) session_fifo_event_t; |
| 100 | /* *INDENT-ON* */ |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 101 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 102 | /* Forward definition */ |
| 103 | typedef struct _session_manager_main session_manager_main_t; |
| 104 | |
| 105 | typedef int |
| 106 | (session_fifo_rx_fn) (vlib_main_t * vm, vlib_node_runtime_t * node, |
| 107 | session_manager_main_t * smm, |
| 108 | session_fifo_event_t * e0, stream_session_t * s0, |
| 109 | u32 thread_index, int *n_tx_pkts); |
| 110 | |
Florin Coras | d79b41e | 2017-03-04 05:37:52 -0800 | [diff] [blame] | 111 | extern session_fifo_rx_fn session_tx_fifo_peek_and_snd; |
| 112 | extern session_fifo_rx_fn session_tx_fifo_dequeue_and_snd; |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 113 | |
Florin Coras | 6534b7a | 2017-07-18 05:38:03 -0400 | [diff] [blame] | 114 | u8 session_node_lookup_fifo_event (svm_fifo_t * f, session_fifo_event_t * e); |
| 115 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 116 | struct _session_manager_main |
| 117 | { |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 118 | /** Per worker thread session pools */ |
| 119 | stream_session_t **sessions; |
| 120 | |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 121 | /** Per worker-thread count of threads peeking into the session pool */ |
| 122 | u32 *session_peekers; |
| 123 | |
| 124 | /** Per worker-thread rw peekers locks */ |
| 125 | clib_spinlock_t *peekers_readers_locks; |
| 126 | clib_spinlock_t *peekers_write_locks; |
| 127 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 128 | /** Pool of listen sessions. Same type as stream sessions to ease lookups */ |
Florin Coras | 561af9b | 2017-12-09 10:19:43 -0800 | [diff] [blame] | 129 | stream_session_t **listen_sessions; |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 130 | |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 131 | /** Per-proto, per-worker enqueue epoch counters */ |
Florin Coras | 93e6580 | 2017-11-29 00:07:11 -0500 | [diff] [blame] | 132 | u32 *current_enqueue_epoch[TRANSPORT_N_PROTO]; |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 133 | |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 134 | /** Per-proto, per-worker thread vector of sessions to enqueue */ |
| 135 | u32 **session_to_enqueue[TRANSPORT_N_PROTO]; |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 136 | |
| 137 | /** per-worker tx buffer free lists */ |
| 138 | u32 **tx_buffers; |
| 139 | |
| 140 | /** Per worker-thread vector of partially read events */ |
Dave Barach | acd2a6a | 2017-05-16 17:41:34 -0400 | [diff] [blame] | 141 | session_fifo_event_t **free_event_vector; |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 142 | |
| 143 | /** per-worker active event vectors */ |
Dave Barach | acd2a6a | 2017-05-16 17:41:34 -0400 | [diff] [blame] | 144 | session_fifo_event_t **pending_event_vector; |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 145 | |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 146 | /** per-worker postponed disconnects */ |
| 147 | session_fifo_event_t **pending_disconnects; |
| 148 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 149 | /** vpp fifo event queue */ |
Florin Coras | e86a8ed | 2018-01-05 03:20:25 -0800 | [diff] [blame] | 150 | svm_queue_t **vpp_event_queues; |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 151 | |
Florin Coras | b384b54 | 2018-01-15 01:08:33 -0800 | [diff] [blame] | 152 | /** Event queues memfd segment initialized only if so configured */ |
| 153 | ssvm_private_t evt_qs_segment; |
| 154 | |
Florin Coras | 93e6580 | 2017-11-29 00:07:11 -0500 | [diff] [blame] | 155 | /** Unique segment name counter */ |
| 156 | u32 unique_segment_name_counter; |
| 157 | |
| 158 | /** Per transport rx function that can either dequeue or peek */ |
Florin Coras | 561af9b | 2017-12-09 10:19:43 -0800 | [diff] [blame] | 159 | session_fifo_rx_fn **session_tx_fns; |
| 160 | |
| 161 | /** Per session type output nodes. Could optimize to group nodes by |
| 162 | * fib but lookup would then require session type parsing in session node. |
| 163 | * Trade memory for speed, for now */ |
| 164 | u32 *session_type_to_next; |
Florin Coras | 93e6580 | 2017-11-29 00:07:11 -0500 | [diff] [blame] | 165 | |
| 166 | /** Session manager is enabled */ |
| 167 | u8 is_enabled; |
| 168 | |
Dave Barach | 10d8cc6 | 2017-05-30 09:30:07 -0400 | [diff] [blame] | 169 | /** vpp fifo event queue configured length */ |
| 170 | u32 configured_event_queue_length; |
| 171 | |
Florin Coras | 93e6580 | 2017-11-29 00:07:11 -0500 | [diff] [blame] | 172 | /* |
| 173 | * Config parameters |
| 174 | */ |
| 175 | |
Florin Coras | b384b54 | 2018-01-15 01:08:33 -0800 | [diff] [blame] | 176 | /** Session ssvm segment configs*/ |
| 177 | uword session_baseva; |
| 178 | u32 segment_timeout; |
| 179 | u32 evt_qs_segment_size; |
| 180 | u8 evt_qs_use_memfd_seg; |
| 181 | |
| 182 | /** Session table size parameters */ |
Florin Coras | 66b1131 | 2017-07-31 17:18:03 -0700 | [diff] [blame] | 183 | u32 configured_v4_session_table_buckets; |
| 184 | u32 configured_v4_session_table_memory; |
| 185 | u32 configured_v4_halfopen_table_buckets; |
| 186 | u32 configured_v4_halfopen_table_memory; |
| 187 | u32 configured_v6_session_table_buckets; |
| 188 | u32 configured_v6_session_table_memory; |
| 189 | u32 configured_v6_halfopen_table_buckets; |
| 190 | u32 configured_v6_halfopen_table_memory; |
| 191 | |
Florin Coras | 93e6580 | 2017-11-29 00:07:11 -0500 | [diff] [blame] | 192 | /** Transport table (preallocation) size parameters */ |
| 193 | u32 local_endpoints_table_memory; |
| 194 | u32 local_endpoints_table_buckets; |
Florin Coras | e04c299 | 2017-03-01 08:17:34 -0800 | [diff] [blame] | 195 | |
Dave Barach | 2c25a62 | 2017-06-26 11:35:07 -0400 | [diff] [blame] | 196 | /** Preallocate session config parameter */ |
| 197 | u32 preallocated_sessions; |
| 198 | |
Florin Coras | 3e350af | 2017-03-30 02:54:28 -0700 | [diff] [blame] | 199 | #if SESSION_DBG |
| 200 | /** |
| 201 | * last event poll time by thread |
| 202 | * Debug only. Will cause false cache-line sharing as-is |
| 203 | */ |
| 204 | f64 *last_event_poll_by_thread; |
| 205 | #endif |
| 206 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 207 | }; |
| 208 | |
| 209 | extern session_manager_main_t session_manager_main; |
Florin Coras | e04c299 | 2017-03-01 08:17:34 -0800 | [diff] [blame] | 210 | extern vlib_node_registration_t session_queue_node; |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 211 | |
| 212 | /* |
| 213 | * Session manager function |
| 214 | */ |
| 215 | always_inline session_manager_main_t * |
| 216 | vnet_get_session_manager_main () |
| 217 | { |
| 218 | return &session_manager_main; |
| 219 | } |
| 220 | |
Dave Barach | 2c25a62 | 2017-06-26 11:35:07 -0400 | [diff] [blame] | 221 | always_inline u8 |
| 222 | stream_session_is_valid (u32 si, u8 thread_index) |
| 223 | { |
| 224 | stream_session_t *s; |
| 225 | s = pool_elt_at_index (session_manager_main.sessions[thread_index], si); |
| 226 | if (s->thread_index != thread_index || s->session_index != si |
Dave Barach | 52851e6 | 2017-08-07 09:35:25 -0400 | [diff] [blame] | 227 | /* || s->server_rx_fifo->master_session_index != si |
| 228 | || s->server_tx_fifo->master_session_index != si |
| 229 | || s->server_rx_fifo->master_thread_index != thread_index |
| 230 | || s->server_tx_fifo->master_thread_index != thread_index */ ) |
Dave Barach | 2c25a62 | 2017-06-26 11:35:07 -0400 | [diff] [blame] | 231 | return 0; |
| 232 | return 1; |
| 233 | } |
| 234 | |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 235 | stream_session_t *session_alloc (u32 thread_index); |
| 236 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 237 | always_inline stream_session_t * |
Florin Coras | cea194d | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 238 | session_get (u32 si, u32 thread_index) |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 239 | { |
Dave Barach | 2c25a62 | 2017-06-26 11:35:07 -0400 | [diff] [blame] | 240 | ASSERT (stream_session_is_valid (si, thread_index)); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 241 | return pool_elt_at_index (session_manager_main.sessions[thread_index], si); |
| 242 | } |
| 243 | |
| 244 | always_inline stream_session_t * |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 245 | session_get_if_valid (u64 si, u32 thread_index) |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 246 | { |
| 247 | if (thread_index >= vec_len (session_manager_main.sessions)) |
| 248 | return 0; |
| 249 | |
| 250 | if (pool_is_free_index (session_manager_main.sessions[thread_index], si)) |
| 251 | return 0; |
| 252 | |
Dave Barach | 2c25a62 | 2017-06-26 11:35:07 -0400 | [diff] [blame] | 253 | ASSERT (stream_session_is_valid (si, thread_index)); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 254 | return pool_elt_at_index (session_manager_main.sessions[thread_index], si); |
| 255 | } |
| 256 | |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 257 | always_inline u64 |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 258 | session_handle (stream_session_t * s) |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 259 | { |
| 260 | return ((u64) s->thread_index << 32) | (u64) s->session_index; |
| 261 | } |
| 262 | |
| 263 | always_inline u32 |
Florin Coras | cea194d | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 264 | session_index_from_handle (u64 handle) |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 265 | { |
| 266 | return handle & 0xFFFFFFFF; |
| 267 | } |
| 268 | |
| 269 | always_inline u32 |
Florin Coras | cea194d | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 270 | session_thread_from_handle (u64 handle) |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 271 | { |
| 272 | return handle >> 32; |
| 273 | } |
| 274 | |
| 275 | always_inline void |
Florin Coras | cea194d | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 276 | session_parse_handle (u64 handle, u32 * index, u32 * thread_index) |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 277 | { |
Florin Coras | cea194d | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 278 | *index = session_index_from_handle (handle); |
| 279 | *thread_index = session_thread_from_handle (handle); |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 280 | } |
| 281 | |
| 282 | always_inline stream_session_t * |
Florin Coras | cea194d | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 283 | session_get_from_handle (u64 handle) |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 284 | { |
| 285 | session_manager_main_t *smm = &session_manager_main; |
Florin Coras | cea194d | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 286 | return |
| 287 | pool_elt_at_index (smm->sessions[session_thread_from_handle (handle)], |
| 288 | session_index_from_handle (handle)); |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 289 | } |
| 290 | |
Florin Coras | 561af9b | 2017-12-09 10:19:43 -0800 | [diff] [blame] | 291 | always_inline transport_proto_t |
| 292 | session_get_transport_proto (stream_session_t * s) |
| 293 | { |
| 294 | return (s->session_type >> 1); |
| 295 | } |
| 296 | |
| 297 | always_inline session_type_t |
| 298 | session_type_from_proto_and_ip (transport_proto_t proto, u8 is_ip4) |
| 299 | { |
| 300 | return (proto << 1 | is_ip4); |
| 301 | } |
| 302 | |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 303 | /** |
| 304 | * Acquires a lock that blocks a session pool from expanding. |
| 305 | * |
| 306 | * This is typically used for safely peeking into other threads' |
| 307 | * pools in order to clone elements. Lock should be dropped as soon |
| 308 | * as possible by calling @ref session_pool_remove_peeker. |
| 309 | * |
| 310 | * NOTE: Avoid using pool_elt_at_index while the lock is held because |
| 311 | * it may lead to free elt bitmap expansion/contraction! |
| 312 | */ |
| 313 | always_inline void |
| 314 | session_pool_add_peeker (u32 thread_index) |
| 315 | { |
| 316 | session_manager_main_t *smm = &session_manager_main; |
| 317 | if (thread_index == vlib_get_thread_index ()) |
| 318 | return; |
| 319 | clib_spinlock_lock_if_init (&smm->peekers_readers_locks[thread_index]); |
| 320 | smm->session_peekers[thread_index] += 1; |
| 321 | if (smm->session_peekers[thread_index] == 1) |
| 322 | clib_spinlock_lock_if_init (&smm->peekers_write_locks[thread_index]); |
| 323 | clib_spinlock_unlock_if_init (&smm->peekers_readers_locks[thread_index]); |
| 324 | } |
| 325 | |
| 326 | always_inline void |
| 327 | session_pool_remove_peeker (u32 thread_index) |
| 328 | { |
| 329 | session_manager_main_t *smm = &session_manager_main; |
| 330 | if (thread_index == vlib_get_thread_index ()) |
| 331 | return; |
| 332 | ASSERT (session_manager_main.session_peekers[thread_index] > 0); |
| 333 | clib_spinlock_lock_if_init (&smm->peekers_readers_locks[thread_index]); |
| 334 | smm->session_peekers[thread_index] -= 1; |
| 335 | if (smm->session_peekers[thread_index] == 0) |
| 336 | clib_spinlock_unlock_if_init (&smm->peekers_write_locks[thread_index]); |
| 337 | clib_spinlock_unlock_if_init (&smm->peekers_readers_locks[thread_index]); |
| 338 | } |
| 339 | |
| 340 | /** |
| 341 | * Get session from handle and 'lock' pool resize if not in same thread |
| 342 | * |
| 343 | * Caller should drop the peek 'lock' as soon as possible. |
| 344 | */ |
| 345 | always_inline stream_session_t * |
| 346 | session_get_from_handle_safe (u64 handle) |
| 347 | { |
| 348 | session_manager_main_t *smm = &session_manager_main; |
| 349 | u32 thread_index = session_thread_from_handle (handle); |
| 350 | if (thread_index == vlib_get_thread_index ()) |
| 351 | { |
| 352 | return pool_elt_at_index (smm->sessions[thread_index], |
| 353 | session_index_from_handle (handle)); |
| 354 | } |
| 355 | else |
| 356 | { |
| 357 | session_pool_add_peeker (thread_index); |
| 358 | /* Don't use pool_elt_at index. See @ref session_pool_add_peeker */ |
| 359 | return smm->sessions[thread_index] + session_index_from_handle (handle); |
| 360 | } |
| 361 | } |
| 362 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 363 | always_inline stream_session_t * |
| 364 | stream_session_listener_get (u8 sst, u64 si) |
| 365 | { |
| 366 | return pool_elt_at_index (session_manager_main.listen_sessions[sst], si); |
| 367 | } |
| 368 | |
| 369 | always_inline u32 |
| 370 | stream_session_get_index (stream_session_t * s) |
| 371 | { |
| 372 | if (s->session_state == SESSION_STATE_LISTENING) |
| 373 | return s - session_manager_main.listen_sessions[s->session_type]; |
| 374 | |
| 375 | return s - session_manager_main.sessions[s->thread_index]; |
| 376 | } |
| 377 | |
| 378 | always_inline u32 |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 379 | stream_session_max_rx_enqueue (transport_connection_t * tc) |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 380 | { |
Florin Coras | cea194d | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 381 | stream_session_t *s = session_get (tc->s_index, tc->thread_index); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 382 | return svm_fifo_max_enqueue (s->server_rx_fifo); |
| 383 | } |
| 384 | |
Florin Coras | e04c299 | 2017-03-01 08:17:34 -0800 | [diff] [blame] | 385 | always_inline u32 |
Florin Coras | 93992a9 | 2017-05-24 18:03:56 -0700 | [diff] [blame] | 386 | stream_session_rx_fifo_size (transport_connection_t * tc) |
Florin Coras | e04c299 | 2017-03-01 08:17:34 -0800 | [diff] [blame] | 387 | { |
Florin Coras | cea194d | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 388 | stream_session_t *s = session_get (tc->s_index, tc->thread_index); |
Florin Coras | e04c299 | 2017-03-01 08:17:34 -0800 | [diff] [blame] | 389 | return s->server_rx_fifo->nitems; |
| 390 | } |
| 391 | |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 392 | always_inline u32 |
| 393 | session_get_index (stream_session_t * s) |
| 394 | { |
| 395 | return (s - session_manager_main.sessions[s->thread_index]); |
| 396 | } |
| 397 | |
| 398 | always_inline stream_session_t * |
| 399 | session_clone_safe (u32 session_index, u32 thread_index) |
| 400 | { |
| 401 | stream_session_t *old_s, *new_s; |
| 402 | u32 current_thread_index = vlib_get_thread_index (); |
| 403 | |
| 404 | /* If during the memcpy pool is reallocated AND the memory allocator |
| 405 | * decides to give the old chunk of memory to somebody in a hurry to |
| 406 | * scribble something on it, we have a problem. So add this thread as |
| 407 | * a session pool peeker. |
| 408 | */ |
| 409 | session_pool_add_peeker (thread_index); |
| 410 | new_s = session_alloc (current_thread_index); |
| 411 | old_s = session_manager_main.sessions[thread_index] + session_index; |
| 412 | clib_memcpy (new_s, old_s, sizeof (*new_s)); |
| 413 | session_pool_remove_peeker (thread_index); |
| 414 | new_s->thread_index = current_thread_index; |
| 415 | new_s->session_index = session_get_index (new_s); |
| 416 | return new_s; |
| 417 | } |
| 418 | |
| 419 | transport_connection_t *session_get_transport (stream_session_t * s); |
| 420 | |
Florin Coras | 93992a9 | 2017-05-24 18:03:56 -0700 | [diff] [blame] | 421 | u32 stream_session_tx_fifo_max_dequeue (transport_connection_t * tc); |
| 422 | |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 423 | stream_session_t *session_alloc (u32 thread_index); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 424 | int |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 425 | session_enqueue_stream_connection (transport_connection_t * tc, |
| 426 | vlib_buffer_t * b, u32 offset, |
| 427 | u8 queue_event, u8 is_in_order); |
| 428 | int session_enqueue_dgram_connection (stream_session_t * s, vlib_buffer_t * b, |
| 429 | u8 proto, u8 queue_event); |
| 430 | int stream_session_peek_bytes (transport_connection_t * tc, u8 * buffer, |
| 431 | u32 offset, u32 max_bytes); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 432 | u32 stream_session_dequeue_drop (transport_connection_t * tc, u32 max_bytes); |
| 433 | |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 434 | int session_stream_connect_notify (transport_connection_t * tc, u8 is_fail); |
| 435 | int session_dgram_connect_notify (transport_connection_t * tc, |
| 436 | u32 old_thread_index, |
| 437 | stream_session_t ** new_session); |
Florin Coras | c28764f | 2017-04-26 00:08:42 -0700 | [diff] [blame] | 438 | void stream_session_init_fifos_pointers (transport_connection_t * tc, |
| 439 | u32 rx_pointer, u32 tx_pointer); |
Florin Coras | e69f495 | 2017-03-07 10:06:24 -0800 | [diff] [blame] | 440 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 441 | void stream_session_accept_notify (transport_connection_t * tc); |
| 442 | void stream_session_disconnect_notify (transport_connection_t * tc); |
| 443 | void stream_session_delete_notify (transport_connection_t * tc); |
| 444 | void stream_session_reset_notify (transport_connection_t * tc); |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 445 | int stream_session_accept (transport_connection_t * tc, u32 listener_index, |
| 446 | u8 notify); |
| 447 | int session_open (u32 app_index, session_endpoint_t * tep, u32 opaque); |
Florin Coras | cea194d | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 448 | int stream_session_listen (stream_session_t * s, session_endpoint_t * tep); |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 449 | int stream_session_stop_listen (stream_session_t * s); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 450 | void stream_session_disconnect (stream_session_t * s); |
Florin Coras | 2f8d8fa | 2018-01-26 06:36:04 -0800 | [diff] [blame^] | 451 | void stream_session_disconnect_transport (stream_session_t * s); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 452 | void stream_session_cleanup (stream_session_t * s); |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 453 | void session_send_session_evt_to_thread (u64 session_handle, |
| 454 | fifo_event_type_t evt_type, |
| 455 | u32 thread_index); |
Florin Coras | b384b54 | 2018-01-15 01:08:33 -0800 | [diff] [blame] | 456 | ssvm_private_t *session_manager_get_evt_q_segment (void); |
Florin Coras | 3eb5062 | 2017-07-13 01:24:57 -0400 | [diff] [blame] | 457 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 458 | u8 *format_stream_session (u8 * s, va_list * args); |
Florin Coras | 3eb5062 | 2017-07-13 01:24:57 -0400 | [diff] [blame] | 459 | uword unformat_stream_session (unformat_input_t * input, va_list * args); |
| 460 | uword unformat_transport_connection (unformat_input_t * input, |
| 461 | va_list * args); |
| 462 | |
Florin Coras | 561af9b | 2017-12-09 10:19:43 -0800 | [diff] [blame] | 463 | void session_register_transport (transport_proto_t transport_proto, |
| 464 | const transport_proto_vft_t * vft, u8 is_ip4, |
| 465 | u32 output_node); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 466 | |
Florin Coras | e04c299 | 2017-03-01 08:17:34 -0800 | [diff] [blame] | 467 | clib_error_t *vnet_session_enable_disable (vlib_main_t * vm, u8 is_en); |
| 468 | |
Florin Coras | e86a8ed | 2018-01-05 03:20:25 -0800 | [diff] [blame] | 469 | always_inline svm_queue_t * |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 470 | session_manager_get_vpp_event_queue (u32 thread_index) |
| 471 | { |
| 472 | return session_manager_main.vpp_event_queues[thread_index]; |
| 473 | } |
| 474 | |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 475 | int session_manager_flush_enqueue_events (u8 proto, u32 thread_index); |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 476 | |
| 477 | always_inline u64 |
| 478 | listen_session_get_handle (stream_session_t * s) |
| 479 | { |
| 480 | ASSERT (s->session_state == SESSION_STATE_LISTENING); |
| 481 | return ((u64) s->session_type << 32) | s->session_index; |
| 482 | } |
| 483 | |
| 484 | always_inline stream_session_t * |
| 485 | listen_session_get_from_handle (u64 handle) |
| 486 | { |
| 487 | session_manager_main_t *smm = &session_manager_main; |
| 488 | stream_session_t *s; |
| 489 | u32 type, index; |
| 490 | type = handle >> 32; |
| 491 | index = handle & 0xFFFFFFFF; |
| 492 | |
| 493 | if (pool_is_free_index (smm->listen_sessions[type], index)) |
| 494 | return 0; |
| 495 | |
| 496 | s = pool_elt_at_index (smm->listen_sessions[type], index); |
| 497 | ASSERT (s->session_state == SESSION_STATE_LISTENING); |
| 498 | return s; |
| 499 | } |
| 500 | |
| 501 | always_inline stream_session_t * |
| 502 | listen_session_new (session_type_t type) |
| 503 | { |
| 504 | stream_session_t *s; |
Dave Barach | 1015a1e | 2017-05-08 19:15:03 -0400 | [diff] [blame] | 505 | pool_get_aligned (session_manager_main.listen_sessions[type], s, |
| 506 | CLIB_CACHE_LINE_BYTES); |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 507 | memset (s, 0, sizeof (*s)); |
| 508 | |
| 509 | s->session_type = type; |
| 510 | s->session_state = SESSION_STATE_LISTENING; |
| 511 | s->session_index = s - session_manager_main.listen_sessions[type]; |
| 512 | |
| 513 | return s; |
| 514 | } |
| 515 | |
| 516 | always_inline stream_session_t * |
| 517 | listen_session_get (session_type_t type, u32 index) |
| 518 | { |
| 519 | return pool_elt_at_index (session_manager_main.listen_sessions[type], |
| 520 | index); |
| 521 | } |
| 522 | |
| 523 | always_inline void |
| 524 | listen_session_del (stream_session_t * s) |
| 525 | { |
| 526 | pool_put (session_manager_main.listen_sessions[s->session_type], s); |
| 527 | } |
| 528 | |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 529 | transport_connection_t *listen_session_get_transport (stream_session_t * s); |
| 530 | |
Florin Coras | cea194d | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 531 | int |
| 532 | listen_session_get_local_session_endpoint (stream_session_t * listener, |
| 533 | session_endpoint_t * sep); |
| 534 | |
Florin Coras | 04e5344 | 2017-07-16 17:12:15 -0700 | [diff] [blame] | 535 | always_inline stream_session_t * |
Florin Coras | 561af9b | 2017-12-09 10:19:43 -0800 | [diff] [blame] | 536 | session_manager_get_listener (u8 session_type, u32 index) |
Florin Coras | 04e5344 | 2017-07-16 17:12:15 -0700 | [diff] [blame] | 537 | { |
Florin Coras | 561af9b | 2017-12-09 10:19:43 -0800 | [diff] [blame] | 538 | return |
| 539 | pool_elt_at_index (session_manager_main.listen_sessions[session_type], |
| 540 | index); |
Florin Coras | 04e5344 | 2017-07-16 17:12:15 -0700 | [diff] [blame] | 541 | } |
| 542 | |
Florin Coras | 561af9b | 2017-12-09 10:19:43 -0800 | [diff] [blame] | 543 | /** |
| 544 | * Set peek or dequeue function for given session type |
| 545 | * |
| 546 | * Reliable transport protocols will probably want to use a peek function |
| 547 | */ |
Florin Coras | 04e5344 | 2017-07-16 17:12:15 -0700 | [diff] [blame] | 548 | always_inline void |
Florin Coras | 561af9b | 2017-12-09 10:19:43 -0800 | [diff] [blame] | 549 | session_manager_set_transport_rx_fn (session_type_t type, u8 is_peek) |
Florin Coras | 04e5344 | 2017-07-16 17:12:15 -0700 | [diff] [blame] | 550 | { |
Florin Coras | 04e5344 | 2017-07-16 17:12:15 -0700 | [diff] [blame] | 551 | session_manager_main.session_tx_fns[type] = (is_peek) ? |
| 552 | session_tx_fifo_peek_and_snd : session_tx_fifo_dequeue_and_snd; |
| 553 | } |
| 554 | |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 555 | always_inline u8 |
| 556 | session_manager_is_enabled () |
| 557 | { |
| 558 | return session_manager_main.is_enabled == 1; |
| 559 | } |
| 560 | |
Florin Coras | cea194d | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 561 | #define session_cli_return_if_not_enabled() \ |
| 562 | do { \ |
| 563 | if (!session_manager_main.is_enabled) \ |
| 564 | return clib_error_return(0, "session layer is not enabled"); \ |
| 565 | } while (0) |
| 566 | |
Dave Barach | 2a86391 | 2017-11-28 10:11:42 -0500 | [diff] [blame] | 567 | void session_node_enable_disable (u8 is_en); |
| 568 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 569 | #endif /* __included_session_h__ */ |
| 570 | |
| 571 | /* |
| 572 | * fd.io coding-style-patch-verification: ON |
| 573 | * |
| 574 | * Local Variables: |
| 575 | * eval: (c-set-style "gnu") |
| 576 | * End: |
| 577 | */ |