Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 1 | /* |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 2 | * Copyright (c) 2017-2019 Cisco and/or its affiliates. |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 3 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | * you may not use this file except in compliance with the License. |
| 5 | * You may obtain a copy of the License at: |
| 6 | * |
| 7 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | * |
| 9 | * Unless required by applicable law or agreed to in writing, software |
| 10 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | * See the License for the specific language governing permissions and |
| 13 | * limitations under the License. |
| 14 | */ |
| 15 | #ifndef __included_session_h__ |
| 16 | #define __included_session_h__ |
| 17 | |
Florin Coras | 2062ec0 | 2019-07-15 13:15:18 -0700 | [diff] [blame] | 18 | #include <vppinfra/llist.h> |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 19 | #include <vnet/session/session_types.h> |
Florin Coras | 04e5344 | 2017-07-16 17:12:15 -0700 | [diff] [blame] | 20 | #include <vnet/session/session_lookup.h> |
Florin Coras | 3e350af | 2017-03-30 02:54:28 -0700 | [diff] [blame] | 21 | #include <vnet/session/session_debug.h> |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 22 | #include <svm/message_queue.h> |
Florin Coras | 04943b4 | 2020-12-25 11:45:40 -0800 | [diff] [blame] | 23 | #include <svm/fifo_segment.h> |
Marvin Liu | 0654242 | 2022-08-16 06:49:09 +0000 | [diff] [blame] | 24 | #include <vlib/dma/dma.h> |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 25 | |
Florin Coras | 3e350af | 2017-03-30 02:54:28 -0700 | [diff] [blame] | 26 | #define foreach_session_input_error \ |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 27 | _(NO_SESSION, "No session drops") \ |
| 28 | _(NO_LISTENER, "No listener for dst port drops") \ |
| 29 | _(ENQUEUED, "Packets pushed into rx fifo") \ |
| 30 | _(NOT_READY, "Session not ready packets") \ |
| 31 | _(FIFO_FULL, "Packets dropped for lack of rx fifo space") \ |
| 32 | _(EVENT_FIFO_FULL, "Events not sent for lack of event fifo space") \ |
| 33 | _(API_QUEUE_FULL, "Sessions not created for lack of API queue space") \ |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 34 | |
| 35 | typedef enum |
| 36 | { |
| 37 | #define _(sym,str) SESSION_ERROR_##sym, |
| 38 | foreach_session_input_error |
| 39 | #undef _ |
| 40 | SESSION_N_ERROR, |
Florin Coras | 00e01d3 | 2019-10-21 16:07:46 -0700 | [diff] [blame] | 41 | } session_input_error_t; |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 42 | |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 43 | typedef struct session_tx_context_ |
| 44 | { |
| 45 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 46 | session_t *s; |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 47 | transport_proto_vft_t *transport_vft; |
| 48 | transport_connection_t *tc; |
Florin Coras | 70f879d | 2020-03-13 17:54:42 +0000 | [diff] [blame] | 49 | transport_send_params_t sp; |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 50 | u32 max_dequeue; |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 51 | u32 left_to_snd; |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 52 | u32 max_len_to_snd; |
| 53 | u16 deq_per_first_buf; |
| 54 | u16 deq_per_buf; |
Florin Coras | f08f26d | 2018-05-10 13:20:47 -0700 | [diff] [blame] | 55 | u16 n_segs_per_evt; |
Simon Zhang | ae16a98 | 2020-03-31 20:56:22 +0800 | [diff] [blame] | 56 | u16 n_bufs_needed; |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 57 | u8 n_bufs_per_seg; |
| 58 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline1); |
| 59 | session_dgram_hdr_t hdr; |
Florin Coras | 1d7a663 | 2021-05-17 23:44:53 -0700 | [diff] [blame] | 60 | |
| 61 | /** Vector of tx buffer free lists */ |
| 62 | u32 *tx_buffers; |
Florin Coras | f66cc80 | 2021-04-08 19:10:07 -0700 | [diff] [blame] | 63 | vlib_buffer_t **transport_pending_bufs; |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 64 | } session_tx_context_t; |
| 65 | |
Florin Coras | 2062ec0 | 2019-07-15 13:15:18 -0700 | [diff] [blame] | 66 | typedef struct session_evt_elt |
| 67 | { |
| 68 | clib_llist_anchor_t evt_list; |
| 69 | session_event_t evt; |
| 70 | } session_evt_elt_t; |
| 71 | |
Florin Coras | 458089b | 2019-08-21 16:20:44 -0700 | [diff] [blame] | 72 | typedef struct session_ctrl_evt_data_ |
| 73 | { |
| 74 | u8 data[SESSION_CTRL_MSG_MAX_SIZE]; |
| 75 | } session_evt_ctrl_data_t; |
| 76 | |
Florin Coras | 7da8829 | 2021-03-18 15:04:34 -0700 | [diff] [blame] | 77 | typedef enum session_wrk_state_ |
| 78 | { |
| 79 | SESSION_WRK_POLLING, |
| 80 | SESSION_WRK_INTERRUPT, |
| 81 | SESSION_WRK_IDLE, |
| 82 | } __clib_packed session_wrk_state_t; |
| 83 | |
| 84 | typedef enum session_wrk_flags_ |
| 85 | { |
| 86 | SESSION_WRK_F_ADAPTIVE = 1 << 0, |
| 87 | } __clib_packed session_wrk_flag_t; |
| 88 | |
Marvin Liu | 0654242 | 2022-08-16 06:49:09 +0000 | [diff] [blame] | 89 | #define DMA_TRANS_SIZE 1024 |
| 90 | typedef struct |
| 91 | { |
| 92 | u32 *pending_tx_buffers; |
| 93 | u16 *pending_tx_nexts; |
| 94 | } session_dma_transfer; |
| 95 | |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 96 | typedef struct session_worker_ |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 97 | { |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 98 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 99 | |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 100 | /** Worker session pool */ |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 101 | session_t *sessions; |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 102 | |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 103 | /** vpp event message queue for worker */ |
| 104 | svm_msg_q_t *vpp_event_queue; |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 105 | |
Florin Coras | d67f112 | 2018-05-21 17:47:40 -0700 | [diff] [blame] | 106 | /** vlib_time_now last time around the track */ |
Florin Coras | a8e71c8 | 2019-10-22 19:01:39 -0700 | [diff] [blame] | 107 | clib_time_type_t last_vlib_time; |
| 108 | |
| 109 | /** vlib_time_now rounded to us precision and as u64 */ |
| 110 | clib_us_time_t last_vlib_us_time; |
Florin Coras | d67f112 | 2018-05-21 17:47:40 -0700 | [diff] [blame] | 111 | |
Florin Coras | 60183db | 2019-07-20 15:53:16 -0700 | [diff] [blame] | 112 | /** Convenience pointer to this worker's vlib_main */ |
| 113 | vlib_main_t *vm; |
| 114 | |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 115 | /** Per-proto vector of sessions to enqueue */ |
Florin Coras | 07063b8 | 2020-03-13 04:44:51 +0000 | [diff] [blame] | 116 | u32 **session_to_enqueue; |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 117 | |
Florin Coras | 7da8829 | 2021-03-18 15:04:34 -0700 | [diff] [blame] | 118 | /** Timerfd used to periodically signal wrk session queue node */ |
Florin Coras | 1c19aef | 2021-04-06 15:54:14 -0700 | [diff] [blame] | 119 | int timerfd; |
Florin Coras | 7da8829 | 2021-03-18 15:04:34 -0700 | [diff] [blame] | 120 | |
| 121 | /** Worker flags */ |
| 122 | session_wrk_flag_t flags; |
| 123 | |
| 124 | /** Worker state */ |
| 125 | session_wrk_state_t state; |
| 126 | |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 127 | /** Context for session tx */ |
| 128 | session_tx_context_t ctx; |
| 129 | |
Florin Coras | 2062ec0 | 2019-07-15 13:15:18 -0700 | [diff] [blame] | 130 | /** Pool of session event list elements */ |
| 131 | session_evt_elt_t *event_elts; |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 132 | |
Florin Coras | 458089b | 2019-08-21 16:20:44 -0700 | [diff] [blame] | 133 | /** Pool of ctrl events data buffers */ |
| 134 | session_evt_ctrl_data_t *ctrl_evts_data; |
| 135 | |
Florin Coras | b0ffbee | 2019-07-21 19:23:46 -0700 | [diff] [blame] | 136 | /** Head of control events list */ |
| 137 | clib_llist_index_t ctrl_head; |
| 138 | |
Florin Coras | 2062ec0 | 2019-07-15 13:15:18 -0700 | [diff] [blame] | 139 | /** Head of list of elements */ |
| 140 | clib_llist_index_t new_head; |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 141 | |
Florin Coras | 2062ec0 | 2019-07-15 13:15:18 -0700 | [diff] [blame] | 142 | /** Head of list of pending events */ |
Florin Coras | 60183db | 2019-07-20 15:53:16 -0700 | [diff] [blame] | 143 | clib_llist_index_t old_head; |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 144 | |
Florin Coras | 2a7ea2e | 2019-10-16 22:06:08 -0700 | [diff] [blame] | 145 | /** Vector of buffers to be sent */ |
| 146 | u32 *pending_tx_buffers; |
| 147 | |
| 148 | /** Vector of nexts for the pending tx buffers */ |
| 149 | u16 *pending_tx_nexts; |
| 150 | |
Florin Coras | 7da8829 | 2021-03-18 15:04:34 -0700 | [diff] [blame] | 151 | /** Clib file for timerfd. Used only if adaptive mode is on */ |
| 152 | uword timerfd_file; |
| 153 | |
Florin Coras | af97221 | 2021-05-05 09:54:00 -0700 | [diff] [blame] | 154 | /** List of pending connects for first worker */ |
| 155 | clib_llist_index_t pending_connects; |
| 156 | |
| 157 | /** Flag that is set if main thread signaled to handle connects */ |
Florin Coras | 0f27339 | 2021-05-21 15:48:18 -0700 | [diff] [blame] | 158 | u32 n_pending_connects; |
Florin Coras | af97221 | 2021-05-05 09:54:00 -0700 | [diff] [blame] | 159 | |
Florin Coras | a48dffe | 2021-05-16 10:58:53 -0700 | [diff] [blame] | 160 | /** Main thread loops in poll mode without a connect */ |
| 161 | u32 no_connect_loops; |
| 162 | |
Florin Coras | e09bd48 | 2022-03-21 10:38:01 -0700 | [diff] [blame] | 163 | /** List head for first worker evts pending handling on main */ |
| 164 | clib_llist_index_t evts_pending_main; |
| 165 | |
Marvin Liu | 0654242 | 2022-08-16 06:49:09 +0000 | [diff] [blame] | 166 | int config_index; |
| 167 | u8 dma_enabled; |
| 168 | session_dma_transfer *dma_trans; |
| 169 | u16 trans_head; |
| 170 | u16 trans_tail; |
| 171 | u16 trans_size; |
| 172 | u16 batch_num; |
| 173 | vlib_dma_batch_t *batch; |
| 174 | |
Florin Coras | cca9618 | 2019-07-19 07:34:13 -0700 | [diff] [blame] | 175 | #if SESSION_DEBUG |
| 176 | /** last event poll time by thread */ |
Florin Coras | a8e71c8 | 2019-10-22 19:01:39 -0700 | [diff] [blame] | 177 | clib_time_type_t last_event_poll; |
Florin Coras | cca9618 | 2019-07-19 07:34:13 -0700 | [diff] [blame] | 178 | #endif |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 179 | } session_worker_t; |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 180 | |
Florin Coras | 60183db | 2019-07-20 15:53:16 -0700 | [diff] [blame] | 181 | typedef int (session_fifo_rx_fn) (session_worker_t * wrk, |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 182 | vlib_node_runtime_t * node, |
Florin Coras | 60183db | 2019-07-20 15:53:16 -0700 | [diff] [blame] | 183 | session_evt_elt_t * e, int *n_tx_packets); |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 184 | |
| 185 | extern session_fifo_rx_fn session_tx_fifo_peek_and_snd; |
| 186 | extern session_fifo_rx_fn session_tx_fifo_dequeue_and_snd; |
| 187 | extern session_fifo_rx_fn session_tx_fifo_dequeue_internal; |
| 188 | |
| 189 | u8 session_node_lookup_fifo_event (svm_fifo_t * f, session_event_t * e); |
| 190 | |
Florin Coras | 5384cca | 2022-01-20 18:10:26 -0800 | [diff] [blame] | 191 | typedef void (*session_update_time_fn) (f64 time_now, u8 thread_index); |
| 192 | |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 193 | typedef struct session_main_ |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 194 | { |
| 195 | /** Worker contexts */ |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 196 | session_worker_t *wrk; |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 197 | |
Florin Coras | 5384cca | 2022-01-20 18:10:26 -0800 | [diff] [blame] | 198 | /** Vector of transport update time functions */ |
| 199 | session_update_time_fn *update_time_fns; |
| 200 | |
Florin Coras | 6c10ab2 | 2020-11-08 16:50:39 -0800 | [diff] [blame] | 201 | /** Event queues memfd segment */ |
Florin Coras | 8afe1b8 | 2021-11-17 23:38:54 -0800 | [diff] [blame] | 202 | fifo_segment_t wrk_mqs_segment; |
Florin Coras | b384b54 | 2018-01-15 01:08:33 -0800 | [diff] [blame] | 203 | |
Florin Coras | 93e6580 | 2017-11-29 00:07:11 -0500 | [diff] [blame] | 204 | /** Unique segment name counter */ |
| 205 | u32 unique_segment_name_counter; |
| 206 | |
| 207 | /** Per transport rx function that can either dequeue or peek */ |
Florin Coras | 561af9b | 2017-12-09 10:19:43 -0800 | [diff] [blame] | 208 | session_fifo_rx_fn **session_tx_fns; |
| 209 | |
| 210 | /** Per session type output nodes. Could optimize to group nodes by |
| 211 | * fib but lookup would then require session type parsing in session node. |
| 212 | * Trade memory for speed, for now */ |
| 213 | u32 *session_type_to_next; |
Florin Coras | 93e6580 | 2017-11-29 00:07:11 -0500 | [diff] [blame] | 214 | |
Florin Coras | fb50bc3 | 2021-05-18 00:28:59 -0700 | [diff] [blame] | 215 | /** Thread for cl and ho that rely on cl allocs */ |
| 216 | u32 transport_cl_thread; |
| 217 | |
Florin Coras | 07063b8 | 2020-03-13 04:44:51 +0000 | [diff] [blame] | 218 | transport_proto_t last_transport_proto_type; |
| 219 | |
Florin Coras | 53d8d4f | 2022-03-09 13:55:38 -0800 | [diff] [blame] | 220 | /** Number of workers at pool realloc barrier */ |
Florin Coras | 5b47a6d | 2022-03-26 01:25:02 -0700 | [diff] [blame] | 221 | volatile u32 pool_realloc_at_barrier; |
| 222 | |
| 223 | /** Number of workers doing reallocs */ |
| 224 | volatile u32 pool_realloc_doing_work; |
Florin Coras | 53d8d4f | 2022-03-09 13:55:38 -0800 | [diff] [blame] | 225 | |
| 226 | /** Lock to synchronize parallel forced reallocs */ |
| 227 | clib_spinlock_t pool_realloc_lock; |
| 228 | |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 229 | /* |
| 230 | * Config parameters |
| 231 | */ |
| 232 | |
Florin Coras | 93e6580 | 2017-11-29 00:07:11 -0500 | [diff] [blame] | 233 | /** Session manager is enabled */ |
| 234 | u8 is_enabled; |
Florin Coras | e10da62 | 2020-10-25 14:00:29 -0700 | [diff] [blame] | 235 | |
| 236 | /** Session manager initialized (not necessarily enabled) */ |
| 237 | u8 is_initialized; |
| 238 | |
Nathan Skrzypczak | 31bd035 | 2019-11-07 17:55:01 +0100 | [diff] [blame] | 239 | /** Enable session manager at startup */ |
| 240 | u8 session_enable_asap; |
Florin Coras | 93e6580 | 2017-11-29 00:07:11 -0500 | [diff] [blame] | 241 | |
Florin Coras | e92c946 | 2020-11-25 08:44:16 -0800 | [diff] [blame] | 242 | /** Poll session node in main thread */ |
| 243 | u8 poll_main; |
| 244 | |
Florin Coras | 41d5f54 | 2021-01-15 13:49:33 -0800 | [diff] [blame] | 245 | /** Allocate private rx mqs for external apps */ |
| 246 | u8 use_private_rx_mqs; |
| 247 | |
Florin Coras | 7da8829 | 2021-03-18 15:04:34 -0700 | [diff] [blame] | 248 | /** Do not enable session queue node adaptive mode */ |
| 249 | u8 no_adaptive; |
| 250 | |
Dave Barach | 10d8cc6 | 2017-05-30 09:30:07 -0400 | [diff] [blame] | 251 | /** vpp fifo event queue configured length */ |
Florin Coras | 8afe1b8 | 2021-11-17 23:38:54 -0800 | [diff] [blame] | 252 | u32 configured_wrk_mq_length; |
Dave Barach | 10d8cc6 | 2017-05-30 09:30:07 -0400 | [diff] [blame] | 253 | |
Florin Coras | b384b54 | 2018-01-15 01:08:33 -0800 | [diff] [blame] | 254 | /** Session ssvm segment configs*/ |
Florin Coras | 8afe1b8 | 2021-11-17 23:38:54 -0800 | [diff] [blame] | 255 | uword wrk_mqs_segment_size; |
Florin Coras | b384b54 | 2018-01-15 01:08:33 -0800 | [diff] [blame] | 256 | |
Marvin Liu | 0654242 | 2022-08-16 06:49:09 +0000 | [diff] [blame] | 257 | /** Session enable dma*/ |
| 258 | u8 dma_enabled; |
| 259 | |
Florin Coras | b384b54 | 2018-01-15 01:08:33 -0800 | [diff] [blame] | 260 | /** Session table size parameters */ |
Florin Coras | 66b1131 | 2017-07-31 17:18:03 -0700 | [diff] [blame] | 261 | u32 configured_v4_session_table_buckets; |
| 262 | u32 configured_v4_session_table_memory; |
| 263 | u32 configured_v4_halfopen_table_buckets; |
| 264 | u32 configured_v4_halfopen_table_memory; |
| 265 | u32 configured_v6_session_table_buckets; |
| 266 | u32 configured_v6_session_table_memory; |
| 267 | u32 configured_v6_halfopen_table_buckets; |
| 268 | u32 configured_v6_halfopen_table_memory; |
| 269 | |
Florin Coras | 93e6580 | 2017-11-29 00:07:11 -0500 | [diff] [blame] | 270 | /** Transport table (preallocation) size parameters */ |
| 271 | u32 local_endpoints_table_memory; |
| 272 | u32 local_endpoints_table_buckets; |
Florin Coras | e04c299 | 2017-03-01 08:17:34 -0800 | [diff] [blame] | 273 | |
Dave Barach | 2c25a62 | 2017-06-26 11:35:07 -0400 | [diff] [blame] | 274 | /** Preallocate session config parameter */ |
| 275 | u32 preallocated_sessions; |
| 276 | |
Filip Tehlar | 0046e97 | 2021-06-26 22:12:08 +0000 | [diff] [blame] | 277 | u16 msg_id_base; |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 278 | } session_main_t; |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 279 | |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 280 | extern session_main_t session_main; |
Florin Coras | e04c299 | 2017-03-01 08:17:34 -0800 | [diff] [blame] | 281 | extern vlib_node_registration_t session_queue_node; |
Florin Coras | fd542f1 | 2018-05-16 19:28:24 -0700 | [diff] [blame] | 282 | extern vlib_node_registration_t session_queue_process_node; |
Florin Coras | 653e43f | 2019-03-04 10:56:23 -0800 | [diff] [blame] | 283 | extern vlib_node_registration_t session_queue_pre_input_node; |
Florin Coras | fd542f1 | 2018-05-16 19:28:24 -0700 | [diff] [blame] | 284 | |
Florin Coras | 5484daa | 2020-03-27 23:55:06 +0000 | [diff] [blame] | 285 | typedef enum session_q_process_evt_ |
| 286 | { |
| 287 | SESSION_Q_PROCESS_RUN_ON_MAIN = 1, |
| 288 | SESSION_Q_PROCESS_STOP |
| 289 | } session_q_process_evt_t; |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 290 | |
Florin Coras | 07063b8 | 2020-03-13 04:44:51 +0000 | [diff] [blame] | 291 | #define TRANSPORT_PROTO_INVALID (session_main.last_transport_proto_type + 1) |
| 292 | #define TRANSPORT_N_PROTOS (session_main.last_transport_proto_type + 1) |
| 293 | |
Florin Coras | 2062ec0 | 2019-07-15 13:15:18 -0700 | [diff] [blame] | 294 | static inline void |
Florin Coras | 60183db | 2019-07-20 15:53:16 -0700 | [diff] [blame] | 295 | session_evt_add_old (session_worker_t * wrk, session_evt_elt_t * elt) |
Florin Coras | 2062ec0 | 2019-07-15 13:15:18 -0700 | [diff] [blame] | 296 | { |
| 297 | clib_llist_add_tail (wrk->event_elts, evt_list, elt, |
Florin Coras | 46b8b1a | 2021-05-17 19:09:33 -0700 | [diff] [blame] | 298 | clib_llist_elt (wrk->event_elts, wrk->old_head)); |
Florin Coras | 2062ec0 | 2019-07-15 13:15:18 -0700 | [diff] [blame] | 299 | } |
| 300 | |
Florin Coras | dd97a48 | 2019-11-02 14:32:52 -0700 | [diff] [blame] | 301 | static inline void |
| 302 | session_evt_add_head_old (session_worker_t * wrk, session_evt_elt_t * elt) |
| 303 | { |
| 304 | clib_llist_add (wrk->event_elts, evt_list, elt, |
Florin Coras | 46b8b1a | 2021-05-17 19:09:33 -0700 | [diff] [blame] | 305 | clib_llist_elt (wrk->event_elts, wrk->old_head)); |
Florin Coras | dd97a48 | 2019-11-02 14:32:52 -0700 | [diff] [blame] | 306 | } |
| 307 | |
| 308 | |
Florin Coras | 458089b | 2019-08-21 16:20:44 -0700 | [diff] [blame] | 309 | static inline u32 |
| 310 | session_evt_ctrl_data_alloc (session_worker_t * wrk) |
| 311 | { |
| 312 | session_evt_ctrl_data_t *data; |
| 313 | pool_get (wrk->ctrl_evts_data, data); |
| 314 | return (data - wrk->ctrl_evts_data); |
| 315 | } |
| 316 | |
Florin Coras | b0ffbee | 2019-07-21 19:23:46 -0700 | [diff] [blame] | 317 | static inline session_evt_elt_t * |
| 318 | session_evt_alloc_ctrl (session_worker_t * wrk) |
Florin Coras | 2062ec0 | 2019-07-15 13:15:18 -0700 | [diff] [blame] | 319 | { |
Florin Coras | b0ffbee | 2019-07-21 19:23:46 -0700 | [diff] [blame] | 320 | session_evt_elt_t *elt; |
Florin Coras | 46b8b1a | 2021-05-17 19:09:33 -0700 | [diff] [blame] | 321 | clib_llist_get (wrk->event_elts, elt); |
Florin Coras | 2062ec0 | 2019-07-15 13:15:18 -0700 | [diff] [blame] | 322 | clib_llist_add_tail (wrk->event_elts, evt_list, elt, |
Florin Coras | 46b8b1a | 2021-05-17 19:09:33 -0700 | [diff] [blame] | 323 | clib_llist_elt (wrk->event_elts, wrk->ctrl_head)); |
Florin Coras | b0ffbee | 2019-07-21 19:23:46 -0700 | [diff] [blame] | 324 | return elt; |
Florin Coras | 2062ec0 | 2019-07-15 13:15:18 -0700 | [diff] [blame] | 325 | } |
| 326 | |
Florin Coras | 458089b | 2019-08-21 16:20:44 -0700 | [diff] [blame] | 327 | static inline void * |
| 328 | session_evt_ctrl_data (session_worker_t * wrk, session_evt_elt_t * elt) |
| 329 | { |
| 330 | return (void *) (pool_elt_at_index (wrk->ctrl_evts_data, |
| 331 | elt->evt.ctrl_data_index)); |
| 332 | } |
| 333 | |
| 334 | static inline void |
| 335 | session_evt_ctrl_data_free (session_worker_t * wrk, session_evt_elt_t * elt) |
| 336 | { |
| 337 | ASSERT (elt->evt.event_type > SESSION_IO_EVT_BUILTIN_TX); |
| 338 | pool_put_index (wrk->ctrl_evts_data, elt->evt.ctrl_data_index); |
| 339 | } |
| 340 | |
Florin Coras | 26dd6de | 2019-07-23 23:54:47 -0700 | [diff] [blame] | 341 | static inline session_evt_elt_t * |
| 342 | session_evt_alloc_new (session_worker_t * wrk) |
| 343 | { |
| 344 | session_evt_elt_t *elt; |
Florin Coras | 46b8b1a | 2021-05-17 19:09:33 -0700 | [diff] [blame] | 345 | clib_llist_get (wrk->event_elts, elt); |
Florin Coras | 26dd6de | 2019-07-23 23:54:47 -0700 | [diff] [blame] | 346 | clib_llist_add_tail (wrk->event_elts, evt_list, elt, |
Florin Coras | 46b8b1a | 2021-05-17 19:09:33 -0700 | [diff] [blame] | 347 | clib_llist_elt (wrk->event_elts, wrk->new_head)); |
Florin Coras | 26dd6de | 2019-07-23 23:54:47 -0700 | [diff] [blame] | 348 | return elt; |
| 349 | } |
| 350 | |
| 351 | static inline session_evt_elt_t * |
| 352 | session_evt_alloc_old (session_worker_t * wrk) |
| 353 | { |
| 354 | session_evt_elt_t *elt; |
Florin Coras | 46b8b1a | 2021-05-17 19:09:33 -0700 | [diff] [blame] | 355 | clib_llist_get (wrk->event_elts, elt); |
Florin Coras | 26dd6de | 2019-07-23 23:54:47 -0700 | [diff] [blame] | 356 | clib_llist_add_tail (wrk->event_elts, evt_list, elt, |
Florin Coras | 46b8b1a | 2021-05-17 19:09:33 -0700 | [diff] [blame] | 357 | clib_llist_elt (wrk->event_elts, wrk->old_head)); |
Florin Coras | 26dd6de | 2019-07-23 23:54:47 -0700 | [diff] [blame] | 358 | return elt; |
| 359 | } |
| 360 | |
Florin Coras | 41d5f54 | 2021-01-15 13:49:33 -0800 | [diff] [blame] | 361 | int session_wrk_handle_mq (session_worker_t *wrk, svm_msg_q_t *mq); |
| 362 | |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 363 | session_t *session_alloc (u32 thread_index); |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 364 | void session_free (session_t * s); |
| 365 | void session_free_w_fifos (session_t * s); |
Florin Coras | ea72764 | 2021-05-07 19:39:43 -0700 | [diff] [blame] | 366 | void session_cleanup_half_open (session_handle_t ho_handle); |
Srikanth Akula | e140d5d | 2019-11-18 11:49:58 -0800 | [diff] [blame] | 367 | u8 session_is_valid (u32 si, u8 thread_index); |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 368 | |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 369 | always_inline session_t * |
Florin Coras | cea194d | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 370 | session_get (u32 si, u32 thread_index) |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 371 | { |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 372 | ASSERT (session_is_valid (si, thread_index)); |
| 373 | return pool_elt_at_index (session_main.wrk[thread_index].sessions, si); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 374 | } |
| 375 | |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 376 | always_inline session_t * |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 377 | session_get_if_valid (u64 si, u32 thread_index) |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 378 | { |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 379 | if (thread_index >= vec_len (session_main.wrk)) |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 380 | return 0; |
| 381 | |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 382 | if (pool_is_free_index (session_main.wrk[thread_index].sessions, si)) |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 383 | return 0; |
| 384 | |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 385 | ASSERT (session_is_valid (si, thread_index)); |
| 386 | return pool_elt_at_index (session_main.wrk[thread_index].sessions, si); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 387 | } |
| 388 | |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 389 | always_inline session_t * |
Florin Coras | f8f516a | 2018-02-08 15:10:09 -0800 | [diff] [blame] | 390 | session_get_from_handle (session_handle_t handle) |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 391 | { |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 392 | session_main_t *smm = &session_main; |
Florin Coras | f8f516a | 2018-02-08 15:10:09 -0800 | [diff] [blame] | 393 | u32 session_index, thread_index; |
| 394 | session_parse_handle (handle, &session_index, &thread_index); |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 395 | return pool_elt_at_index (smm->wrk[thread_index].sessions, session_index); |
Florin Coras | f8f516a | 2018-02-08 15:10:09 -0800 | [diff] [blame] | 396 | } |
| 397 | |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 398 | always_inline session_t * |
Florin Coras | f8f516a | 2018-02-08 15:10:09 -0800 | [diff] [blame] | 399 | session_get_from_handle_if_valid (session_handle_t handle) |
| 400 | { |
| 401 | u32 session_index, thread_index; |
| 402 | session_parse_handle (handle, &session_index, &thread_index); |
| 403 | return session_get_if_valid (session_index, thread_index); |
| 404 | } |
| 405 | |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 406 | u64 session_segment_handle (session_t * s); |
Florin Coras | fa76a76 | 2018-11-29 12:40:10 -0800 | [diff] [blame] | 407 | |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 408 | /** |
Florin Coras | 6bd8d3f | 2022-03-14 21:17:25 -0700 | [diff] [blame] | 409 | * Get session from handle and avoid pool validation if no same thread |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 410 | * |
Florin Coras | 6bd8d3f | 2022-03-14 21:17:25 -0700 | [diff] [blame] | 411 | * Peekers are fine because pool grows with barrier (see @ref session_alloc) |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 412 | */ |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 413 | always_inline session_t * |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 414 | session_get_from_handle_safe (u64 handle) |
| 415 | { |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 416 | u32 thread_index = session_thread_from_handle (handle); |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 417 | session_worker_t *wrk = &session_main.wrk[thread_index]; |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 418 | |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 419 | if (thread_index == vlib_get_thread_index ()) |
| 420 | { |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 421 | return pool_elt_at_index (wrk->sessions, |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 422 | session_index_from_handle (handle)); |
| 423 | } |
| 424 | else |
| 425 | { |
Florin Coras | 6bd8d3f | 2022-03-14 21:17:25 -0700 | [diff] [blame] | 426 | /* Don't use pool_elt_at index to avoid pool bitmap reallocs */ |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 427 | return wrk->sessions + session_index_from_handle (handle); |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 428 | } |
| 429 | } |
| 430 | |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 431 | always_inline session_t * |
| 432 | session_clone_safe (u32 session_index, u32 thread_index) |
| 433 | { |
Florin Coras | 6bd8d3f | 2022-03-14 21:17:25 -0700 | [diff] [blame] | 434 | u32 current_thread_index = vlib_get_thread_index (), new_index; |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 435 | session_t *old_s, *new_s; |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 436 | |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 437 | new_s = session_alloc (current_thread_index); |
Florin Coras | 6bd8d3f | 2022-03-14 21:17:25 -0700 | [diff] [blame] | 438 | new_index = new_s->session_index; |
| 439 | /* Session pools are reallocated with barrier (see @ref session_alloc) */ |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 440 | old_s = session_main.wrk[thread_index].sessions + session_index; |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 441 | clib_memcpy_fast (new_s, old_s, sizeof (*new_s)); |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 442 | new_s->thread_index = current_thread_index; |
Florin Coras | 6bd8d3f | 2022-03-14 21:17:25 -0700 | [diff] [blame] | 443 | new_s->session_index = new_index; |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 444 | return new_s; |
| 445 | } |
| 446 | |
Florin Coras | 89a9f61 | 2021-05-11 11:55:07 -0700 | [diff] [blame] | 447 | int session_open (session_endpoint_cfg_t *sep, session_handle_t *rsh); |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 448 | int session_listen (session_t * s, session_endpoint_cfg_t * sep); |
| 449 | int session_stop_listen (session_t * s); |
liuyacan | 534468e | 2021-05-09 03:50:40 +0000 | [diff] [blame] | 450 | void session_half_close (session_t *s); |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 451 | void session_close (session_t * s); |
Florin Coras | dfb3b87 | 2019-08-16 17:48:44 -0700 | [diff] [blame] | 452 | void session_reset (session_t * s); |
liuyacan | 534468e | 2021-05-09 03:50:40 +0000 | [diff] [blame] | 453 | void session_transport_half_close (session_t *s); |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 454 | void session_transport_close (session_t * s); |
Florin Coras | dfb3b87 | 2019-08-16 17:48:44 -0700 | [diff] [blame] | 455 | void session_transport_reset (session_t * s); |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 456 | void session_transport_cleanup (session_t * s); |
| 457 | int session_send_io_evt_to_thread (svm_fifo_t * f, |
| 458 | session_evt_type_t evt_type); |
Florin Coras | 653e43f | 2019-03-04 10:56:23 -0800 | [diff] [blame] | 459 | int session_enqueue_notify (session_t * s); |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 460 | int session_dequeue_notify (session_t * s); |
| 461 | int session_send_io_evt_to_thread_custom (void *data, u32 thread_index, |
| 462 | session_evt_type_t evt_type); |
| 463 | void session_send_rpc_evt_to_thread (u32 thread_index, void *fp, |
| 464 | void *rpc_args); |
Nathan Skrzypczak | 60f3e65 | 2019-03-19 13:57:31 +0100 | [diff] [blame] | 465 | void session_send_rpc_evt_to_thread_force (u32 thread_index, void *fp, |
| 466 | void *rpc_args); |
Florin Coras | 26dd6de | 2019-07-23 23:54:47 -0700 | [diff] [blame] | 467 | void session_add_self_custom_tx_evt (transport_connection_t * tc, |
| 468 | u8 has_prio); |
Florin Coras | 70f879d | 2020-03-13 17:54:42 +0000 | [diff] [blame] | 469 | void sesssion_reschedule_tx (transport_connection_t * tc); |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 470 | transport_connection_t *session_get_transport (session_t * s); |
Florin Coras | 09d18c2 | 2019-04-24 11:10:02 -0700 | [diff] [blame] | 471 | void session_get_endpoint (session_t * s, transport_endpoint_t * tep, |
| 472 | u8 is_lcl); |
Florin Coras | 04ae827 | 2021-04-12 19:55:37 -0700 | [diff] [blame] | 473 | int session_transport_attribute (session_t *s, u8 is_get, |
| 474 | transport_endpt_attr_t *attr); |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 475 | |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 476 | u8 *format_session (u8 * s, va_list * args); |
| 477 | uword unformat_session (unformat_input_t * input, va_list * args); |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 478 | uword unformat_transport_connection (unformat_input_t * input, |
| 479 | va_list * args); |
| 480 | |
| 481 | /* |
| 482 | * Interface to transport protos |
| 483 | */ |
| 484 | |
| 485 | int session_enqueue_stream_connection (transport_connection_t * tc, |
| 486 | vlib_buffer_t * b, u32 offset, |
| 487 | u8 queue_event, u8 is_in_order); |
| 488 | int session_enqueue_dgram_connection (session_t * s, |
| 489 | session_dgram_hdr_t * hdr, |
| 490 | vlib_buffer_t * b, u8 proto, |
| 491 | u8 queue_event); |
Florin Coras | 00e01d3 | 2019-10-21 16:07:46 -0700 | [diff] [blame] | 492 | int session_stream_connect_notify (transport_connection_t * tc, |
| 493 | session_error_t err); |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 494 | int session_dgram_connect_notify (transport_connection_t * tc, |
| 495 | u32 old_thread_index, |
| 496 | session_t ** new_session); |
Florin Coras | a27a46e | 2019-02-18 13:02:28 -0800 | [diff] [blame] | 497 | int session_stream_accept_notify (transport_connection_t * tc); |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 498 | void session_transport_closing_notify (transport_connection_t * tc); |
| 499 | void session_transport_delete_notify (transport_connection_t * tc); |
Florin Coras | ea72764 | 2021-05-07 19:39:43 -0700 | [diff] [blame] | 500 | void session_half_open_delete_notify (transport_connection_t *tc); |
Florin Coras | bab6c52 | 2021-05-13 08:36:56 -0700 | [diff] [blame] | 501 | void session_half_open_migrate_notify (transport_connection_t *tc); |
| 502 | int session_half_open_migrated_notify (transport_connection_t *tc); |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 503 | void session_transport_closed_notify (transport_connection_t * tc); |
| 504 | void session_transport_reset_notify (transport_connection_t * tc); |
Florin Coras | c9940fc | 2019-02-05 20:55:11 -0800 | [diff] [blame] | 505 | int session_stream_accept (transport_connection_t * tc, u32 listener_index, |
Nathan Skrzypczak | 2f0f96b | 2019-06-13 10:14:28 +0200 | [diff] [blame] | 506 | u32 thread_index, u8 notify); |
Florin Coras | e759bb5 | 2020-04-08 01:55:39 +0000 | [diff] [blame] | 507 | int session_dgram_accept (transport_connection_t * tc, u32 listener_index, |
| 508 | u32 thread_index); |
Florin Coras | 07063b8 | 2020-03-13 04:44:51 +0000 | [diff] [blame] | 509 | /** |
| 510 | * Initialize session layer for given transport proto and ip version |
| 511 | * |
| 512 | * Allocates per session type (transport proto + ip version) data structures |
| 513 | * and adds arc from session queue node to session type output node. |
| 514 | * |
| 515 | * @param transport_proto transport proto to be registered |
| 516 | * @param vft virtual function table for transport |
| 517 | * @param is_ip4 flag that indicates if transports uses ipv4 |
| 518 | * as underlying network layer |
| 519 | * @param output_node output node for transport |
| 520 | */ |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 521 | void session_register_transport (transport_proto_t transport_proto, |
| 522 | const transport_proto_vft_t * vft, u8 is_ip4, |
| 523 | u32 output_node); |
Florin Coras | 07063b8 | 2020-03-13 04:44:51 +0000 | [diff] [blame] | 524 | transport_proto_t session_add_transport_proto (void); |
Florin Coras | 5384cca | 2022-01-20 18:10:26 -0800 | [diff] [blame] | 525 | void session_register_update_time_fn (session_update_time_fn fn, u8 is_add); |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 526 | int session_tx_fifo_peek_bytes (transport_connection_t * tc, u8 * buffer, |
| 527 | u32 offset, u32 max_bytes); |
| 528 | u32 session_tx_fifo_dequeue_drop (transport_connection_t * tc, u32 max_bytes); |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 529 | |
Steven Luong | d810a6e | 2022-10-25 13:09:11 -0700 | [diff] [blame^] | 530 | always_inline void |
| 531 | session_set_state (session_t *s, session_state_t session_state) |
| 532 | { |
| 533 | s->session_state = session_state; |
| 534 | SESSION_EVT (SESSION_EVT_STATE_CHANGE, s); |
| 535 | } |
| 536 | |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 537 | always_inline u32 |
Florin Coras | d2aab83 | 2018-05-22 11:39:59 -0700 | [diff] [blame] | 538 | transport_max_rx_enqueue (transport_connection_t * tc) |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 539 | { |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 540 | session_t *s = session_get (tc->s_index, tc->thread_index); |
Sirshak Das | 28aa539 | 2019-02-05 01:33:33 -0600 | [diff] [blame] | 541 | return svm_fifo_max_enqueue_prod (s->rx_fifo); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 542 | } |
| 543 | |
Florin Coras | e04c299 | 2017-03-01 08:17:34 -0800 | [diff] [blame] | 544 | always_inline u32 |
Florin Coras | 42ceddb | 2018-12-12 10:56:01 -0800 | [diff] [blame] | 545 | transport_max_tx_dequeue (transport_connection_t * tc) |
| 546 | { |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 547 | session_t *s = session_get (tc->s_index, tc->thread_index); |
Sirshak Das | 28aa539 | 2019-02-05 01:33:33 -0600 | [diff] [blame] | 548 | return svm_fifo_max_dequeue_cons (s->tx_fifo); |
Florin Coras | 42ceddb | 2018-12-12 10:56:01 -0800 | [diff] [blame] | 549 | } |
| 550 | |
| 551 | always_inline u32 |
Florin Coras | f65074e | 2019-03-31 17:17:11 -0700 | [diff] [blame] | 552 | transport_max_rx_dequeue (transport_connection_t * tc) |
| 553 | { |
| 554 | session_t *s = session_get (tc->s_index, tc->thread_index); |
| 555 | return svm_fifo_max_dequeue (s->rx_fifo); |
| 556 | } |
| 557 | |
| 558 | always_inline u32 |
Florin Coras | d2aab83 | 2018-05-22 11:39:59 -0700 | [diff] [blame] | 559 | transport_rx_fifo_size (transport_connection_t * tc) |
Florin Coras | e04c299 | 2017-03-01 08:17:34 -0800 | [diff] [blame] | 560 | { |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 561 | session_t *s = session_get (tc->s_index, tc->thread_index); |
Florin Coras | f22f4e5 | 2019-12-19 16:10:58 -0800 | [diff] [blame] | 562 | return svm_fifo_size (s->rx_fifo); |
Florin Coras | e04c299 | 2017-03-01 08:17:34 -0800 | [diff] [blame] | 563 | } |
| 564 | |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 565 | always_inline u32 |
Florin Coras | d2aab83 | 2018-05-22 11:39:59 -0700 | [diff] [blame] | 566 | transport_tx_fifo_size (transport_connection_t * tc) |
| 567 | { |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 568 | session_t *s = session_get (tc->s_index, tc->thread_index); |
Florin Coras | f22f4e5 | 2019-12-19 16:10:58 -0800 | [diff] [blame] | 569 | return svm_fifo_size (s->tx_fifo); |
Florin Coras | d2aab83 | 2018-05-22 11:39:59 -0700 | [diff] [blame] | 570 | } |
| 571 | |
Florin Coras | 7ac053b | 2018-11-05 15:57:21 -0800 | [diff] [blame] | 572 | always_inline u8 |
| 573 | transport_rx_fifo_has_ooo_data (transport_connection_t * tc) |
| 574 | { |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 575 | session_t *s = session_get (tc->c_index, tc->thread_index); |
| 576 | return svm_fifo_has_ooo_data (s->rx_fifo); |
Florin Coras | 7ac053b | 2018-11-05 15:57:21 -0800 | [diff] [blame] | 577 | } |
| 578 | |
Florin Coras | 5a41fd5 | 2021-04-19 10:17:26 -0700 | [diff] [blame] | 579 | always_inline void |
| 580 | transport_rx_fifo_req_deq_ntf (transport_connection_t *tc) |
| 581 | { |
| 582 | session_t *s = session_get (tc->s_index, tc->thread_index); |
| 583 | svm_fifo_add_want_deq_ntf (s->rx_fifo, SVM_FIFO_WANT_DEQ_NOTIF); |
| 584 | } |
| 585 | |
Florin Coras | a8e71c8 | 2019-10-22 19:01:39 -0700 | [diff] [blame] | 586 | always_inline clib_time_type_t |
Florin Coras | d67f112 | 2018-05-21 17:47:40 -0700 | [diff] [blame] | 587 | transport_time_now (u32 thread_index) |
| 588 | { |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 589 | return session_main.wrk[thread_index].last_vlib_time; |
Florin Coras | d67f112 | 2018-05-21 17:47:40 -0700 | [diff] [blame] | 590 | } |
| 591 | |
Florin Coras | a8e71c8 | 2019-10-22 19:01:39 -0700 | [diff] [blame] | 592 | always_inline clib_us_time_t |
| 593 | transport_us_time_now (u32 thread_index) |
| 594 | { |
| 595 | return session_main.wrk[thread_index].last_vlib_us_time; |
| 596 | } |
| 597 | |
Florin Coras | a39f472 | 2020-11-12 10:20:05 -0800 | [diff] [blame] | 598 | always_inline clib_time_type_t |
| 599 | transport_seconds_per_loop (u32 thread_index) |
| 600 | { |
| 601 | return session_main.wrk[thread_index].vm->seconds_per_loop; |
| 602 | } |
| 603 | |
Florin Coras | 3ec66b0 | 2018-08-23 16:27:05 -0700 | [diff] [blame] | 604 | always_inline void |
| 605 | transport_add_tx_event (transport_connection_t * tc) |
| 606 | { |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 607 | session_t *s = session_get (tc->s_index, tc->thread_index); |
| 608 | if (svm_fifo_has_event (s->tx_fifo)) |
Florin Coras | 3ec66b0 | 2018-08-23 16:27:05 -0700 | [diff] [blame] | 609 | return; |
Florin Coras | f6c4313 | 2019-03-01 12:41:21 -0800 | [diff] [blame] | 610 | session_send_io_evt_to_thread (s->tx_fifo, SESSION_IO_EVT_TX); |
Florin Coras | 3ec66b0 | 2018-08-23 16:27:05 -0700 | [diff] [blame] | 611 | } |
| 612 | |
Florin Coras | fb50bc3 | 2021-05-18 00:28:59 -0700 | [diff] [blame] | 613 | always_inline u32 |
| 614 | transport_cl_thread (void) |
| 615 | { |
| 616 | return session_main.transport_cl_thread; |
| 617 | } |
| 618 | |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 619 | /* |
| 620 | * Listen sessions |
| 621 | */ |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 622 | |
| 623 | always_inline u64 |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 624 | listen_session_get_handle (session_t * s) |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 625 | { |
Aloys Augustin | cdb7170 | 2019-04-08 17:54:39 +0200 | [diff] [blame] | 626 | ASSERT (s->session_state == SESSION_STATE_LISTENING || |
| 627 | session_get_transport_proto (s) == TRANSPORT_PROTO_QUIC); |
Florin Coras | 5c9083d | 2018-04-13 06:39:07 -0700 | [diff] [blame] | 628 | return session_handle (s); |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 629 | } |
| 630 | |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 631 | always_inline session_t * |
Florin Coras | f8f516a | 2018-02-08 15:10:09 -0800 | [diff] [blame] | 632 | listen_session_get_from_handle (session_handle_t handle) |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 633 | { |
Florin Coras | 5c9083d | 2018-04-13 06:39:07 -0700 | [diff] [blame] | 634 | return session_get_from_handle (handle); |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 635 | } |
| 636 | |
Florin Coras | 371ca50 | 2018-02-21 12:07:41 -0800 | [diff] [blame] | 637 | always_inline void |
Florin Coras | 5c9083d | 2018-04-13 06:39:07 -0700 | [diff] [blame] | 638 | listen_session_parse_handle (session_handle_t handle, u32 * index, |
| 639 | u32 * thread_index) |
Florin Coras | 371ca50 | 2018-02-21 12:07:41 -0800 | [diff] [blame] | 640 | { |
Florin Coras | 5c9083d | 2018-04-13 06:39:07 -0700 | [diff] [blame] | 641 | session_parse_handle (handle, index, thread_index); |
Florin Coras | 371ca50 | 2018-02-21 12:07:41 -0800 | [diff] [blame] | 642 | } |
| 643 | |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 644 | always_inline session_t * |
Florin Coras | ba7d8f5 | 2019-02-22 13:11:38 -0800 | [diff] [blame] | 645 | listen_session_alloc (u8 thread_index, session_type_t type) |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 646 | { |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 647 | session_t *s; |
Florin Coras | 5c9083d | 2018-04-13 06:39:07 -0700 | [diff] [blame] | 648 | s = session_alloc (thread_index); |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 649 | s->session_type = type; |
| 650 | s->session_state = SESSION_STATE_LISTENING; |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 651 | return s; |
| 652 | } |
| 653 | |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 654 | always_inline session_t * |
Florin Coras | ba7d8f5 | 2019-02-22 13:11:38 -0800 | [diff] [blame] | 655 | listen_session_get (u32 ls_index) |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 656 | { |
Florin Coras | ba7d8f5 | 2019-02-22 13:11:38 -0800 | [diff] [blame] | 657 | return session_get (ls_index, 0); |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 658 | } |
| 659 | |
| 660 | always_inline void |
Florin Coras | ba7d8f5 | 2019-02-22 13:11:38 -0800 | [diff] [blame] | 661 | listen_session_free (session_t * s) |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 662 | { |
Florin Coras | 87b7e3d | 2020-03-27 15:06:07 +0000 | [diff] [blame] | 663 | ASSERT (!s->rx_fifo); |
Florin Coras | 5c9083d | 2018-04-13 06:39:07 -0700 | [diff] [blame] | 664 | session_free (s); |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 665 | } |
| 666 | |
Florin Coras | ea72764 | 2021-05-07 19:39:43 -0700 | [diff] [blame] | 667 | always_inline session_t * |
| 668 | ho_session_alloc (void) |
| 669 | { |
| 670 | session_t *s; |
| 671 | ASSERT (vlib_get_thread_index () == 0); |
| 672 | s = session_alloc (0); |
| 673 | s->session_state = SESSION_STATE_CONNECTING; |
| 674 | s->flags |= SESSION_F_HALF_OPEN; |
Florin Coras | f8c8809 | 2022-02-09 19:17:02 -0800 | [diff] [blame] | 675 | /* Not ideal. Half-opens are only allocated from main with worker barrier |
| 676 | * but can be cleaned up, i.e., session_half_open_free, from main without |
| 677 | * a barrier. In debug images, the free_bitmap can grow while workers peek |
| 678 | * the sessions pool, e.g., session_half_open_migrate_notify, and as a |
| 679 | * result crash while validating the session. To avoid this, grow the bitmap |
| 680 | * now. */ |
| 681 | if (CLIB_DEBUG) |
| 682 | { |
| 683 | session_t *sp = session_main.wrk[0].sessions; |
| 684 | clib_bitmap_validate (pool_header (sp)->free_bitmap, s->session_index); |
| 685 | } |
Florin Coras | ea72764 | 2021-05-07 19:39:43 -0700 | [diff] [blame] | 686 | return s; |
| 687 | } |
| 688 | |
| 689 | always_inline session_t * |
| 690 | ho_session_get (u32 ho_index) |
| 691 | { |
| 692 | return session_get (ho_index, 0 /* half-open thread */); |
| 693 | } |
| 694 | |
| 695 | always_inline void |
| 696 | ho_session_free (session_t *s) |
| 697 | { |
| 698 | ASSERT (!s->rx_fifo && s->thread_index == 0); |
| 699 | session_free (s); |
| 700 | } |
| 701 | |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 702 | transport_connection_t *listen_session_get_transport (session_t * s); |
Florin Coras | 3cbc04b | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 703 | |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 704 | /* |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 705 | * Session layer functions |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 706 | */ |
| 707 | |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 708 | always_inline session_main_t * |
| 709 | vnet_get_session_main () |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 710 | { |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 711 | return &session_main; |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 712 | } |
| 713 | |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 714 | always_inline session_worker_t * |
| 715 | session_main_get_worker (u32 thread_index) |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 716 | { |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 717 | return &session_main.wrk[thread_index]; |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 718 | } |
| 719 | |
Florin Coras | 5bb23ec | 2019-08-31 09:45:13 -0700 | [diff] [blame] | 720 | static inline session_worker_t * |
| 721 | session_main_get_worker_if_valid (u32 thread_index) |
| 722 | { |
Nathan Skrzypczak | 901467e | 2019-12-04 11:02:28 +0100 | [diff] [blame] | 723 | if (thread_index > vec_len (session_main.wrk)) |
Florin Coras | 5bb23ec | 2019-08-31 09:45:13 -0700 | [diff] [blame] | 724 | return 0; |
| 725 | return &session_main.wrk[thread_index]; |
| 726 | } |
| 727 | |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 728 | always_inline svm_msg_q_t * |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 729 | session_main_get_vpp_event_queue (u32 thread_index) |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 730 | { |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 731 | return session_main.wrk[thread_index].vpp_event_queue; |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 732 | } |
Florin Coras | fd542f1 | 2018-05-16 19:28:24 -0700 | [diff] [blame] | 733 | |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 734 | always_inline u8 |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 735 | session_main_is_enabled () |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 736 | { |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 737 | return session_main.is_enabled == 1; |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 738 | } |
| 739 | |
Florin Coras | cea194d | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 740 | #define session_cli_return_if_not_enabled() \ |
| 741 | do { \ |
Florin Coras | 5bb23ec | 2019-08-31 09:45:13 -0700 | [diff] [blame] | 742 | if (!session_main.is_enabled) \ |
| 743 | return clib_error_return (0, "session layer is not enabled"); \ |
Florin Coras | cea194d | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 744 | } while (0) |
| 745 | |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 746 | int session_main_flush_enqueue_events (u8 proto, u32 thread_index); |
| 747 | int session_main_flush_all_enqueue_events (u8 transport_proto); |
Florin Coras | 5484daa | 2020-03-27 23:55:06 +0000 | [diff] [blame] | 748 | void session_queue_run_on_main_thread (vlib_main_t * vm); |
Florin Coras | 2a7ea2e | 2019-10-16 22:06:08 -0700 | [diff] [blame] | 749 | |
Florin Coras | 5484daa | 2020-03-27 23:55:06 +0000 | [diff] [blame] | 750 | /** |
| 751 | * Add session node pending buffer with custom node |
| 752 | * |
| 753 | * @param thread_index worker thread expected to send the buffer |
| 754 | * @param bi buffer index |
| 755 | * @param next_node next node edge index for buffer. Edge to next node |
| 756 | * must exist |
| 757 | */ |
Florin Coras | 2a7ea2e | 2019-10-16 22:06:08 -0700 | [diff] [blame] | 758 | always_inline void |
Florin Coras | 5484daa | 2020-03-27 23:55:06 +0000 | [diff] [blame] | 759 | session_add_pending_tx_buffer (u32 thread_index, u32 bi, u32 next_node) |
Florin Coras | 2a7ea2e | 2019-10-16 22:06:08 -0700 | [diff] [blame] | 760 | { |
| 761 | session_worker_t *wrk = session_main_get_worker (thread_index); |
| 762 | vec_add1 (wrk->pending_tx_buffers, bi); |
Florin Coras | 5484daa | 2020-03-27 23:55:06 +0000 | [diff] [blame] | 763 | vec_add1 (wrk->pending_tx_nexts, next_node); |
Florin Coras | 7da8829 | 2021-03-18 15:04:34 -0700 | [diff] [blame] | 764 | if (PREDICT_FALSE (wrk->state == SESSION_WRK_INTERRUPT)) |
| 765 | vlib_node_set_interrupt_pending (wrk->vm, session_queue_node.index); |
Florin Coras | 2a7ea2e | 2019-10-16 22:06:08 -0700 | [diff] [blame] | 766 | } |
| 767 | |
Florin Coras | 8f10b90 | 2021-04-02 18:32:00 -0700 | [diff] [blame] | 768 | always_inline void |
| 769 | session_wrk_update_time (session_worker_t *wrk, f64 now) |
| 770 | { |
| 771 | wrk->last_vlib_time = now; |
| 772 | wrk->last_vlib_us_time = wrk->last_vlib_time * CLIB_US_TIME_FREQ; |
| 773 | } |
| 774 | |
Florin Coras | 7da8829 | 2021-03-18 15:04:34 -0700 | [diff] [blame] | 775 | void session_wrk_enable_adaptive_mode (session_worker_t *wrk); |
Florin Coras | 8afe1b8 | 2021-11-17 23:38:54 -0800 | [diff] [blame] | 776 | fifo_segment_t *session_main_get_wrk_mqs_segment (void); |
Dave Barach | 2a86391 | 2017-11-28 10:11:42 -0500 | [diff] [blame] | 777 | void session_node_enable_disable (u8 is_en); |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 778 | clib_error_t *vnet_session_enable_disable (vlib_main_t * vm, u8 is_en); |
Damjan Marion | dd298e8 | 2022-10-12 16:02:18 +0200 | [diff] [blame] | 779 | void session_wrk_handle_evts_main_rpc (void *); |
Dave Barach | 2a86391 | 2017-11-28 10:11:42 -0500 | [diff] [blame] | 780 | |
Andreas Schultz | 1a8c437 | 2020-03-20 09:39:59 +0100 | [diff] [blame] | 781 | session_t *session_alloc_for_connection (transport_connection_t * tc); |
Florin Coras | 2c876f9 | 2021-05-10 21:12:27 -0700 | [diff] [blame] | 782 | session_t *session_alloc_for_half_open (transport_connection_t *tc); |
Andreas Schultz | 1a8c437 | 2020-03-20 09:39:59 +0100 | [diff] [blame] | 783 | |
Florin Coras | 53d8d4f | 2022-03-09 13:55:38 -0800 | [diff] [blame] | 784 | typedef void (pool_safe_realloc_rpc_fn) (void *rpc_args); |
| 785 | |
| 786 | typedef struct |
| 787 | { |
| 788 | u8 ph[STRUCT_OFFSET_OF (pool_header_t, max_elts) + 4]; |
| 789 | u32 flag; |
| 790 | } pool_safe_realloc_header_t; |
| 791 | |
| 792 | STATIC_ASSERT_SIZEOF (pool_safe_realloc_header_t, sizeof (pool_header_t)); |
| 793 | |
| 794 | #define POOL_REALLOC_SAFE_ELT_THRESH 32 |
| 795 | |
| 796 | #define pool_realloc_flag(PH) \ |
| 797 | ((pool_safe_realloc_header_t *) pool_header (PH))->flag |
| 798 | |
Florin Coras | d3915dc | 2022-03-31 15:42:17 -0700 | [diff] [blame] | 799 | typedef struct pool_realloc_rpc_args_ |
| 800 | { |
| 801 | void **pool; |
| 802 | uword elt_size; |
| 803 | uword align; |
| 804 | } pool_realloc_rpc_args_t; |
Florin Coras | 53d8d4f | 2022-03-09 13:55:38 -0800 | [diff] [blame] | 805 | |
| 806 | always_inline void |
Florin Coras | d3915dc | 2022-03-31 15:42:17 -0700 | [diff] [blame] | 807 | pool_program_safe_realloc_rpc (void *args) |
Florin Coras | 53d8d4f | 2022-03-09 13:55:38 -0800 | [diff] [blame] | 808 | { |
Florin Coras | d3915dc | 2022-03-31 15:42:17 -0700 | [diff] [blame] | 809 | vlib_main_t *vm = vlib_get_main (); |
| 810 | u32 free_elts, max_elts, n_alloc; |
| 811 | pool_realloc_rpc_args_t *pra; |
| 812 | |
| 813 | ASSERT (vlib_get_thread_index () == 0); |
| 814 | pra = (pool_realloc_rpc_args_t *) args; |
| 815 | |
| 816 | vlib_worker_thread_barrier_sync (vm); |
| 817 | |
| 818 | free_elts = _pool_free_elts (*pra->pool, pra->elt_size); |
| 819 | if (free_elts < POOL_REALLOC_SAFE_ELT_THRESH) |
| 820 | { |
| 821 | max_elts = _vec_max_len (*pra->pool, pra->elt_size); |
| 822 | n_alloc = clib_max (2 * max_elts, POOL_REALLOC_SAFE_ELT_THRESH); |
| 823 | _pool_alloc (pra->pool, free_elts + n_alloc, pra->align, 0, |
| 824 | pra->elt_size); |
| 825 | } |
| 826 | pool_realloc_flag (*pra->pool) = 0; |
| 827 | clib_mem_free (args); |
| 828 | |
| 829 | vlib_worker_thread_barrier_release (vm); |
| 830 | } |
| 831 | |
| 832 | always_inline void |
| 833 | pool_program_safe_realloc (void **p, u32 elt_size, u32 align) |
| 834 | { |
| 835 | pool_realloc_rpc_args_t *pra; |
| 836 | |
Florin Coras | 53d8d4f | 2022-03-09 13:55:38 -0800 | [diff] [blame] | 837 | /* Reuse pad as a realloc flag */ |
Florin Coras | d3915dc | 2022-03-31 15:42:17 -0700 | [diff] [blame] | 838 | if (pool_realloc_flag (*p)) |
Florin Coras | 53d8d4f | 2022-03-09 13:55:38 -0800 | [diff] [blame] | 839 | return; |
| 840 | |
Florin Coras | d3915dc | 2022-03-31 15:42:17 -0700 | [diff] [blame] | 841 | pra = clib_mem_alloc (sizeof (*pra)); |
| 842 | pra->pool = p; |
| 843 | pra->elt_size = elt_size; |
| 844 | pra->align = align; |
| 845 | pool_realloc_flag (*p) = 1; |
| 846 | |
| 847 | session_send_rpc_evt_to_thread (0 /* thread index */, |
| 848 | pool_program_safe_realloc_rpc, pra); |
Florin Coras | 53d8d4f | 2022-03-09 13:55:38 -0800 | [diff] [blame] | 849 | } |
| 850 | |
Florin Coras | 53d8d4f | 2022-03-09 13:55:38 -0800 | [diff] [blame] | 851 | #define pool_needs_realloc(P) \ |
| 852 | ((!P) || \ |
| 853 | (vec_len (pool_header (P)->free_indices) < POOL_REALLOC_SAFE_ELT_THRESH && \ |
| 854 | pool_free_elts (P) < POOL_REALLOC_SAFE_ELT_THRESH)) |
| 855 | |
Florin Coras | d3915dc | 2022-03-31 15:42:17 -0700 | [diff] [blame] | 856 | #define pool_get_aligned_safe(P, E, align) \ |
Florin Coras | 53d8d4f | 2022-03-09 13:55:38 -0800 | [diff] [blame] | 857 | do \ |
| 858 | { \ |
Florin Coras | 53d8d4f | 2022-03-09 13:55:38 -0800 | [diff] [blame] | 859 | if (PREDICT_FALSE (pool_needs_realloc (P))) \ |
| 860 | { \ |
| 861 | if (PREDICT_FALSE (!(P))) \ |
| 862 | { \ |
Florin Coras | d3915dc | 2022-03-31 15:42:17 -0700 | [diff] [blame] | 863 | pool_alloc_aligned (P, POOL_REALLOC_SAFE_ELT_THRESH, align); \ |
Florin Coras | 53d8d4f | 2022-03-09 13:55:38 -0800 | [diff] [blame] | 864 | } \ |
Florin Coras | 5b47a6d | 2022-03-26 01:25:02 -0700 | [diff] [blame] | 865 | else if (PREDICT_FALSE (!pool_free_elts (P))) \ |
Florin Coras | 53d8d4f | 2022-03-09 13:55:38 -0800 | [diff] [blame] | 866 | { \ |
Florin Coras | 4b20830 | 2022-03-30 13:50:19 -0700 | [diff] [blame] | 867 | vlib_workers_sync (); \ |
Florin Coras | 5b47a6d | 2022-03-26 01:25:02 -0700 | [diff] [blame] | 868 | pool_alloc_aligned (P, pool_max_len (P), align); \ |
Florin Coras | 4b20830 | 2022-03-30 13:50:19 -0700 | [diff] [blame] | 869 | vlib_workers_continue (); \ |
Florin Coras | 5b47a6d | 2022-03-26 01:25:02 -0700 | [diff] [blame] | 870 | ALWAYS_ASSERT (pool_free_elts (P) > 0); \ |
Florin Coras | 53d8d4f | 2022-03-09 13:55:38 -0800 | [diff] [blame] | 871 | } \ |
| 872 | else \ |
| 873 | { \ |
Florin Coras | d3915dc | 2022-03-31 15:42:17 -0700 | [diff] [blame] | 874 | pool_program_safe_realloc ((void **) &(P), sizeof ((P)[0]), \ |
| 875 | _vec_align (P, align)); \ |
Florin Coras | 53d8d4f | 2022-03-09 13:55:38 -0800 | [diff] [blame] | 876 | } \ |
| 877 | } \ |
| 878 | pool_get_aligned (P, E, align); \ |
| 879 | } \ |
| 880 | while (0) |
| 881 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 882 | #endif /* __included_session_h__ */ |
| 883 | |
| 884 | /* |
| 885 | * fd.io coding-style-patch-verification: ON |
| 886 | * |
| 887 | * Local Variables: |
| 888 | * eval: (c-set-style "gnu") |
| 889 | * End: |
| 890 | */ |