blob: 4984ff011d90968dc0db52f253e863347826a9eb [file] [log] [blame]
Dave Barach68b0fb02017-02-28 15:15:56 -05001/*
Florin Coras288eaab2019-02-03 15:26:14 -08002 * Copyright (c) 2017-2019 Cisco and/or its affiliates.
Dave Barach68b0fb02017-02-28 15:15:56 -05003 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15#ifndef __included_session_h__
16#define __included_session_h__
17
Florin Coras2062ec02019-07-15 13:15:18 -070018#include <vppinfra/llist.h>
Florin Coras288eaab2019-02-03 15:26:14 -080019#include <vnet/session/session_types.h>
Florin Coras04e53442017-07-16 17:12:15 -070020#include <vnet/session/session_lookup.h>
Florin Coras3e350af2017-03-30 02:54:28 -070021#include <vnet/session/session_debug.h>
Florin Coras288eaab2019-02-03 15:26:14 -080022#include <svm/message_queue.h>
Florin Coras04943b42020-12-25 11:45:40 -080023#include <svm/fifo_segment.h>
Dave Barach68b0fb02017-02-28 15:15:56 -050024
Florin Coras3e350af2017-03-30 02:54:28 -070025#define foreach_session_input_error \
Dave Barach68b0fb02017-02-28 15:15:56 -050026_(NO_SESSION, "No session drops") \
27_(NO_LISTENER, "No listener for dst port drops") \
28_(ENQUEUED, "Packets pushed into rx fifo") \
29_(NOT_READY, "Session not ready packets") \
30_(FIFO_FULL, "Packets dropped for lack of rx fifo space") \
31_(EVENT_FIFO_FULL, "Events not sent for lack of event fifo space") \
32_(API_QUEUE_FULL, "Sessions not created for lack of API queue space") \
Dave Barach68b0fb02017-02-28 15:15:56 -050033
34typedef enum
35{
36#define _(sym,str) SESSION_ERROR_##sym,
37 foreach_session_input_error
38#undef _
39 SESSION_N_ERROR,
Florin Coras00e01d32019-10-21 16:07:46 -070040} session_input_error_t;
Dave Barach68b0fb02017-02-28 15:15:56 -050041
Florin Coras8e43d042018-05-04 15:46:57 -070042typedef struct session_tx_context_
43{
44 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
Florin Coras288eaab2019-02-03 15:26:14 -080045 session_t *s;
Florin Coras8e43d042018-05-04 15:46:57 -070046 transport_proto_vft_t *transport_vft;
47 transport_connection_t *tc;
Florin Coras70f879d2020-03-13 17:54:42 +000048 transport_send_params_t sp;
Florin Coras8e43d042018-05-04 15:46:57 -070049 u32 max_dequeue;
Florin Coras8e43d042018-05-04 15:46:57 -070050 u32 left_to_snd;
Florin Coras8e43d042018-05-04 15:46:57 -070051 u32 max_len_to_snd;
52 u16 deq_per_first_buf;
53 u16 deq_per_buf;
Florin Corasf08f26d2018-05-10 13:20:47 -070054 u16 n_segs_per_evt;
Simon Zhangae16a982020-03-31 20:56:22 +080055 u16 n_bufs_needed;
Florin Coras8e43d042018-05-04 15:46:57 -070056 u8 n_bufs_per_seg;
57 CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
58 session_dgram_hdr_t hdr;
Florin Coras1d7a6632021-05-17 23:44:53 -070059
60 /** Vector of tx buffer free lists */
61 u32 *tx_buffers;
Florin Coras8e43d042018-05-04 15:46:57 -070062} session_tx_context_t;
63
Florin Coras2062ec02019-07-15 13:15:18 -070064typedef struct session_evt_elt
65{
66 clib_llist_anchor_t evt_list;
67 session_event_t evt;
68} session_evt_elt_t;
69
Florin Coras458089b2019-08-21 16:20:44 -070070typedef struct session_ctrl_evt_data_
71{
72 u8 data[SESSION_CTRL_MSG_MAX_SIZE];
73} session_evt_ctrl_data_t;
74
Florin Coras7da88292021-03-18 15:04:34 -070075typedef enum session_wrk_state_
76{
77 SESSION_WRK_POLLING,
78 SESSION_WRK_INTERRUPT,
79 SESSION_WRK_IDLE,
80} __clib_packed session_wrk_state_t;
81
82typedef enum session_wrk_flags_
83{
84 SESSION_WRK_F_ADAPTIVE = 1 << 0,
85} __clib_packed session_wrk_flag_t;
86
Florin Coras31c99552019-03-01 13:00:58 -080087typedef struct session_worker_
Dave Barach68b0fb02017-02-28 15:15:56 -050088{
Florin Coras5a7ca7b2018-10-30 12:01:48 -070089 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
Dave Barach68b0fb02017-02-28 15:15:56 -050090
Florin Coras5a7ca7b2018-10-30 12:01:48 -070091 /** Worker session pool */
Florin Coras288eaab2019-02-03 15:26:14 -080092 session_t *sessions;
Florin Coras3cbc04b2017-10-02 00:18:51 -070093
Florin Coras5a7ca7b2018-10-30 12:01:48 -070094 /** vpp event message queue for worker */
95 svm_msg_q_t *vpp_event_queue;
Florin Coras8e43d042018-05-04 15:46:57 -070096
Florin Corasd67f1122018-05-21 17:47:40 -070097 /** vlib_time_now last time around the track */
Florin Corasa8e71c82019-10-22 19:01:39 -070098 clib_time_type_t last_vlib_time;
99
100 /** vlib_time_now rounded to us precision and as u64 */
101 clib_us_time_t last_vlib_us_time;
Florin Corasd67f1122018-05-21 17:47:40 -0700102
Florin Coras60183db2019-07-20 15:53:16 -0700103 /** Convenience pointer to this worker's vlib_main */
104 vlib_main_t *vm;
105
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700106 /** Per-proto vector of sessions to enqueue */
Florin Coras07063b82020-03-13 04:44:51 +0000107 u32 **session_to_enqueue;
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700108
Florin Coras7da88292021-03-18 15:04:34 -0700109 /** Timerfd used to periodically signal wrk session queue node */
Florin Coras1c19aef2021-04-06 15:54:14 -0700110 int timerfd;
Florin Coras7da88292021-03-18 15:04:34 -0700111
112 /** Worker flags */
113 session_wrk_flag_t flags;
114
115 /** Worker state */
116 session_wrk_state_t state;
117
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700118 /** Context for session tx */
119 session_tx_context_t ctx;
120
Florin Coras2062ec02019-07-15 13:15:18 -0700121 /** Pool of session event list elements */
122 session_evt_elt_t *event_elts;
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700123
Florin Coras458089b2019-08-21 16:20:44 -0700124 /** Pool of ctrl events data buffers */
125 session_evt_ctrl_data_t *ctrl_evts_data;
126
Florin Corasb0ffbee2019-07-21 19:23:46 -0700127 /** Head of control events list */
128 clib_llist_index_t ctrl_head;
129
Florin Coras2062ec02019-07-15 13:15:18 -0700130 /** Head of list of elements */
131 clib_llist_index_t new_head;
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700132
Florin Coras2062ec02019-07-15 13:15:18 -0700133 /** Head of list of pending events */
Florin Coras60183db2019-07-20 15:53:16 -0700134 clib_llist_index_t old_head;
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700135
136 /** Peekers rw lock */
137 clib_rwlock_t peekers_rw_locks;
138
Florin Coras2a7ea2e2019-10-16 22:06:08 -0700139 /** Vector of buffers to be sent */
140 u32 *pending_tx_buffers;
141
142 /** Vector of nexts for the pending tx buffers */
143 u16 *pending_tx_nexts;
144
Florin Coras7da88292021-03-18 15:04:34 -0700145 /** Clib file for timerfd. Used only if adaptive mode is on */
146 uword timerfd_file;
147
Florin Corasaf972212021-05-05 09:54:00 -0700148 /** List of pending connects for first worker */
149 clib_llist_index_t pending_connects;
150
151 /** Flag that is set if main thread signaled to handle connects */
Florin Coras0f273392021-05-21 15:48:18 -0700152 u32 n_pending_connects;
Florin Corasaf972212021-05-05 09:54:00 -0700153
Florin Corasa48dffe2021-05-16 10:58:53 -0700154 /** Main thread loops in poll mode without a connect */
155 u32 no_connect_loops;
156
Florin Corascca96182019-07-19 07:34:13 -0700157#if SESSION_DEBUG
158 /** last event poll time by thread */
Florin Corasa8e71c82019-10-22 19:01:39 -0700159 clib_time_type_t last_event_poll;
Florin Corascca96182019-07-19 07:34:13 -0700160#endif
Florin Coras31c99552019-03-01 13:00:58 -0800161} session_worker_t;
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700162
Florin Coras60183db2019-07-20 15:53:16 -0700163typedef int (session_fifo_rx_fn) (session_worker_t * wrk,
Florin Coras1bcad5c2019-01-09 20:04:38 -0800164 vlib_node_runtime_t * node,
Florin Coras60183db2019-07-20 15:53:16 -0700165 session_evt_elt_t * e, int *n_tx_packets);
Florin Coras1bcad5c2019-01-09 20:04:38 -0800166
167extern session_fifo_rx_fn session_tx_fifo_peek_and_snd;
168extern session_fifo_rx_fn session_tx_fifo_dequeue_and_snd;
169extern session_fifo_rx_fn session_tx_fifo_dequeue_internal;
170
171u8 session_node_lookup_fifo_event (svm_fifo_t * f, session_event_t * e);
172
Florin Coras31c99552019-03-01 13:00:58 -0800173typedef struct session_main_
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700174{
175 /** Worker contexts */
Florin Coras31c99552019-03-01 13:00:58 -0800176 session_worker_t *wrk;
Dave Barach68b0fb02017-02-28 15:15:56 -0500177
Florin Coras6c10ab22020-11-08 16:50:39 -0800178 /** Event queues memfd segment */
Florin Coras04943b42020-12-25 11:45:40 -0800179 fifo_segment_t evt_qs_segment;
Florin Corasb384b542018-01-15 01:08:33 -0800180
Florin Coras93e65802017-11-29 00:07:11 -0500181 /** Unique segment name counter */
182 u32 unique_segment_name_counter;
183
184 /** Per transport rx function that can either dequeue or peek */
Florin Coras561af9b2017-12-09 10:19:43 -0800185 session_fifo_rx_fn **session_tx_fns;
186
187 /** Per session type output nodes. Could optimize to group nodes by
188 * fib but lookup would then require session type parsing in session node.
189 * Trade memory for speed, for now */
190 u32 *session_type_to_next;
Florin Coras93e65802017-11-29 00:07:11 -0500191
Florin Corasfb50bc32021-05-18 00:28:59 -0700192 /** Thread for cl and ho that rely on cl allocs */
193 u32 transport_cl_thread;
194
Florin Coras07063b82020-03-13 04:44:51 +0000195 transport_proto_t last_transport_proto_type;
196
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700197 /*
198 * Config parameters
199 */
200
Florin Coras93e65802017-11-29 00:07:11 -0500201 /** Session manager is enabled */
202 u8 is_enabled;
Florin Corase10da622020-10-25 14:00:29 -0700203
204 /** Session manager initialized (not necessarily enabled) */
205 u8 is_initialized;
206
Nathan Skrzypczak31bd0352019-11-07 17:55:01 +0100207 /** Enable session manager at startup */
208 u8 session_enable_asap;
Florin Coras93e65802017-11-29 00:07:11 -0500209
Florin Corase92c9462020-11-25 08:44:16 -0800210 /** Poll session node in main thread */
211 u8 poll_main;
212
Florin Coras41d5f542021-01-15 13:49:33 -0800213 /** Allocate private rx mqs for external apps */
214 u8 use_private_rx_mqs;
215
Florin Coras7da88292021-03-18 15:04:34 -0700216 /** Do not enable session queue node adaptive mode */
217 u8 no_adaptive;
218
Dave Barach10d8cc62017-05-30 09:30:07 -0400219 /** vpp fifo event queue configured length */
220 u32 configured_event_queue_length;
221
Florin Corasb384b542018-01-15 01:08:33 -0800222 /** Session ssvm segment configs*/
223 uword session_baseva;
Florin Corasa332c462018-01-31 06:52:17 -0800224 uword session_va_space_size;
Florin Corasb4c14912019-02-09 13:51:25 -0800225 uword evt_qs_segment_size;
Florin Corasb384b542018-01-15 01:08:33 -0800226
227 /** Session table size parameters */
Florin Coras66b11312017-07-31 17:18:03 -0700228 u32 configured_v4_session_table_buckets;
229 u32 configured_v4_session_table_memory;
230 u32 configured_v4_halfopen_table_buckets;
231 u32 configured_v4_halfopen_table_memory;
232 u32 configured_v6_session_table_buckets;
233 u32 configured_v6_session_table_memory;
234 u32 configured_v6_halfopen_table_buckets;
235 u32 configured_v6_halfopen_table_memory;
236
Florin Coras93e65802017-11-29 00:07:11 -0500237 /** Transport table (preallocation) size parameters */
238 u32 local_endpoints_table_memory;
239 u32 local_endpoints_table_buckets;
Florin Corase04c2992017-03-01 08:17:34 -0800240
Dave Barach2c25a622017-06-26 11:35:07 -0400241 /** Preallocate session config parameter */
242 u32 preallocated_sessions;
243
Florin Coras31c99552019-03-01 13:00:58 -0800244} session_main_t;
Dave Barach68b0fb02017-02-28 15:15:56 -0500245
Florin Coras31c99552019-03-01 13:00:58 -0800246extern session_main_t session_main;
Florin Corase04c2992017-03-01 08:17:34 -0800247extern vlib_node_registration_t session_queue_node;
Florin Corasfd542f12018-05-16 19:28:24 -0700248extern vlib_node_registration_t session_queue_process_node;
Florin Coras653e43f2019-03-04 10:56:23 -0800249extern vlib_node_registration_t session_queue_pre_input_node;
Florin Corasfd542f12018-05-16 19:28:24 -0700250
Florin Coras5484daa2020-03-27 23:55:06 +0000251typedef enum session_q_process_evt_
252{
253 SESSION_Q_PROCESS_RUN_ON_MAIN = 1,
254 SESSION_Q_PROCESS_STOP
255} session_q_process_evt_t;
Dave Barach68b0fb02017-02-28 15:15:56 -0500256
Florin Coras07063b82020-03-13 04:44:51 +0000257#define TRANSPORT_PROTO_INVALID (session_main.last_transport_proto_type + 1)
258#define TRANSPORT_N_PROTOS (session_main.last_transport_proto_type + 1)
259
Florin Coras2062ec02019-07-15 13:15:18 -0700260static inline void
Florin Coras60183db2019-07-20 15:53:16 -0700261session_evt_add_old (session_worker_t * wrk, session_evt_elt_t * elt)
Florin Coras2062ec02019-07-15 13:15:18 -0700262{
263 clib_llist_add_tail (wrk->event_elts, evt_list, elt,
Florin Coras46b8b1a2021-05-17 19:09:33 -0700264 clib_llist_elt (wrk->event_elts, wrk->old_head));
Florin Coras2062ec02019-07-15 13:15:18 -0700265}
266
Florin Corasdd97a482019-11-02 14:32:52 -0700267static inline void
268session_evt_add_head_old (session_worker_t * wrk, session_evt_elt_t * elt)
269{
270 clib_llist_add (wrk->event_elts, evt_list, elt,
Florin Coras46b8b1a2021-05-17 19:09:33 -0700271 clib_llist_elt (wrk->event_elts, wrk->old_head));
Florin Corasdd97a482019-11-02 14:32:52 -0700272}
273
274
Florin Coras458089b2019-08-21 16:20:44 -0700275static inline u32
276session_evt_ctrl_data_alloc (session_worker_t * wrk)
277{
278 session_evt_ctrl_data_t *data;
279 pool_get (wrk->ctrl_evts_data, data);
280 return (data - wrk->ctrl_evts_data);
281}
282
Florin Corasb0ffbee2019-07-21 19:23:46 -0700283static inline session_evt_elt_t *
284session_evt_alloc_ctrl (session_worker_t * wrk)
Florin Coras2062ec02019-07-15 13:15:18 -0700285{
Florin Corasb0ffbee2019-07-21 19:23:46 -0700286 session_evt_elt_t *elt;
Florin Coras46b8b1a2021-05-17 19:09:33 -0700287 clib_llist_get (wrk->event_elts, elt);
Florin Coras2062ec02019-07-15 13:15:18 -0700288 clib_llist_add_tail (wrk->event_elts, evt_list, elt,
Florin Coras46b8b1a2021-05-17 19:09:33 -0700289 clib_llist_elt (wrk->event_elts, wrk->ctrl_head));
Florin Corasb0ffbee2019-07-21 19:23:46 -0700290 return elt;
Florin Coras2062ec02019-07-15 13:15:18 -0700291}
292
Florin Coras458089b2019-08-21 16:20:44 -0700293static inline void *
294session_evt_ctrl_data (session_worker_t * wrk, session_evt_elt_t * elt)
295{
296 return (void *) (pool_elt_at_index (wrk->ctrl_evts_data,
297 elt->evt.ctrl_data_index));
298}
299
300static inline void
301session_evt_ctrl_data_free (session_worker_t * wrk, session_evt_elt_t * elt)
302{
303 ASSERT (elt->evt.event_type > SESSION_IO_EVT_BUILTIN_TX);
304 pool_put_index (wrk->ctrl_evts_data, elt->evt.ctrl_data_index);
305}
306
Florin Coras26dd6de2019-07-23 23:54:47 -0700307static inline session_evt_elt_t *
308session_evt_alloc_new (session_worker_t * wrk)
309{
310 session_evt_elt_t *elt;
Florin Coras46b8b1a2021-05-17 19:09:33 -0700311 clib_llist_get (wrk->event_elts, elt);
Florin Coras26dd6de2019-07-23 23:54:47 -0700312 clib_llist_add_tail (wrk->event_elts, evt_list, elt,
Florin Coras46b8b1a2021-05-17 19:09:33 -0700313 clib_llist_elt (wrk->event_elts, wrk->new_head));
Florin Coras26dd6de2019-07-23 23:54:47 -0700314 return elt;
315}
316
317static inline session_evt_elt_t *
318session_evt_alloc_old (session_worker_t * wrk)
319{
320 session_evt_elt_t *elt;
Florin Coras46b8b1a2021-05-17 19:09:33 -0700321 clib_llist_get (wrk->event_elts, elt);
Florin Coras26dd6de2019-07-23 23:54:47 -0700322 clib_llist_add_tail (wrk->event_elts, evt_list, elt,
Florin Coras46b8b1a2021-05-17 19:09:33 -0700323 clib_llist_elt (wrk->event_elts, wrk->old_head));
Florin Coras26dd6de2019-07-23 23:54:47 -0700324 return elt;
325}
326
Florin Coras41d5f542021-01-15 13:49:33 -0800327int session_wrk_handle_mq (session_worker_t *wrk, svm_msg_q_t *mq);
328
Florin Coras288eaab2019-02-03 15:26:14 -0800329session_t *session_alloc (u32 thread_index);
Florin Coras288eaab2019-02-03 15:26:14 -0800330void session_free (session_t * s);
331void session_free_w_fifos (session_t * s);
Florin Corasea727642021-05-07 19:39:43 -0700332void session_cleanup_half_open (session_handle_t ho_handle);
Srikanth Akulae140d5d2019-11-18 11:49:58 -0800333u8 session_is_valid (u32 si, u8 thread_index);
Florin Coras3cbc04b2017-10-02 00:18:51 -0700334
Florin Coras288eaab2019-02-03 15:26:14 -0800335always_inline session_t *
Florin Corascea194d2017-10-02 00:18:51 -0700336session_get (u32 si, u32 thread_index)
Dave Barach68b0fb02017-02-28 15:15:56 -0500337{
Florin Coras31c99552019-03-01 13:00:58 -0800338 ASSERT (session_is_valid (si, thread_index));
339 return pool_elt_at_index (session_main.wrk[thread_index].sessions, si);
Dave Barach68b0fb02017-02-28 15:15:56 -0500340}
341
Florin Coras288eaab2019-02-03 15:26:14 -0800342always_inline session_t *
Florin Coras3cbc04b2017-10-02 00:18:51 -0700343session_get_if_valid (u64 si, u32 thread_index)
Dave Barach68b0fb02017-02-28 15:15:56 -0500344{
Florin Coras31c99552019-03-01 13:00:58 -0800345 if (thread_index >= vec_len (session_main.wrk))
Dave Barach68b0fb02017-02-28 15:15:56 -0500346 return 0;
347
Florin Coras31c99552019-03-01 13:00:58 -0800348 if (pool_is_free_index (session_main.wrk[thread_index].sessions, si))
Dave Barach68b0fb02017-02-28 15:15:56 -0500349 return 0;
350
Florin Coras31c99552019-03-01 13:00:58 -0800351 ASSERT (session_is_valid (si, thread_index));
352 return pool_elt_at_index (session_main.wrk[thread_index].sessions, si);
Dave Barach68b0fb02017-02-28 15:15:56 -0500353}
354
Florin Coras288eaab2019-02-03 15:26:14 -0800355always_inline session_t *
Florin Corasf8f516a2018-02-08 15:10:09 -0800356session_get_from_handle (session_handle_t handle)
Florin Coras6cf30ad2017-04-04 23:08:23 -0700357{
Florin Coras31c99552019-03-01 13:00:58 -0800358 session_main_t *smm = &session_main;
Florin Corasf8f516a2018-02-08 15:10:09 -0800359 u32 session_index, thread_index;
360 session_parse_handle (handle, &session_index, &thread_index);
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700361 return pool_elt_at_index (smm->wrk[thread_index].sessions, session_index);
Florin Corasf8f516a2018-02-08 15:10:09 -0800362}
363
Florin Coras288eaab2019-02-03 15:26:14 -0800364always_inline session_t *
Florin Corasf8f516a2018-02-08 15:10:09 -0800365session_get_from_handle_if_valid (session_handle_t handle)
366{
367 u32 session_index, thread_index;
368 session_parse_handle (handle, &session_index, &thread_index);
369 return session_get_if_valid (session_index, thread_index);
370}
371
Florin Coras31c99552019-03-01 13:00:58 -0800372u64 session_segment_handle (session_t * s);
Florin Corasfa76a762018-11-29 12:40:10 -0800373
Florin Coras3cbc04b2017-10-02 00:18:51 -0700374/**
375 * Acquires a lock that blocks a session pool from expanding.
376 *
377 * This is typically used for safely peeking into other threads'
378 * pools in order to clone elements. Lock should be dropped as soon
379 * as possible by calling @ref session_pool_remove_peeker.
380 *
381 * NOTE: Avoid using pool_elt_at_index while the lock is held because
382 * it may lead to free elt bitmap expansion/contraction!
383 */
384always_inline void
385session_pool_add_peeker (u32 thread_index)
386{
Florin Coras31c99552019-03-01 13:00:58 -0800387 session_worker_t *wrk = &session_main.wrk[thread_index];
Florin Coras3cbc04b2017-10-02 00:18:51 -0700388 if (thread_index == vlib_get_thread_index ())
389 return;
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700390 clib_rwlock_reader_lock (&wrk->peekers_rw_locks);
Florin Coras3cbc04b2017-10-02 00:18:51 -0700391}
392
393always_inline void
394session_pool_remove_peeker (u32 thread_index)
395{
Florin Coras31c99552019-03-01 13:00:58 -0800396 session_worker_t *wrk = &session_main.wrk[thread_index];
Florin Coras3cbc04b2017-10-02 00:18:51 -0700397 if (thread_index == vlib_get_thread_index ())
398 return;
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700399 clib_rwlock_reader_unlock (&wrk->peekers_rw_locks);
Florin Coras3cbc04b2017-10-02 00:18:51 -0700400}
401
402/**
403 * Get session from handle and 'lock' pool resize if not in same thread
404 *
405 * Caller should drop the peek 'lock' as soon as possible.
406 */
Florin Coras288eaab2019-02-03 15:26:14 -0800407always_inline session_t *
Florin Coras3cbc04b2017-10-02 00:18:51 -0700408session_get_from_handle_safe (u64 handle)
409{
Florin Coras3cbc04b2017-10-02 00:18:51 -0700410 u32 thread_index = session_thread_from_handle (handle);
Florin Coras31c99552019-03-01 13:00:58 -0800411 session_worker_t *wrk = &session_main.wrk[thread_index];
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700412
Florin Coras3cbc04b2017-10-02 00:18:51 -0700413 if (thread_index == vlib_get_thread_index ())
414 {
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700415 return pool_elt_at_index (wrk->sessions,
Florin Coras3cbc04b2017-10-02 00:18:51 -0700416 session_index_from_handle (handle));
417 }
418 else
419 {
420 session_pool_add_peeker (thread_index);
421 /* Don't use pool_elt_at index. See @ref session_pool_add_peeker */
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700422 return wrk->sessions + session_index_from_handle (handle);
Florin Coras3cbc04b2017-10-02 00:18:51 -0700423 }
424}
425
Dave Barach68b0fb02017-02-28 15:15:56 -0500426always_inline u32
Florin Coras1ee78302019-02-05 15:51:15 -0800427session_get_index (session_t * s)
428{
Florin Coras31c99552019-03-01 13:00:58 -0800429 return (s - session_main.wrk[s->thread_index].sessions);
Florin Coras1ee78302019-02-05 15:51:15 -0800430}
431
432always_inline session_t *
433session_clone_safe (u32 session_index, u32 thread_index)
434{
435 session_t *old_s, *new_s;
436 u32 current_thread_index = vlib_get_thread_index ();
437
438 /* If during the memcpy pool is reallocated AND the memory allocator
439 * decides to give the old chunk of memory to somebody in a hurry to
440 * scribble something on it, we have a problem. So add this thread as
441 * a session pool peeker.
442 */
443 session_pool_add_peeker (thread_index);
444 new_s = session_alloc (current_thread_index);
Florin Coras31c99552019-03-01 13:00:58 -0800445 old_s = session_main.wrk[thread_index].sessions + session_index;
Florin Coras1ee78302019-02-05 15:51:15 -0800446 clib_memcpy_fast (new_s, old_s, sizeof (*new_s));
447 session_pool_remove_peeker (thread_index);
448 new_s->thread_index = current_thread_index;
449 new_s->session_index = session_get_index (new_s);
450 return new_s;
451}
452
Florin Coras89a9f612021-05-11 11:55:07 -0700453int session_open (session_endpoint_cfg_t *sep, session_handle_t *rsh);
Florin Coras1ee78302019-02-05 15:51:15 -0800454int session_listen (session_t * s, session_endpoint_cfg_t * sep);
455int session_stop_listen (session_t * s);
liuyacan534468e2021-05-09 03:50:40 +0000456void session_half_close (session_t *s);
Florin Coras1ee78302019-02-05 15:51:15 -0800457void session_close (session_t * s);
Florin Corasdfb3b872019-08-16 17:48:44 -0700458void session_reset (session_t * s);
liuyacan534468e2021-05-09 03:50:40 +0000459void session_transport_half_close (session_t *s);
Florin Coras1ee78302019-02-05 15:51:15 -0800460void session_transport_close (session_t * s);
Florin Corasdfb3b872019-08-16 17:48:44 -0700461void session_transport_reset (session_t * s);
Florin Coras1ee78302019-02-05 15:51:15 -0800462void session_transport_cleanup (session_t * s);
463int session_send_io_evt_to_thread (svm_fifo_t * f,
464 session_evt_type_t evt_type);
Florin Coras653e43f2019-03-04 10:56:23 -0800465int session_enqueue_notify (session_t * s);
Florin Coras1ee78302019-02-05 15:51:15 -0800466int session_dequeue_notify (session_t * s);
467int session_send_io_evt_to_thread_custom (void *data, u32 thread_index,
468 session_evt_type_t evt_type);
469void session_send_rpc_evt_to_thread (u32 thread_index, void *fp,
470 void *rpc_args);
Nathan Skrzypczak60f3e652019-03-19 13:57:31 +0100471void session_send_rpc_evt_to_thread_force (u32 thread_index, void *fp,
472 void *rpc_args);
Florin Coras26dd6de2019-07-23 23:54:47 -0700473void session_add_self_custom_tx_evt (transport_connection_t * tc,
474 u8 has_prio);
Florin Coras70f879d2020-03-13 17:54:42 +0000475void sesssion_reschedule_tx (transport_connection_t * tc);
Florin Coras1ee78302019-02-05 15:51:15 -0800476transport_connection_t *session_get_transport (session_t * s);
Florin Coras09d18c22019-04-24 11:10:02 -0700477void session_get_endpoint (session_t * s, transport_endpoint_t * tep,
478 u8 is_lcl);
Florin Coras04ae8272021-04-12 19:55:37 -0700479int session_transport_attribute (session_t *s, u8 is_get,
480 transport_endpt_attr_t *attr);
Florin Coras1ee78302019-02-05 15:51:15 -0800481
Florin Coras31c99552019-03-01 13:00:58 -0800482u8 *format_session (u8 * s, va_list * args);
483uword unformat_session (unformat_input_t * input, va_list * args);
Florin Coras1ee78302019-02-05 15:51:15 -0800484uword unformat_transport_connection (unformat_input_t * input,
485 va_list * args);
486
487/*
488 * Interface to transport protos
489 */
490
491int session_enqueue_stream_connection (transport_connection_t * tc,
492 vlib_buffer_t * b, u32 offset,
493 u8 queue_event, u8 is_in_order);
494int session_enqueue_dgram_connection (session_t * s,
495 session_dgram_hdr_t * hdr,
496 vlib_buffer_t * b, u8 proto,
497 u8 queue_event);
Florin Coras00e01d32019-10-21 16:07:46 -0700498int session_stream_connect_notify (transport_connection_t * tc,
499 session_error_t err);
Florin Coras1ee78302019-02-05 15:51:15 -0800500int session_dgram_connect_notify (transport_connection_t * tc,
501 u32 old_thread_index,
502 session_t ** new_session);
Florin Corasa27a46e2019-02-18 13:02:28 -0800503int session_stream_accept_notify (transport_connection_t * tc);
Florin Coras1ee78302019-02-05 15:51:15 -0800504void session_transport_closing_notify (transport_connection_t * tc);
505void session_transport_delete_notify (transport_connection_t * tc);
Florin Corasea727642021-05-07 19:39:43 -0700506void session_half_open_delete_notify (transport_connection_t *tc);
Florin Corasbab6c522021-05-13 08:36:56 -0700507void session_half_open_migrate_notify (transport_connection_t *tc);
508int session_half_open_migrated_notify (transport_connection_t *tc);
Florin Coras1ee78302019-02-05 15:51:15 -0800509void session_transport_closed_notify (transport_connection_t * tc);
510void session_transport_reset_notify (transport_connection_t * tc);
Florin Corasc9940fc2019-02-05 20:55:11 -0800511int session_stream_accept (transport_connection_t * tc, u32 listener_index,
Nathan Skrzypczak2f0f96b2019-06-13 10:14:28 +0200512 u32 thread_index, u8 notify);
Florin Corase759bb52020-04-08 01:55:39 +0000513int session_dgram_accept (transport_connection_t * tc, u32 listener_index,
514 u32 thread_index);
Florin Coras07063b82020-03-13 04:44:51 +0000515/**
516 * Initialize session layer for given transport proto and ip version
517 *
518 * Allocates per session type (transport proto + ip version) data structures
519 * and adds arc from session queue node to session type output node.
520 *
521 * @param transport_proto transport proto to be registered
522 * @param vft virtual function table for transport
523 * @param is_ip4 flag that indicates if transports uses ipv4
524 * as underlying network layer
525 * @param output_node output node for transport
526 */
Florin Coras1ee78302019-02-05 15:51:15 -0800527void session_register_transport (transport_proto_t transport_proto,
528 const transport_proto_vft_t * vft, u8 is_ip4,
529 u32 output_node);
Florin Coras07063b82020-03-13 04:44:51 +0000530transport_proto_t session_add_transport_proto (void);
Florin Coras31c99552019-03-01 13:00:58 -0800531int session_tx_fifo_peek_bytes (transport_connection_t * tc, u8 * buffer,
532 u32 offset, u32 max_bytes);
533u32 session_tx_fifo_dequeue_drop (transport_connection_t * tc, u32 max_bytes);
Florin Coras1ee78302019-02-05 15:51:15 -0800534
535always_inline u32
Florin Corasd2aab832018-05-22 11:39:59 -0700536transport_max_rx_enqueue (transport_connection_t * tc)
Dave Barach68b0fb02017-02-28 15:15:56 -0500537{
Florin Coras288eaab2019-02-03 15:26:14 -0800538 session_t *s = session_get (tc->s_index, tc->thread_index);
Sirshak Das28aa5392019-02-05 01:33:33 -0600539 return svm_fifo_max_enqueue_prod (s->rx_fifo);
Dave Barach68b0fb02017-02-28 15:15:56 -0500540}
541
Florin Corase04c2992017-03-01 08:17:34 -0800542always_inline u32
Florin Coras42ceddb2018-12-12 10:56:01 -0800543transport_max_tx_dequeue (transport_connection_t * tc)
544{
Florin Coras288eaab2019-02-03 15:26:14 -0800545 session_t *s = session_get (tc->s_index, tc->thread_index);
Sirshak Das28aa5392019-02-05 01:33:33 -0600546 return svm_fifo_max_dequeue_cons (s->tx_fifo);
Florin Coras42ceddb2018-12-12 10:56:01 -0800547}
548
549always_inline u32
Florin Corasf65074e2019-03-31 17:17:11 -0700550transport_max_rx_dequeue (transport_connection_t * tc)
551{
552 session_t *s = session_get (tc->s_index, tc->thread_index);
553 return svm_fifo_max_dequeue (s->rx_fifo);
554}
555
556always_inline u32
Florin Corasd2aab832018-05-22 11:39:59 -0700557transport_rx_fifo_size (transport_connection_t * tc)
Florin Corase04c2992017-03-01 08:17:34 -0800558{
Florin Coras288eaab2019-02-03 15:26:14 -0800559 session_t *s = session_get (tc->s_index, tc->thread_index);
Florin Corasf22f4e52019-12-19 16:10:58 -0800560 return svm_fifo_size (s->rx_fifo);
Florin Corase04c2992017-03-01 08:17:34 -0800561}
562
Florin Coras3cbc04b2017-10-02 00:18:51 -0700563always_inline u32
Florin Corasd2aab832018-05-22 11:39:59 -0700564transport_tx_fifo_size (transport_connection_t * tc)
565{
Florin Coras288eaab2019-02-03 15:26:14 -0800566 session_t *s = session_get (tc->s_index, tc->thread_index);
Florin Corasf22f4e52019-12-19 16:10:58 -0800567 return svm_fifo_size (s->tx_fifo);
Florin Corasd2aab832018-05-22 11:39:59 -0700568}
569
Florin Coras7ac053b2018-11-05 15:57:21 -0800570always_inline u8
571transport_rx_fifo_has_ooo_data (transport_connection_t * tc)
572{
Florin Coras288eaab2019-02-03 15:26:14 -0800573 session_t *s = session_get (tc->c_index, tc->thread_index);
574 return svm_fifo_has_ooo_data (s->rx_fifo);
Florin Coras7ac053b2018-11-05 15:57:21 -0800575}
576
Florin Coras5a41fd52021-04-19 10:17:26 -0700577always_inline void
578transport_rx_fifo_req_deq_ntf (transport_connection_t *tc)
579{
580 session_t *s = session_get (tc->s_index, tc->thread_index);
581 svm_fifo_add_want_deq_ntf (s->rx_fifo, SVM_FIFO_WANT_DEQ_NOTIF);
582}
583
Florin Corasa8e71c82019-10-22 19:01:39 -0700584always_inline clib_time_type_t
Florin Corasd67f1122018-05-21 17:47:40 -0700585transport_time_now (u32 thread_index)
586{
Florin Coras31c99552019-03-01 13:00:58 -0800587 return session_main.wrk[thread_index].last_vlib_time;
Florin Corasd67f1122018-05-21 17:47:40 -0700588}
589
Florin Corasa8e71c82019-10-22 19:01:39 -0700590always_inline clib_us_time_t
591transport_us_time_now (u32 thread_index)
592{
593 return session_main.wrk[thread_index].last_vlib_us_time;
594}
595
Florin Corasa39f4722020-11-12 10:20:05 -0800596always_inline clib_time_type_t
597transport_seconds_per_loop (u32 thread_index)
598{
599 return session_main.wrk[thread_index].vm->seconds_per_loop;
600}
601
Florin Coras3ec66b02018-08-23 16:27:05 -0700602always_inline void
603transport_add_tx_event (transport_connection_t * tc)
604{
Florin Coras288eaab2019-02-03 15:26:14 -0800605 session_t *s = session_get (tc->s_index, tc->thread_index);
606 if (svm_fifo_has_event (s->tx_fifo))
Florin Coras3ec66b02018-08-23 16:27:05 -0700607 return;
Florin Corasf6c43132019-03-01 12:41:21 -0800608 session_send_io_evt_to_thread (s->tx_fifo, SESSION_IO_EVT_TX);
Florin Coras3ec66b02018-08-23 16:27:05 -0700609}
610
Florin Corasfb50bc32021-05-18 00:28:59 -0700611always_inline u32
612transport_cl_thread (void)
613{
614 return session_main.transport_cl_thread;
615}
616
Florin Coras1ee78302019-02-05 15:51:15 -0800617/*
618 * Listen sessions
619 */
Florin Coras6cf30ad2017-04-04 23:08:23 -0700620
621always_inline u64
Florin Coras288eaab2019-02-03 15:26:14 -0800622listen_session_get_handle (session_t * s)
Florin Coras6cf30ad2017-04-04 23:08:23 -0700623{
Aloys Augustincdb71702019-04-08 17:54:39 +0200624 ASSERT (s->session_state == SESSION_STATE_LISTENING ||
625 session_get_transport_proto (s) == TRANSPORT_PROTO_QUIC);
Florin Coras5c9083d2018-04-13 06:39:07 -0700626 return session_handle (s);
Florin Coras6cf30ad2017-04-04 23:08:23 -0700627}
628
Florin Coras288eaab2019-02-03 15:26:14 -0800629always_inline session_t *
Florin Corasf8f516a2018-02-08 15:10:09 -0800630listen_session_get_from_handle (session_handle_t handle)
Florin Coras6cf30ad2017-04-04 23:08:23 -0700631{
Florin Coras5c9083d2018-04-13 06:39:07 -0700632 return session_get_from_handle (handle);
Florin Coras6cf30ad2017-04-04 23:08:23 -0700633}
634
Florin Coras371ca502018-02-21 12:07:41 -0800635always_inline void
Florin Coras5c9083d2018-04-13 06:39:07 -0700636listen_session_parse_handle (session_handle_t handle, u32 * index,
637 u32 * thread_index)
Florin Coras371ca502018-02-21 12:07:41 -0800638{
Florin Coras5c9083d2018-04-13 06:39:07 -0700639 session_parse_handle (handle, index, thread_index);
Florin Coras371ca502018-02-21 12:07:41 -0800640}
641
Florin Coras288eaab2019-02-03 15:26:14 -0800642always_inline session_t *
Florin Corasba7d8f52019-02-22 13:11:38 -0800643listen_session_alloc (u8 thread_index, session_type_t type)
Florin Coras6cf30ad2017-04-04 23:08:23 -0700644{
Florin Coras288eaab2019-02-03 15:26:14 -0800645 session_t *s;
Florin Coras5c9083d2018-04-13 06:39:07 -0700646 s = session_alloc (thread_index);
Florin Coras6cf30ad2017-04-04 23:08:23 -0700647 s->session_type = type;
648 s->session_state = SESSION_STATE_LISTENING;
Florin Coras6cf30ad2017-04-04 23:08:23 -0700649 return s;
650}
651
Florin Coras288eaab2019-02-03 15:26:14 -0800652always_inline session_t *
Florin Corasba7d8f52019-02-22 13:11:38 -0800653listen_session_get (u32 ls_index)
Florin Coras6cf30ad2017-04-04 23:08:23 -0700654{
Florin Corasba7d8f52019-02-22 13:11:38 -0800655 return session_get (ls_index, 0);
Florin Coras6cf30ad2017-04-04 23:08:23 -0700656}
657
658always_inline void
Florin Corasba7d8f52019-02-22 13:11:38 -0800659listen_session_free (session_t * s)
Florin Coras6cf30ad2017-04-04 23:08:23 -0700660{
Florin Coras87b7e3d2020-03-27 15:06:07 +0000661 ASSERT (!s->rx_fifo);
Florin Coras5c9083d2018-04-13 06:39:07 -0700662 session_free (s);
Florin Coras6cf30ad2017-04-04 23:08:23 -0700663}
664
Florin Corasea727642021-05-07 19:39:43 -0700665always_inline session_t *
666ho_session_alloc (void)
667{
668 session_t *s;
669 ASSERT (vlib_get_thread_index () == 0);
670 s = session_alloc (0);
671 s->session_state = SESSION_STATE_CONNECTING;
672 s->flags |= SESSION_F_HALF_OPEN;
673 return s;
674}
675
676always_inline session_t *
677ho_session_get (u32 ho_index)
678{
679 return session_get (ho_index, 0 /* half-open thread */);
680}
681
682always_inline void
683ho_session_free (session_t *s)
684{
685 ASSERT (!s->rx_fifo && s->thread_index == 0);
686 session_free (s);
687}
688
Florin Coras288eaab2019-02-03 15:26:14 -0800689transport_connection_t *listen_session_get_transport (session_t * s);
Florin Coras3cbc04b2017-10-02 00:18:51 -0700690
Florin Coras1ee78302019-02-05 15:51:15 -0800691/*
Florin Coras31c99552019-03-01 13:00:58 -0800692 * Session layer functions
Florin Coras1ee78302019-02-05 15:51:15 -0800693 */
694
Florin Coras31c99552019-03-01 13:00:58 -0800695always_inline session_main_t *
696vnet_get_session_main ()
Florin Coras1ee78302019-02-05 15:51:15 -0800697{
Florin Coras31c99552019-03-01 13:00:58 -0800698 return &session_main;
Florin Coras1ee78302019-02-05 15:51:15 -0800699}
700
Florin Coras31c99552019-03-01 13:00:58 -0800701always_inline session_worker_t *
702session_main_get_worker (u32 thread_index)
Florin Coras1ee78302019-02-05 15:51:15 -0800703{
Florin Coras31c99552019-03-01 13:00:58 -0800704 return &session_main.wrk[thread_index];
Florin Coras1ee78302019-02-05 15:51:15 -0800705}
706
Florin Coras5bb23ec2019-08-31 09:45:13 -0700707static inline session_worker_t *
708session_main_get_worker_if_valid (u32 thread_index)
709{
Nathan Skrzypczak901467e2019-12-04 11:02:28 +0100710 if (thread_index > vec_len (session_main.wrk))
Florin Coras5bb23ec2019-08-31 09:45:13 -0700711 return 0;
712 return &session_main.wrk[thread_index];
713}
714
Florin Coras1ee78302019-02-05 15:51:15 -0800715always_inline svm_msg_q_t *
Florin Coras31c99552019-03-01 13:00:58 -0800716session_main_get_vpp_event_queue (u32 thread_index)
Florin Coras1ee78302019-02-05 15:51:15 -0800717{
Florin Coras31c99552019-03-01 13:00:58 -0800718 return session_main.wrk[thread_index].vpp_event_queue;
Florin Coras1ee78302019-02-05 15:51:15 -0800719}
Florin Corasfd542f12018-05-16 19:28:24 -0700720
Florin Coras6cf30ad2017-04-04 23:08:23 -0700721always_inline u8
Florin Coras31c99552019-03-01 13:00:58 -0800722session_main_is_enabled ()
Florin Coras6cf30ad2017-04-04 23:08:23 -0700723{
Florin Coras31c99552019-03-01 13:00:58 -0800724 return session_main.is_enabled == 1;
Florin Coras6cf30ad2017-04-04 23:08:23 -0700725}
726
Florin Corascea194d2017-10-02 00:18:51 -0700727#define session_cli_return_if_not_enabled() \
728do { \
Florin Coras5bb23ec2019-08-31 09:45:13 -0700729 if (!session_main.is_enabled) \
730 return clib_error_return (0, "session layer is not enabled"); \
Florin Corascea194d2017-10-02 00:18:51 -0700731} while (0)
732
Florin Coras31c99552019-03-01 13:00:58 -0800733int session_main_flush_enqueue_events (u8 proto, u32 thread_index);
734int session_main_flush_all_enqueue_events (u8 transport_proto);
Florin Coras5484daa2020-03-27 23:55:06 +0000735void session_queue_run_on_main_thread (vlib_main_t * vm);
Florin Coras2a7ea2e2019-10-16 22:06:08 -0700736
Florin Coras5484daa2020-03-27 23:55:06 +0000737/**
738 * Add session node pending buffer with custom node
739 *
740 * @param thread_index worker thread expected to send the buffer
741 * @param bi buffer index
742 * @param next_node next node edge index for buffer. Edge to next node
743 * must exist
744 */
Florin Coras2a7ea2e2019-10-16 22:06:08 -0700745always_inline void
Florin Coras5484daa2020-03-27 23:55:06 +0000746session_add_pending_tx_buffer (u32 thread_index, u32 bi, u32 next_node)
Florin Coras2a7ea2e2019-10-16 22:06:08 -0700747{
748 session_worker_t *wrk = session_main_get_worker (thread_index);
749 vec_add1 (wrk->pending_tx_buffers, bi);
Florin Coras5484daa2020-03-27 23:55:06 +0000750 vec_add1 (wrk->pending_tx_nexts, next_node);
Florin Coras7da88292021-03-18 15:04:34 -0700751 if (PREDICT_FALSE (wrk->state == SESSION_WRK_INTERRUPT))
752 vlib_node_set_interrupt_pending (wrk->vm, session_queue_node.index);
Florin Coras2a7ea2e2019-10-16 22:06:08 -0700753}
754
Florin Coras8f10b902021-04-02 18:32:00 -0700755always_inline void
756session_wrk_update_time (session_worker_t *wrk, f64 now)
757{
758 wrk->last_vlib_time = now;
759 wrk->last_vlib_us_time = wrk->last_vlib_time * CLIB_US_TIME_FREQ;
760}
761
Florin Coras7da88292021-03-18 15:04:34 -0700762void session_wrk_enable_adaptive_mode (session_worker_t *wrk);
Florin Coras04943b42020-12-25 11:45:40 -0800763fifo_segment_t *session_main_get_evt_q_segment (void);
Dave Barach2a863912017-11-28 10:11:42 -0500764void session_node_enable_disable (u8 is_en);
Florin Coras1ee78302019-02-05 15:51:15 -0800765clib_error_t *vnet_session_enable_disable (vlib_main_t * vm, u8 is_en);
Dave Barach2a863912017-11-28 10:11:42 -0500766
Andreas Schultz1a8c4372020-03-20 09:39:59 +0100767session_t *session_alloc_for_connection (transport_connection_t * tc);
Florin Coras2c876f92021-05-10 21:12:27 -0700768session_t *session_alloc_for_half_open (transport_connection_t *tc);
Andreas Schultz1a8c4372020-03-20 09:39:59 +0100769
Dave Barach68b0fb02017-02-28 15:15:56 -0500770#endif /* __included_session_h__ */
771
772/*
773 * fd.io coding-style-patch-verification: ON
774 *
775 * Local Variables:
776 * eval: (c-set-style "gnu")
777 * End:
778 */