blob: 1b9f9de2a1b09daf70eb0f3380145d4cdc7ce807 [file] [log] [blame]
Dave Barach68b0fb02017-02-28 15:15:56 -05001/*
Florin Coras288eaab2019-02-03 15:26:14 -08002 * Copyright (c) 2017-2019 Cisco and/or its affiliates.
Dave Barach68b0fb02017-02-28 15:15:56 -05003 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15/**
16 * @file
17 * @brief Session and session manager
18 */
19
20#include <vnet/session/session.h>
Florin Coras04e53442017-07-16 17:12:15 -070021#include <vnet/session/application.h>
Dave Barach68b0fb02017-02-28 15:15:56 -050022#include <vnet/dpo/load_balance.h>
23#include <vnet/fib/ip4_fib.h>
Florin Corasd82f4712022-04-25 16:15:02 -070024#include <vlib/stats/stats.h>
Marvin Liu06542422022-08-16 06:49:09 +000025#include <vlib/dma/dma.h>
Dave Barach68b0fb02017-02-28 15:15:56 -050026
Florin Coras31c99552019-03-01 13:00:58 -080027session_main_t session_main;
Florin Coras3eb50622017-07-13 01:24:57 -040028
Florin Coras3c2fed52018-07-04 04:15:05 -070029static inline int
30session_send_evt_to_thread (void *data, void *args, u32 thread_index,
31 session_evt_type_t evt_type)
Florin Coras3cbc04b2017-10-02 00:18:51 -070032{
Florin Coras7da88292021-03-18 15:04:34 -070033 session_worker_t *wrk = session_main_get_worker (thread_index);
Florin Coras52207f12018-07-12 14:48:06 -070034 session_event_t *evt;
Florin Coras3c2fed52018-07-04 04:15:05 -070035 svm_msg_q_msg_t msg;
36 svm_msg_q_t *mq;
Florin Coras3cbc04b2017-10-02 00:18:51 -070037
Florin Coras7da88292021-03-18 15:04:34 -070038 mq = wrk->vpp_event_queue;
Nathan Skrzypczakb4c67752019-06-20 09:58:37 +020039 if (PREDICT_FALSE (svm_msg_q_lock (mq)))
40 return -1;
Florin Coras80d100c2022-04-19 18:57:24 -070041 if (PREDICT_FALSE (svm_msg_q_or_ring_is_full (mq, SESSION_MQ_IO_EVT_RING)))
Florin Coras3c2fed52018-07-04 04:15:05 -070042 {
43 svm_msg_q_unlock (mq);
44 return -2;
45 }
Florin Coras3c2fed52018-07-04 04:15:05 -070046 switch (evt_type)
47 {
Florin Corasf6c43132019-03-01 12:41:21 -080048 case SESSION_CTRL_EVT_RPC:
Nathan Skrzypczak1afa7af2019-09-13 17:14:57 +020049 msg = svm_msg_q_alloc_msg_w_ring (mq, SESSION_MQ_IO_EVT_RING);
50 evt = (session_event_t *) svm_msg_q_msg_data (mq, &msg);
Florin Coras3c2fed52018-07-04 04:15:05 -070051 evt->rpc_args.fp = data;
52 evt->rpc_args.arg = args;
53 break;
Aloys Augustin0c7f54d2019-07-03 16:59:43 +020054 case SESSION_IO_EVT_RX:
Florin Corasf6c43132019-03-01 12:41:21 -080055 case SESSION_IO_EVT_TX:
Florin Coras844a36d2018-12-20 09:50:50 -080056 case SESSION_IO_EVT_TX_FLUSH:
Florin Corasf6c43132019-03-01 12:41:21 -080057 case SESSION_IO_EVT_BUILTIN_RX:
Nathan Skrzypczak1afa7af2019-09-13 17:14:57 +020058 msg = svm_msg_q_alloc_msg_w_ring (mq, SESSION_MQ_IO_EVT_RING);
59 evt = (session_event_t *) svm_msg_q_msg_data (mq, &msg);
Florin Corasc0737e92019-03-04 14:19:39 -080060 evt->session_index = *(u32 *) data;
Florin Coras3c2fed52018-07-04 04:15:05 -070061 break;
Florin Corasaeb7c1c2023-03-10 10:22:21 -080062 case SESSION_IO_EVT_TX_MAIN:
Florin Corasf6c43132019-03-01 12:41:21 -080063 case SESSION_CTRL_EVT_CLOSE:
Florin Coras73cad332019-08-28 17:12:32 -070064 case SESSION_CTRL_EVT_RESET:
Nathan Skrzypczak1afa7af2019-09-13 17:14:57 +020065 msg = svm_msg_q_alloc_msg_w_ring (mq, SESSION_MQ_IO_EVT_RING);
66 evt = (session_event_t *) svm_msg_q_msg_data (mq, &msg);
Florin Coras288eaab2019-02-03 15:26:14 -080067 evt->session_handle = session_handle ((session_t *) data);
Florin Coras3c2fed52018-07-04 04:15:05 -070068 break;
69 default:
70 clib_warning ("evt unhandled!");
71 svm_msg_q_unlock (mq);
72 return -1;
73 }
Nathan Skrzypczak1afa7af2019-09-13 17:14:57 +020074 evt->event_type = evt_type;
Florin Coras3c2fed52018-07-04 04:15:05 -070075
Florin Coras52207f12018-07-12 14:48:06 -070076 svm_msg_q_add_and_unlock (mq, &msg);
Florin Coras7da88292021-03-18 15:04:34 -070077
78 if (PREDICT_FALSE (wrk->state == SESSION_WRK_INTERRUPT))
79 vlib_node_set_interrupt_pending (wrk->vm, session_queue_node.index);
80
Florin Coras3c2fed52018-07-04 04:15:05 -070081 return 0;
Florin Coras3cbc04b2017-10-02 00:18:51 -070082}
83
Florin Coras3c2fed52018-07-04 04:15:05 -070084int
85session_send_io_evt_to_thread (svm_fifo_t * f, session_evt_type_t evt_type)
Florin Coras3cbc04b2017-10-02 00:18:51 -070086{
Florin Corasc547e912020-12-08 17:50:45 -080087 return session_send_evt_to_thread (&f->shr->master_session_index, 0,
Florin Corasc0737e92019-03-04 14:19:39 -080088 f->master_thread_index, evt_type);
Florin Coras3c2fed52018-07-04 04:15:05 -070089}
90
91int
Florin Corasef915342018-09-29 10:23:06 -070092session_send_io_evt_to_thread_custom (void *data, u32 thread_index,
Florin Coras3c2fed52018-07-04 04:15:05 -070093 session_evt_type_t evt_type)
94{
Florin Corasef915342018-09-29 10:23:06 -070095 return session_send_evt_to_thread (data, 0, thread_index, evt_type);
Florin Coras3c2fed52018-07-04 04:15:05 -070096}
97
98int
Florin Coras288eaab2019-02-03 15:26:14 -080099session_send_ctrl_evt_to_thread (session_t * s, session_evt_type_t evt_type)
Florin Coras3c2fed52018-07-04 04:15:05 -0700100{
liuyacan534468e2021-05-09 03:50:40 +0000101 /* only events supported are disconnect, shutdown and reset */
102 ASSERT (evt_type == SESSION_CTRL_EVT_CLOSE ||
103 evt_type == SESSION_CTRL_EVT_HALF_CLOSE ||
104 evt_type == SESSION_CTRL_EVT_RESET);
Florin Coras458089b2019-08-21 16:20:44 -0700105 return session_send_evt_to_thread (s, 0, s->thread_index, evt_type);
Florin Coras3cbc04b2017-10-02 00:18:51 -0700106}
107
108void
Nathan Skrzypczak60f3e652019-03-19 13:57:31 +0100109session_send_rpc_evt_to_thread_force (u32 thread_index, void *fp,
110 void *rpc_args)
111{
112 session_send_evt_to_thread (fp, rpc_args, thread_index,
113 SESSION_CTRL_EVT_RPC);
114}
115
116void
Florin Coras3cbc04b2017-10-02 00:18:51 -0700117session_send_rpc_evt_to_thread (u32 thread_index, void *fp, void *rpc_args)
118{
119 if (thread_index != vlib_get_thread_index ())
Nathan Skrzypczak60f3e652019-03-19 13:57:31 +0100120 session_send_rpc_evt_to_thread_force (thread_index, fp, rpc_args);
Florin Coras3cbc04b2017-10-02 00:18:51 -0700121 else
122 {
123 void (*fnp) (void *) = fp;
124 fnp (rpc_args);
125 }
126}
127
Florin Coras26dd6de2019-07-23 23:54:47 -0700128void
129session_add_self_custom_tx_evt (transport_connection_t * tc, u8 has_prio)
130{
Florin Coras7da88292021-03-18 15:04:34 -0700131 session_t *s = session_get (tc->s_index, tc->thread_index);
Florin Coras26dd6de2019-07-23 23:54:47 -0700132
Florin Coras26dd6de2019-07-23 23:54:47 -0700133 ASSERT (s->thread_index == vlib_get_thread_index ());
Florin Corasbe237bf2019-09-27 08:16:40 -0700134 ASSERT (s->session_state != SESSION_STATE_TRANSPORT_DELETED);
Florin Coras7da88292021-03-18 15:04:34 -0700135
Florin Coras26dd6de2019-07-23 23:54:47 -0700136 if (!(s->flags & SESSION_F_CUSTOM_TX))
137 {
138 s->flags |= SESSION_F_CUSTOM_TX;
Florin Coras6080e0d2020-03-13 20:39:43 +0000139 if (svm_fifo_set_event (s->tx_fifo)
140 || transport_connection_is_descheduled (tc))
Florin Coras26dd6de2019-07-23 23:54:47 -0700141 {
Florin Coras26dd6de2019-07-23 23:54:47 -0700142 session_evt_elt_t *elt;
Florin Coras7da88292021-03-18 15:04:34 -0700143 session_worker_t *wrk;
144
Florin Coras26dd6de2019-07-23 23:54:47 -0700145 wrk = session_main_get_worker (tc->thread_index);
146 if (has_prio)
147 elt = session_evt_alloc_new (wrk);
148 else
149 elt = session_evt_alloc_old (wrk);
150 elt->evt.session_index = tc->s_index;
151 elt->evt.event_type = SESSION_IO_EVT_TX;
Florin Coras6080e0d2020-03-13 20:39:43 +0000152 tc->flags &= ~TRANSPORT_CONNECTION_F_DESCHED;
Florin Coras7da88292021-03-18 15:04:34 -0700153
154 if (PREDICT_FALSE (wrk->state == SESSION_WRK_INTERRUPT))
155 vlib_node_set_interrupt_pending (wrk->vm,
156 session_queue_node.index);
Florin Coras26dd6de2019-07-23 23:54:47 -0700157 }
158 }
159}
160
Florin Coras70f879d2020-03-13 17:54:42 +0000161void
162sesssion_reschedule_tx (transport_connection_t * tc)
163{
164 session_worker_t *wrk = session_main_get_worker (tc->thread_index);
165 session_evt_elt_t *elt;
166
167 ASSERT (tc->thread_index == vlib_get_thread_index ());
168
169 elt = session_evt_alloc_new (wrk);
170 elt->evt.session_index = tc->s_index;
171 elt->evt.event_type = SESSION_IO_EVT_TX;
Florin Coras7da88292021-03-18 15:04:34 -0700172
173 if (PREDICT_FALSE (wrk->state == SESSION_WRK_INTERRUPT))
174 vlib_node_set_interrupt_pending (wrk->vm, session_queue_node.index);
Florin Coras70f879d2020-03-13 17:54:42 +0000175}
176
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800177static void
Florin Corasdfb3b872019-08-16 17:48:44 -0700178session_program_transport_ctrl_evt (session_t * s, session_evt_type_t evt)
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800179{
180 u32 thread_index = vlib_get_thread_index ();
Florin Coras2062ec02019-07-15 13:15:18 -0700181 session_evt_elt_t *elt;
Florin Coras31c99552019-03-01 13:00:58 -0800182 session_worker_t *wrk;
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800183
184 /* If we are in the handler thread, or being called with the worker barrier
185 * held, just append a new event to pending disconnects vector. */
186 if (vlib_thread_is_main_w_barrier () || thread_index == s->thread_index)
187 {
Florin Coras31c99552019-03-01 13:00:58 -0800188 wrk = session_main_get_worker (s->thread_index);
Florin Corasb0ffbee2019-07-21 19:23:46 -0700189 elt = session_evt_alloc_ctrl (wrk);
Florin Coras2062ec02019-07-15 13:15:18 -0700190 clib_memset (&elt->evt, 0, sizeof (session_event_t));
191 elt->evt.session_handle = session_handle (s);
Florin Corasdfb3b872019-08-16 17:48:44 -0700192 elt->evt.event_type = evt;
Florin Coras7da88292021-03-18 15:04:34 -0700193
194 if (PREDICT_FALSE (wrk->state == SESSION_WRK_INTERRUPT))
195 vlib_node_set_interrupt_pending (wrk->vm, session_queue_node.index);
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800196 }
197 else
Florin Corasdfb3b872019-08-16 17:48:44 -0700198 session_send_ctrl_evt_to_thread (s, evt);
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800199}
200
Florin Coras288eaab2019-02-03 15:26:14 -0800201session_t *
Florin Coras3cbc04b2017-10-02 00:18:51 -0700202session_alloc (u32 thread_index)
Dave Barach68b0fb02017-02-28 15:15:56 -0500203{
Florin Coras31c99552019-03-01 13:00:58 -0800204 session_worker_t *wrk = &session_main.wrk[thread_index];
Florin Coras288eaab2019-02-03 15:26:14 -0800205 session_t *s;
Damjan Marion66d4cb52022-03-17 18:59:46 +0100206
Florin Corasd3915dc2022-03-31 15:42:17 -0700207 pool_get_aligned_safe (wrk->sessions, s, CLIB_CACHE_LINE_BYTES);
Dave Barachb7b92992018-10-17 10:38:51 -0400208 clib_memset (s, 0, sizeof (*s));
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700209 s->session_index = s - wrk->sessions;
Florin Coras3cbc04b2017-10-02 00:18:51 -0700210 s->thread_index = thread_index;
Nathan Skrzypczak2f0f96b2019-06-13 10:14:28 +0200211 s->app_index = APP_INVALID_INDEX;
Florin Coras6bd8d3f2022-03-14 21:17:25 -0700212
Florin Coras3cbc04b2017-10-02 00:18:51 -0700213 return s;
214}
215
Florin Coras371ca502018-02-21 12:07:41 -0800216void
Florin Coras288eaab2019-02-03 15:26:14 -0800217session_free (session_t * s)
Florin Coras3cbc04b2017-10-02 00:18:51 -0700218{
Steven Luong20de85b2022-10-17 10:39:06 -0700219 session_worker_t *wrk = &session_main.wrk[s->thread_index];
220
Florin Corascca96182019-07-19 07:34:13 -0700221 SESSION_EVT (SESSION_EVT_FREE, s);
Steven Luong20de85b2022-10-17 10:39:06 -0700222 if (CLIB_DEBUG)
223 clib_memset (s, 0xFA, sizeof (*s));
224 pool_put (wrk->sessions, s);
Florin Coras3cbc04b2017-10-02 00:18:51 -0700225}
226
Srikanth Akulae140d5d2019-11-18 11:49:58 -0800227u8
228session_is_valid (u32 si, u8 thread_index)
229{
230 session_t *s;
231 transport_connection_t *tc;
232
233 s = pool_elt_at_index (session_main.wrk[thread_index].sessions, si);
234
Srikanth Akulae140d5d2019-11-18 11:49:58 -0800235 if (s->thread_index != thread_index || s->session_index != si)
236 return 0;
237
238 if (s->session_state == SESSION_STATE_TRANSPORT_DELETED
239 || s->session_state <= SESSION_STATE_LISTENING)
240 return 1;
241
Florin Corasea727642021-05-07 19:39:43 -0700242 if (s->session_state == SESSION_STATE_CONNECTING &&
243 (s->flags & SESSION_F_HALF_OPEN))
244 return 1;
245
Srikanth Akulae140d5d2019-11-18 11:49:58 -0800246 tc = session_get_transport (s);
247 if (s->connection_index != tc->c_index
248 || s->thread_index != tc->thread_index || tc->s_index != si)
249 return 0;
250
251 return 1;
252}
253
Florin Coras70f26d52019-07-08 11:47:18 -0700254static void
255session_cleanup_notify (session_t * s, session_cleanup_ntf_t ntf)
256{
257 app_worker_t *app_wrk;
258
259 app_wrk = app_worker_get_if_valid (s->app_wrk_index);
260 if (!app_wrk)
261 return;
262 app_worker_cleanup_notify (app_wrk, s, ntf);
263}
264
Florin Coraseb97e5f2018-10-15 21:35:42 -0700265void
Florin Coras288eaab2019-02-03 15:26:14 -0800266session_free_w_fifos (session_t * s)
Florin Coras568ebc72018-09-18 16:12:50 -0700267{
Florin Coras70f26d52019-07-08 11:47:18 -0700268 session_cleanup_notify (s, SESSION_CLEANUP_SESSION);
Florin Coras19223e02019-03-03 14:56:05 -0800269 segment_manager_dealloc_fifos (s->rx_fifo, s->tx_fifo);
Florin Coras568ebc72018-09-18 16:12:50 -0700270 session_free (s);
271}
272
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800273/**
274 * Cleans up session and lookup table.
275 *
276 * Transport connection must still be valid.
277 */
278static void
Florin Coras288eaab2019-02-03 15:26:14 -0800279session_delete (session_t * s)
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800280{
281 int rv;
282
283 /* Delete from the main lookup table. */
284 if ((rv = session_lookup_del_session (s)))
Florin Corasd09236d2019-08-08 17:38:26 -0700285 clib_warning ("session %u hash delete rv %d", s->session_index, rv);
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800286
287 session_free_w_fifos (s);
288}
289
Florin Corasd50ff7f2020-04-16 04:30:22 +0000290void
Florin Corasea727642021-05-07 19:39:43 -0700291session_cleanup_half_open (session_handle_t ho_handle)
Florin Corasd50ff7f2020-04-16 04:30:22 +0000292{
Florin Corasbab6c522021-05-13 08:36:56 -0700293 session_t *ho = session_get_from_handle (ho_handle);
294
295 /* App transports can migrate their half-opens */
296 if (ho->flags & SESSION_F_IS_MIGRATING)
297 {
298 /* Session still migrating, move to closed state to signal that the
299 * session should be removed. */
300 if (ho->connection_index == ~0)
301 {
Steven Luongd810a6e2022-10-25 13:09:11 -0700302 session_set_state (ho, SESSION_STATE_CLOSED);
Florin Corasbab6c522021-05-13 08:36:56 -0700303 return;
304 }
305 /* Migrated transports are no longer half-opens */
306 transport_cleanup (session_get_transport_proto (ho),
307 ho->connection_index, ho->app_index /* overloaded */);
Florin Corasbab6c522021-05-13 08:36:56 -0700308 }
Florin Coras374df7a2021-05-13 11:37:43 -0700309 else
Florin Corasc9fb9872023-03-19 22:03:57 -0700310 {
311 /* Cleanup half-open session lookup table if need be */
312 if (ho->session_state != SESSION_STATE_TRANSPORT_CLOSING)
313 {
314 transport_connection_t *tc;
315 tc = transport_get_half_open (session_get_transport_proto (ho),
316 ho->connection_index);
317 if (tc && !(tc->flags & TRANSPORT_CONNECTION_F_NO_LOOKUP))
318 session_lookup_del_half_open (tc);
319 }
320 transport_cleanup_half_open (session_get_transport_proto (ho),
321 ho->connection_index);
322 }
Florin Coras374df7a2021-05-13 11:37:43 -0700323 session_free (ho);
Florin Corasbab6c522021-05-13 08:36:56 -0700324}
325
326static void
Florin Coras374df7a2021-05-13 11:37:43 -0700327session_half_open_free (session_t *ho)
Florin Corasbab6c522021-05-13 08:36:56 -0700328{
329 app_worker_t *app_wrk;
Florin Corasbab6c522021-05-13 08:36:56 -0700330
Florin Corasc9fb9872023-03-19 22:03:57 -0700331 ASSERT (vlib_get_thread_index () <= transport_cl_thread ());
332 app_wrk = app_worker_get_if_valid (ho->app_wrk_index);
333 if (app_wrk)
334 app_worker_del_half_open (app_wrk, ho);
Florin Corasbab6c522021-05-13 08:36:56 -0700335 session_free (ho);
336}
337
338static void
339session_half_open_free_rpc (void *args)
340{
Florin Coras374df7a2021-05-13 11:37:43 -0700341 session_t *ho = ho_session_get (pointer_to_uword (args));
342 session_half_open_free (ho);
Florin Corasd50ff7f2020-04-16 04:30:22 +0000343}
344
345void
Florin Corasea727642021-05-07 19:39:43 -0700346session_half_open_delete_notify (transport_connection_t *tc)
Florin Corasd50ff7f2020-04-16 04:30:22 +0000347{
Florin Corasc9fb9872023-03-19 22:03:57 -0700348 session_t *ho = ho_session_get (tc->s_index);
349
350 /* Cleanup half-open lookup table if need be */
351 if (ho->session_state != SESSION_STATE_TRANSPORT_CLOSING)
352 {
353 if (!(tc->flags & TRANSPORT_CONNECTION_F_NO_LOOKUP))
354 session_lookup_del_half_open (tc);
355 }
356
Florin Coras374df7a2021-05-13 11:37:43 -0700357 /* Notification from ctrl thread accepted without rpc */
Florin Coras309f7aa2022-03-18 08:33:08 -0700358 if (tc->thread_index == transport_cl_thread ())
Florin Coras374df7a2021-05-13 11:37:43 -0700359 {
Florin Corasc9fb9872023-03-19 22:03:57 -0700360 session_half_open_free (ho);
Florin Coras374df7a2021-05-13 11:37:43 -0700361 }
362 else
363 {
364 void *args = uword_to_pointer ((uword) tc->s_index, void *);
Florin Coras309f7aa2022-03-18 08:33:08 -0700365 session_send_rpc_evt_to_thread_force (transport_cl_thread (),
366 session_half_open_free_rpc, args);
Florin Coras374df7a2021-05-13 11:37:43 -0700367 }
Florin Corasbab6c522021-05-13 08:36:56 -0700368}
Florin Corasea727642021-05-07 19:39:43 -0700369
Florin Corasbab6c522021-05-13 08:36:56 -0700370void
371session_half_open_migrate_notify (transport_connection_t *tc)
372{
373 session_t *ho;
374
Florin Corasc9fb9872023-03-19 22:03:57 -0700375 /* Support half-open migrations only for transports with no lookup */
376 ASSERT (tc->flags & TRANSPORT_CONNECTION_F_NO_LOOKUP);
377
Florin Corasbab6c522021-05-13 08:36:56 -0700378 ho = ho_session_get (tc->s_index);
379 ho->flags |= SESSION_F_IS_MIGRATING;
380 ho->connection_index = ~0;
381}
382
383int
384session_half_open_migrated_notify (transport_connection_t *tc)
385{
386 session_t *ho;
387
388 ho = ho_session_get (tc->s_index);
389
390 /* App probably detached so the half-open must be cleaned up */
391 if (ho->session_state == SESSION_STATE_CLOSED)
392 {
393 session_half_open_delete_notify (tc);
394 return -1;
395 }
396 ho->connection_index = tc->c_index;
397 /* Overload app index for half-open with new thread */
398 ho->app_index = tc->thread_index;
399 return 0;
Florin Corasd50ff7f2020-04-16 04:30:22 +0000400}
401
Andreas Schultz1a8c4372020-03-20 09:39:59 +0100402session_t *
Florin Coras3cbc04b2017-10-02 00:18:51 -0700403session_alloc_for_connection (transport_connection_t * tc)
404{
Florin Coras288eaab2019-02-03 15:26:14 -0800405 session_t *s;
Florin Coras3cbc04b2017-10-02 00:18:51 -0700406 u32 thread_index = tc->thread_index;
407
Florin Coras40903ac2018-06-10 14:41:23 -0700408 ASSERT (thread_index == vlib_get_thread_index ()
409 || transport_protocol_is_cl (tc->proto));
Dave Barach2c25a622017-06-26 11:35:07 -0400410
Florin Coras3cbc04b2017-10-02 00:18:51 -0700411 s = session_alloc (thread_index);
412 s->session_type = session_type_from_proto_and_ip (tc->proto, tc->is_ip4);
Steven Luongd810a6e2022-10-25 13:09:11 -0700413 session_set_state (s, SESSION_STATE_CLOSED);
Dave Barach68b0fb02017-02-28 15:15:56 -0500414
Florin Coras3cbc04b2017-10-02 00:18:51 -0700415 /* Attach transport to session and vice versa */
Dave Barach68b0fb02017-02-28 15:15:56 -0500416 s->connection_index = tc->c_index;
Dave Barach68b0fb02017-02-28 15:15:56 -0500417 tc->s_index = s->session_index;
Florin Coras3cbc04b2017-10-02 00:18:51 -0700418 return s;
419}
420
Florin Coras2c876f92021-05-10 21:12:27 -0700421session_t *
Florin Corasea727642021-05-07 19:39:43 -0700422session_alloc_for_half_open (transport_connection_t *tc)
423{
424 session_t *s;
425
426 s = ho_session_alloc ();
427 s->session_type = session_type_from_proto_and_ip (tc->proto, tc->is_ip4);
428 s->connection_index = tc->c_index;
429 tc->s_index = s->session_index;
430 return s;
431}
432
Florin Coras1f152cd2017-08-18 19:28:03 -0700433/**
434 * Discards bytes from buffer chain
435 *
436 * It discards n_bytes_to_drop starting at first buffer after chain_b
437 */
438always_inline void
439session_enqueue_discard_chain_bytes (vlib_main_t * vm, vlib_buffer_t * b,
440 vlib_buffer_t ** chain_b,
441 u32 n_bytes_to_drop)
442{
443 vlib_buffer_t *next = *chain_b;
444 u32 to_drop = n_bytes_to_drop;
445 ASSERT (b->flags & VLIB_BUFFER_NEXT_PRESENT);
446 while (to_drop && (next->flags & VLIB_BUFFER_NEXT_PRESENT))
447 {
448 next = vlib_get_buffer (vm, next->next_buffer);
449 if (next->current_length > to_drop)
450 {
451 vlib_buffer_advance (next, to_drop);
452 to_drop = 0;
453 }
454 else
455 {
456 to_drop -= next->current_length;
457 next->current_length = 0;
458 }
459 }
460 *chain_b = next;
461
462 if (to_drop == 0)
463 b->total_length_not_including_first_buffer -= n_bytes_to_drop;
464}
465
466/**
467 * Enqueue buffer chain tail
468 */
Florin Corasf6d68ed2017-05-07 19:12:02 -0700469always_inline int
Florin Coras288eaab2019-02-03 15:26:14 -0800470session_enqueue_chain_tail (session_t * s, vlib_buffer_t * b,
Florin Corasf6d68ed2017-05-07 19:12:02 -0700471 u32 offset, u8 is_in_order)
472{
473 vlib_buffer_t *chain_b;
Florin Coras1f152cd2017-08-18 19:28:03 -0700474 u32 chain_bi, len, diff;
Florin Corasf6d68ed2017-05-07 19:12:02 -0700475 vlib_main_t *vm = vlib_get_main ();
Florin Corasb2215d62017-08-01 16:56:58 -0700476 u8 *data;
Florin Coras1f152cd2017-08-18 19:28:03 -0700477 u32 written = 0;
Florin Corasf6d68ed2017-05-07 19:12:02 -0700478 int rv = 0;
479
Florin Coras1f152cd2017-08-18 19:28:03 -0700480 if (is_in_order && offset)
481 {
482 diff = offset - b->current_length;
483 if (diff > b->total_length_not_including_first_buffer)
484 return 0;
485 chain_b = b;
486 session_enqueue_discard_chain_bytes (vm, b, &chain_b, diff);
487 chain_bi = vlib_get_buffer_index (vm, chain_b);
488 }
489 else
490 chain_bi = b->next_buffer;
491
Florin Corasf6d68ed2017-05-07 19:12:02 -0700492 do
493 {
494 chain_b = vlib_get_buffer (vm, chain_bi);
495 data = vlib_buffer_get_current (chain_b);
496 len = chain_b->current_length;
Florin Coras1f152cd2017-08-18 19:28:03 -0700497 if (!len)
498 continue;
Florin Corasf6d68ed2017-05-07 19:12:02 -0700499 if (is_in_order)
500 {
Florin Coras87b15ce2019-04-28 21:16:30 -0700501 rv = svm_fifo_enqueue (s->rx_fifo, len, data);
Florin Coras1f152cd2017-08-18 19:28:03 -0700502 if (rv == len)
503 {
504 written += rv;
505 }
506 else if (rv < len)
Florin Corasf6d68ed2017-05-07 19:12:02 -0700507 {
508 return (rv > 0) ? (written + rv) : written;
509 }
Florin Coras1f152cd2017-08-18 19:28:03 -0700510 else if (rv > len)
511 {
512 written += rv;
513
514 /* written more than what was left in chain */
515 if (written > b->total_length_not_including_first_buffer)
516 return written;
517
518 /* drop the bytes that have already been delivered */
519 session_enqueue_discard_chain_bytes (vm, b, &chain_b, rv - len);
520 }
Florin Corasf6d68ed2017-05-07 19:12:02 -0700521 }
522 else
523 {
Florin Coras288eaab2019-02-03 15:26:14 -0800524 rv = svm_fifo_enqueue_with_offset (s->rx_fifo, offset, len, data);
Florin Corasf6d68ed2017-05-07 19:12:02 -0700525 if (rv)
Florin Coras1f152cd2017-08-18 19:28:03 -0700526 {
527 clib_warning ("failed to enqueue multi-buffer seg");
528 return -1;
529 }
Florin Corasf6d68ed2017-05-07 19:12:02 -0700530 offset += len;
531 }
532 }
533 while ((chain_bi = (chain_b->flags & VLIB_BUFFER_NEXT_PRESENT)
534 ? chain_b->next_buffer : 0));
535
536 if (is_in_order)
537 return written;
538
539 return 0;
540}
541
Ryujiro Shibuyad8f48e22020-01-22 12:11:42 +0000542void
543session_fifo_tuning (session_t * s, svm_fifo_t * f,
544 session_ft_action_t act, u32 len)
545{
546 if (s->flags & SESSION_F_CUSTOM_FIFO_TUNING)
547 {
548 app_worker_t *app_wrk = app_worker_get (s->app_wrk_index);
549 app_worker_session_fifo_tuning (app_wrk, s, f, act, len);
550 if (CLIB_ASSERT_ENABLE)
551 {
552 segment_manager_t *sm;
553 sm = segment_manager_get (f->segment_manager);
Florin Corasc547e912020-12-08 17:50:45 -0800554 ASSERT (f->shr->size >= 4096);
555 ASSERT (f->shr->size <= sm->max_fifo_size);
Ryujiro Shibuyad8f48e22020-01-22 12:11:42 +0000556 }
557 }
558}
559
Dave Barach68b0fb02017-02-28 15:15:56 -0500560/*
561 * Enqueue data for delivery to session peer. Does not notify peer of enqueue
562 * event but on request can queue notification events for later delivery by
563 * calling stream_server_flush_enqueue_events().
564 *
565 * @param tc Transport connection which is to be enqueued data
Florin Corasf6d68ed2017-05-07 19:12:02 -0700566 * @param b Buffer to be enqueued
567 * @param offset Offset at which to start enqueueing if out-of-order
Dave Barach68b0fb02017-02-28 15:15:56 -0500568 * @param queue_event Flag to indicate if peer is to be notified or if event
569 * is to be queued. The former is useful when more data is
570 * enqueued and only one event is to be generated.
Florin Corasf6d68ed2017-05-07 19:12:02 -0700571 * @param is_in_order Flag to indicate if data is in order
Dave Barach68b0fb02017-02-28 15:15:56 -0500572 * @return Number of bytes enqueued or a negative value if enqueueing failed.
573 */
574int
Florin Coras3cbc04b2017-10-02 00:18:51 -0700575session_enqueue_stream_connection (transport_connection_t * tc,
576 vlib_buffer_t * b, u32 offset,
577 u8 queue_event, u8 is_in_order)
Dave Barach68b0fb02017-02-28 15:15:56 -0500578{
Florin Coras288eaab2019-02-03 15:26:14 -0800579 session_t *s;
Florin Coras1f152cd2017-08-18 19:28:03 -0700580 int enqueued = 0, rv, in_order_off;
Dave Barach68b0fb02017-02-28 15:15:56 -0500581
Florin Corascea194d2017-10-02 00:18:51 -0700582 s = session_get (tc->s_index, tc->thread_index);
Dave Barach68b0fb02017-02-28 15:15:56 -0500583
Florin Corasf6d68ed2017-05-07 19:12:02 -0700584 if (is_in_order)
585 {
Florin Coras87b15ce2019-04-28 21:16:30 -0700586 enqueued = svm_fifo_enqueue (s->rx_fifo,
587 b->current_length,
588 vlib_buffer_get_current (b));
Florin Coras1f152cd2017-08-18 19:28:03 -0700589 if (PREDICT_FALSE ((b->flags & VLIB_BUFFER_NEXT_PRESENT)
590 && enqueued >= 0))
Florin Corasf6d68ed2017-05-07 19:12:02 -0700591 {
Florin Coras1f152cd2017-08-18 19:28:03 -0700592 in_order_off = enqueued > b->current_length ? enqueued : 0;
593 rv = session_enqueue_chain_tail (s, b, in_order_off, 1);
594 if (rv > 0)
595 enqueued += rv;
Florin Corasf6d68ed2017-05-07 19:12:02 -0700596 }
597 }
598 else
599 {
Florin Coras288eaab2019-02-03 15:26:14 -0800600 rv = svm_fifo_enqueue_with_offset (s->rx_fifo, offset,
Florin Corasf6d68ed2017-05-07 19:12:02 -0700601 b->current_length,
602 vlib_buffer_get_current (b));
603 if (PREDICT_FALSE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) && !rv))
Florin Coras1f152cd2017-08-18 19:28:03 -0700604 session_enqueue_chain_tail (s, b, offset + b->current_length, 0);
605 /* if something was enqueued, report even this as success for ooo
606 * segment handling */
607 return rv;
Florin Corasf6d68ed2017-05-07 19:12:02 -0700608 }
Dave Barach68b0fb02017-02-28 15:15:56 -0500609
610 if (queue_event)
611 {
612 /* Queue RX event on this fifo. Eventually these will need to be flushed
613 * by calling stream_server_flush_enqueue_events () */
Florin Coras31c99552019-03-01 13:00:58 -0800614 session_worker_t *wrk;
Dave Barach68b0fb02017-02-28 15:15:56 -0500615
Florin Coras31c99552019-03-01 13:00:58 -0800616 wrk = session_main_get_worker (s->thread_index);
Florin Corasd5c604d2019-03-18 09:06:35 -0700617 if (!(s->flags & SESSION_F_RX_EVT))
Dave Barach68b0fb02017-02-28 15:15:56 -0500618 {
Florin Corasd5c604d2019-03-18 09:06:35 -0700619 s->flags |= SESSION_F_RX_EVT;
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700620 vec_add1 (wrk->session_to_enqueue[tc->proto], s->session_index);
Dave Barach68b0fb02017-02-28 15:15:56 -0500621 }
Ryujiro Shibuyad8f48e22020-01-22 12:11:42 +0000622
623 session_fifo_tuning (s, s->rx_fifo, SESSION_FT_ACTION_ENQUEUED, 0);
Dave Barach68b0fb02017-02-28 15:15:56 -0500624 }
625
Chris Lukeb2bcad62017-09-18 08:51:22 -0400626 return enqueued;
Dave Barach68b0fb02017-02-28 15:15:56 -0500627}
628
Florin Coras3cbc04b2017-10-02 00:18:51 -0700629int
Florin Coras288eaab2019-02-03 15:26:14 -0800630session_enqueue_dgram_connection (session_t * s,
Florin Coras7fb0fe12018-04-09 09:24:52 -0700631 session_dgram_hdr_t * hdr,
632 vlib_buffer_t * b, u8 proto, u8 queue_event)
Florin Coras3cbc04b2017-10-02 00:18:51 -0700633{
Florin Corasc95cfa22020-11-24 08:41:17 -0800634 int rv;
Florin Coras3cbc04b2017-10-02 00:18:51 -0700635
Sirshak Das28aa5392019-02-05 01:33:33 -0600636 ASSERT (svm_fifo_max_enqueue_prod (s->rx_fifo)
Florin Coras7fb0fe12018-04-09 09:24:52 -0700637 >= b->current_length + sizeof (*hdr));
638
Florin Corasc95cfa22020-11-24 08:41:17 -0800639 if (PREDICT_TRUE (!(b->flags & VLIB_BUFFER_NEXT_PRESENT)))
Florin Coras3cbc04b2017-10-02 00:18:51 -0700640 {
Florin Corasc95cfa22020-11-24 08:41:17 -0800641 /* *INDENT-OFF* */
642 svm_fifo_seg_t segs[2] = {
643 { (u8 *) hdr, sizeof (*hdr) },
644 { vlib_buffer_get_current (b), b->current_length }
645 };
646 /* *INDENT-ON* */
647
648 rv = svm_fifo_enqueue_segments (s->rx_fifo, segs, 2,
649 0 /* allow_partial */ );
Florin Coras3cbc04b2017-10-02 00:18:51 -0700650 }
Florin Corasc95cfa22020-11-24 08:41:17 -0800651 else
652 {
653 vlib_main_t *vm = vlib_get_main ();
654 svm_fifo_seg_t *segs = 0, *seg;
655 vlib_buffer_t *it = b;
656 u32 n_segs = 1;
657
658 vec_add2 (segs, seg, 1);
659 seg->data = (u8 *) hdr;
660 seg->len = sizeof (*hdr);
661 while (it)
662 {
663 vec_add2 (segs, seg, 1);
664 seg->data = vlib_buffer_get_current (it);
665 seg->len = it->current_length;
666 n_segs++;
667 if (!(it->flags & VLIB_BUFFER_NEXT_PRESENT))
668 break;
669 it = vlib_get_buffer (vm, it->next_buffer);
670 }
671 rv = svm_fifo_enqueue_segments (s->rx_fifo, segs, n_segs,
672 0 /* allow partial */ );
673 vec_free (segs);
674 }
675
676 if (queue_event && rv > 0)
Florin Coras3cbc04b2017-10-02 00:18:51 -0700677 {
678 /* Queue RX event on this fifo. Eventually these will need to be flushed
679 * by calling stream_server_flush_enqueue_events () */
Florin Coras31c99552019-03-01 13:00:58 -0800680 session_worker_t *wrk;
Florin Coras3cbc04b2017-10-02 00:18:51 -0700681
Florin Coras31c99552019-03-01 13:00:58 -0800682 wrk = session_main_get_worker (s->thread_index);
Florin Corasd5c604d2019-03-18 09:06:35 -0700683 if (!(s->flags & SESSION_F_RX_EVT))
Florin Coras3cbc04b2017-10-02 00:18:51 -0700684 {
Florin Corasd5c604d2019-03-18 09:06:35 -0700685 s->flags |= SESSION_F_RX_EVT;
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700686 vec_add1 (wrk->session_to_enqueue[proto], s->session_index);
Florin Coras3cbc04b2017-10-02 00:18:51 -0700687 }
Ryujiro Shibuyad8f48e22020-01-22 12:11:42 +0000688
689 session_fifo_tuning (s, s->rx_fifo, SESSION_FT_ACTION_ENQUEUED, 0);
Florin Coras3cbc04b2017-10-02 00:18:51 -0700690 }
Florin Corasc95cfa22020-11-24 08:41:17 -0800691 return rv > 0 ? rv : 0;
Florin Coras3cbc04b2017-10-02 00:18:51 -0700692}
693
Florin Coras93992a92017-05-24 18:03:56 -0700694int
Florin Coras31c99552019-03-01 13:00:58 -0800695session_tx_fifo_peek_bytes (transport_connection_t * tc, u8 * buffer,
696 u32 offset, u32 max_bytes)
Dave Barach68b0fb02017-02-28 15:15:56 -0500697{
Florin Coras288eaab2019-02-03 15:26:14 -0800698 session_t *s = session_get (tc->s_index, tc->thread_index);
699 return svm_fifo_peek (s->tx_fifo, offset, max_bytes, buffer);
Dave Barach68b0fb02017-02-28 15:15:56 -0500700}
701
702u32
Florin Coras31c99552019-03-01 13:00:58 -0800703session_tx_fifo_dequeue_drop (transport_connection_t * tc, u32 max_bytes)
Dave Barach68b0fb02017-02-28 15:15:56 -0500704{
Florin Coras288eaab2019-02-03 15:26:14 -0800705 session_t *s = session_get (tc->s_index, tc->thread_index);
Vladimir Kropylevf867cf12019-06-17 21:38:00 +0300706 u32 rv;
707
708 rv = svm_fifo_dequeue_drop (s->tx_fifo, max_bytes);
Ryujiro Shibuyad8f48e22020-01-22 12:11:42 +0000709 session_fifo_tuning (s, s->tx_fifo, SESSION_FT_ACTION_DEQUEUED, rv);
Florin Coras0e573f52019-05-07 16:28:16 -0700710
Florin Coras2d379d82019-06-28 12:45:12 -0700711 if (svm_fifo_needs_deq_ntf (s->tx_fifo, max_bytes))
Florin Coras0e573f52019-05-07 16:28:16 -0700712 session_dequeue_notify (s);
713
Vladimir Kropylevf867cf12019-06-17 21:38:00 +0300714 return rv;
Dave Barach68b0fb02017-02-28 15:15:56 -0500715}
716
Florin Coras72b04282019-01-14 17:23:11 -0800717static inline int
Florin Coras288eaab2019-02-03 15:26:14 -0800718session_notify_subscribers (u32 app_index, session_t * s,
Florin Coras72b04282019-01-14 17:23:11 -0800719 svm_fifo_t * f, session_evt_type_t evt_type)
720{
721 app_worker_t *app_wrk;
722 application_t *app;
723 int i;
724
725 app = application_get (app_index);
726 if (!app)
727 return -1;
728
Florin Corasc547e912020-12-08 17:50:45 -0800729 for (i = 0; i < f->shr->n_subscribers; i++)
Florin Coras72b04282019-01-14 17:23:11 -0800730 {
Florin Corasc547e912020-12-08 17:50:45 -0800731 app_wrk = application_get_worker (app, f->shr->subscribers[i]);
Florin Coras72b04282019-01-14 17:23:11 -0800732 if (!app_wrk)
733 continue;
734 if (app_worker_lock_and_send_event (app_wrk, s, evt_type))
735 return -1;
736 }
737
738 return 0;
739}
740
Dave Barach68b0fb02017-02-28 15:15:56 -0500741/**
742 * Notify session peer that new data has been enqueued.
743 *
Florin Coras3c2fed52018-07-04 04:15:05 -0700744 * @param s Stream session for which the event is to be generated.
745 * @param lock Flag to indicate if call should lock message queue.
Dave Barach68b0fb02017-02-28 15:15:56 -0500746 *
Florin Coras3c2fed52018-07-04 04:15:05 -0700747 * @return 0 on success or negative number if failed to send notification.
Dave Barach68b0fb02017-02-28 15:15:56 -0500748 */
Florin Coras3c2fed52018-07-04 04:15:05 -0700749static inline int
Florin Coras653e43f2019-03-04 10:56:23 -0800750session_enqueue_notify_inline (session_t * s)
Dave Barach68b0fb02017-02-28 15:15:56 -0500751{
Florin Coras72b04282019-01-14 17:23:11 -0800752 app_worker_t *app_wrk;
Aloys Augustinb3b267c2019-04-09 11:25:56 +0200753 u32 session_index;
754 u8 n_subscribers;
Steven Luonga468fd72023-03-08 16:28:27 -0800755 u32 thread_index;
Aloys Augustinb3b267c2019-04-09 11:25:56 +0200756
757 session_index = s->session_index;
Steven Luonga468fd72023-03-08 16:28:27 -0800758 thread_index = s->thread_index;
Aloys Augustinb3b267c2019-04-09 11:25:56 +0200759 n_subscribers = svm_fifo_n_subscribers (s->rx_fifo);
Dave Barach68b0fb02017-02-28 15:15:56 -0500760
Florin Coras72b04282019-01-14 17:23:11 -0800761 app_wrk = app_worker_get_if_valid (s->app_wrk_index);
762 if (PREDICT_FALSE (!app_wrk))
Dave Barach818eb542017-08-02 13:56:13 -0400763 {
Florin Coras15531972018-08-12 23:50:53 -0700764 SESSION_DBG ("invalid s->app_index = %d", s->app_wrk_index);
Dave Barach818eb542017-08-02 13:56:13 -0400765 return 0;
766 }
Dave Barach68b0fb02017-02-28 15:15:56 -0500767
Florin Corascca96182019-07-19 07:34:13 -0700768 SESSION_EVT (SESSION_EVT_ENQ, s, svm_fifo_max_dequeue_prod (s->rx_fifo));
Dave Barach68b0fb02017-02-28 15:15:56 -0500769
Florin Corasd5c604d2019-03-18 09:06:35 -0700770 s->flags &= ~SESSION_F_RX_EVT;
Florin Corasda302e42020-04-19 22:41:55 +0000771
772 /* Application didn't confirm accept yet */
773 if (PREDICT_FALSE (s->session_state == SESSION_STATE_ACCEPTING))
774 return 0;
775
Florin Coras72b04282019-01-14 17:23:11 -0800776 if (PREDICT_FALSE (app_worker_lock_and_send_event (app_wrk, s,
Florin Corasf6c43132019-03-01 12:41:21 -0800777 SESSION_IO_EVT_RX)))
Florin Coras72b04282019-01-14 17:23:11 -0800778 return -1;
779
Aloys Augustinb3b267c2019-04-09 11:25:56 +0200780 if (PREDICT_FALSE (n_subscribers))
781 {
Steven Luonga468fd72023-03-08 16:28:27 -0800782 s = session_get (session_index, thread_index);
Aloys Augustinb3b267c2019-04-09 11:25:56 +0200783 return session_notify_subscribers (app_wrk->app_index, s,
784 s->rx_fifo, SESSION_IO_EVT_RX);
785 }
Florin Coras72b04282019-01-14 17:23:11 -0800786
787 return 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500788}
789
Florin Coras0d60a0f2018-06-29 02:02:08 -0700790int
Florin Coras653e43f2019-03-04 10:56:23 -0800791session_enqueue_notify (session_t * s)
792{
793 return session_enqueue_notify_inline (s);
794}
795
Aloys Augustinb3392332019-08-06 16:09:01 +0200796static void
797session_enqueue_notify_rpc (void *arg)
798{
Florin Coras5d8a8062019-08-13 08:35:39 -0700799 u32 session_index = pointer_to_uword (arg);
Aloys Augustinb3392332019-08-06 16:09:01 +0200800 session_t *s;
801
Florin Coras5d8a8062019-08-13 08:35:39 -0700802 s = session_get_if_valid (session_index, vlib_get_thread_index ());
Aloys Augustinb3392332019-08-06 16:09:01 +0200803 if (!s)
804 return;
805
806 session_enqueue_notify (s);
807}
808
809/**
810 * Like session_enqueue_notify, but can be called from a thread that does not
811 * own the session.
812 */
813void
814session_enqueue_notify_thread (session_handle_t sh)
815{
816 u32 thread_index = session_thread_from_handle (sh);
Florin Coras5d8a8062019-08-13 08:35:39 -0700817 u32 session_index = session_index_from_handle (sh);
818
819 /*
820 * Pass session index (u32) as opposed to handle (u64) in case pointers
821 * are not 64-bit.
822 */
Aloys Augustinb3392332019-08-06 16:09:01 +0200823 session_send_rpc_evt_to_thread (thread_index,
Florin Coras5d8a8062019-08-13 08:35:39 -0700824 session_enqueue_notify_rpc,
825 uword_to_pointer (session_index, void *));
Aloys Augustinb3392332019-08-06 16:09:01 +0200826}
827
Florin Coras653e43f2019-03-04 10:56:23 -0800828int
Florin Coras288eaab2019-02-03 15:26:14 -0800829session_dequeue_notify (session_t * s)
Florin Coras0d60a0f2018-06-29 02:02:08 -0700830{
Florin Coras72b04282019-01-14 17:23:11 -0800831 app_worker_t *app_wrk;
Florin Coras0d60a0f2018-06-29 02:02:08 -0700832
Vladimir Kropylev5c89fbf2019-08-30 13:04:06 +0300833 svm_fifo_clear_deq_ntf (s->tx_fifo);
834
Florin Coras72b04282019-01-14 17:23:11 -0800835 app_wrk = app_worker_get_if_valid (s->app_wrk_index);
836 if (PREDICT_FALSE (!app_wrk))
Florin Coras208daa12018-07-02 01:30:51 -0700837 return -1;
838
Florin Coras72b04282019-01-14 17:23:11 -0800839 if (PREDICT_FALSE (app_worker_lock_and_send_event (app_wrk, s,
Florin Corasf6c43132019-03-01 12:41:21 -0800840 SESSION_IO_EVT_TX)))
Florin Coras1bcad5c2019-01-09 20:04:38 -0800841 return -1;
842
Florin Corasc547e912020-12-08 17:50:45 -0800843 if (PREDICT_FALSE (s->tx_fifo->shr->n_subscribers))
Florin Coras72b04282019-01-14 17:23:11 -0800844 return session_notify_subscribers (app_wrk->app_index, s,
Florin Corasf6c43132019-03-01 12:41:21 -0800845 s->tx_fifo, SESSION_IO_EVT_TX);
Florin Coras72b04282019-01-14 17:23:11 -0800846
Florin Coras1bcad5c2019-01-09 20:04:38 -0800847 return 0;
Florin Coras0d60a0f2018-06-29 02:02:08 -0700848}
849
Dave Barach68b0fb02017-02-28 15:15:56 -0500850/**
851 * Flushes queue of sessions that are to be notified of new data
852 * enqueued events.
853 *
854 * @param thread_index Thread index for which the flush is to be performed.
855 * @return 0 on success or a positive number indicating the number of
856 * failures due to API queue being full.
857 */
858int
Florin Coras31c99552019-03-01 13:00:58 -0800859session_main_flush_enqueue_events (u8 transport_proto, u32 thread_index)
Dave Barach68b0fb02017-02-28 15:15:56 -0500860{
Florin Coras31c99552019-03-01 13:00:58 -0800861 session_worker_t *wrk = session_main_get_worker (thread_index);
Florin Coras288eaab2019-02-03 15:26:14 -0800862 session_t *s;
Florin Coras21795132018-09-09 09:40:51 -0700863 int i, errors = 0;
Florin Coras3c2fed52018-07-04 04:15:05 -0700864 u32 *indices;
Dave Barach68b0fb02017-02-28 15:15:56 -0500865
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700866 indices = wrk->session_to_enqueue[transport_proto];
Dave Barach68b0fb02017-02-28 15:15:56 -0500867
Florin Coras3cbc04b2017-10-02 00:18:51 -0700868 for (i = 0; i < vec_len (indices); i++)
Dave Barach68b0fb02017-02-28 15:15:56 -0500869 {
Florin Coras3cbc04b2017-10-02 00:18:51 -0700870 s = session_get_if_valid (indices[i], thread_index);
Florin Coras3c2fed52018-07-04 04:15:05 -0700871 if (PREDICT_FALSE (!s))
872 {
873 errors++;
874 continue;
875 }
Florin Coras653e43f2019-03-04 10:56:23 -0800876
Ryujiro Shibuyad8f48e22020-01-22 12:11:42 +0000877 session_fifo_tuning (s, s->rx_fifo, SESSION_FT_ACTION_ENQUEUED,
878 0 /* TODO/not needed */ );
879
Florin Coras653e43f2019-03-04 10:56:23 -0800880 if (PREDICT_FALSE (session_enqueue_notify_inline (s)))
Florin Coras3cbc04b2017-10-02 00:18:51 -0700881 errors++;
Dave Barach68b0fb02017-02-28 15:15:56 -0500882 }
883
Florin Coras3cbc04b2017-10-02 00:18:51 -0700884 vec_reset_length (indices);
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700885 wrk->session_to_enqueue[transport_proto] = indices;
Dave Barach68b0fb02017-02-28 15:15:56 -0500886
887 return errors;
888}
889
Florin Coras7fb0fe12018-04-09 09:24:52 -0700890int
Florin Coras31c99552019-03-01 13:00:58 -0800891session_main_flush_all_enqueue_events (u8 transport_proto)
Florin Coras7fb0fe12018-04-09 09:24:52 -0700892{
893 vlib_thread_main_t *vtm = vlib_get_thread_main ();
894 int i, errors = 0;
895 for (i = 0; i < 1 + vtm->n_threads; i++)
Florin Coras31c99552019-03-01 13:00:58 -0800896 errors += session_main_flush_enqueue_events (transport_proto, i);
Florin Coras7fb0fe12018-04-09 09:24:52 -0700897 return errors;
898}
899
Florin Coras4fde4ae2020-04-13 16:48:04 +0000900int
901session_stream_connect_notify (transport_connection_t * tc,
902 session_error_t err)
Dave Barach68b0fb02017-02-28 15:15:56 -0500903{
Florin Coras371ca502018-02-21 12:07:41 -0800904 u32 opaque = 0, new_ti, new_si;
Florin Coras15531972018-08-12 23:50:53 -0700905 app_worker_t *app_wrk;
Florin Corasea727642021-05-07 19:39:43 -0700906 session_t *s = 0, *ho;
Dave Barach68b0fb02017-02-28 15:15:56 -0500907
Florin Coras3cbc04b2017-10-02 00:18:51 -0700908 /*
Florin Corasea727642021-05-07 19:39:43 -0700909 * Cleanup half-open table
Florin Coras3cbc04b2017-10-02 00:18:51 -0700910 */
Florin Corascea194d2017-10-02 00:18:51 -0700911 session_lookup_del_half_open (tc);
Florin Coras4eeeaaf2017-09-05 14:03:37 -0400912
Florin Corasea727642021-05-07 19:39:43 -0700913 ho = ho_session_get (tc->s_index);
Florin Corasc9fb9872023-03-19 22:03:57 -0700914 session_set_state (ho, SESSION_STATE_TRANSPORT_CLOSING);
Florin Corasea727642021-05-07 19:39:43 -0700915 opaque = ho->opaque;
916 app_wrk = app_worker_get_if_valid (ho->app_wrk_index);
Florin Coras15531972018-08-12 23:50:53 -0700917 if (!app_wrk)
Florin Corasc87c91d2017-08-16 19:55:49 -0700918 return -1;
Florin Corasa27a46e2019-02-18 13:02:28 -0800919
Florin Coras00e01d32019-10-21 16:07:46 -0700920 if (err)
921 return app_worker_connect_notify (app_wrk, s, err, opaque);
Florin Corasa27a46e2019-02-18 13:02:28 -0800922
923 s = session_alloc_for_connection (tc);
Steven Luongd810a6e2022-10-25 13:09:11 -0700924 session_set_state (s, SESSION_STATE_CONNECTING);
Florin Corasa27a46e2019-02-18 13:02:28 -0800925 s->app_wrk_index = app_wrk->wrk_index;
926 new_si = s->session_index;
927 new_ti = s->thread_index;
928
Florin Coras00e01d32019-10-21 16:07:46 -0700929 if ((err = app_worker_init_connected (app_wrk, s)))
Dave Barach68b0fb02017-02-28 15:15:56 -0500930 {
Florin Corasa27a46e2019-02-18 13:02:28 -0800931 session_free (s);
Florin Coras00e01d32019-10-21 16:07:46 -0700932 app_worker_connect_notify (app_wrk, 0, err, opaque);
Florin Corasa27a46e2019-02-18 13:02:28 -0800933 return -1;
Dave Barach68b0fb02017-02-28 15:15:56 -0500934 }
935
Nathan Skrzypczaka26349d2019-06-26 13:53:08 +0200936 s = session_get (new_si, new_ti);
Steven Luongd810a6e2022-10-25 13:09:11 -0700937 session_set_state (s, SESSION_STATE_READY);
Nathan Skrzypczaka26349d2019-06-26 13:53:08 +0200938 session_lookup_add_connection (tc, session_handle (s));
939
Florin Coras00e01d32019-10-21 16:07:46 -0700940 if (app_worker_connect_notify (app_wrk, s, SESSION_E_NONE, opaque))
Florin Coras6534b7a2017-07-18 05:38:03 -0400941 {
Florin Corasc7fd24e2020-07-24 10:09:07 -0700942 session_lookup_del_connection (tc);
943 /* Avoid notifying app about rejected session cleanup */
Florin Corasa27a46e2019-02-18 13:02:28 -0800944 s = session_get (new_si, new_ti);
Florin Corasc7fd24e2020-07-24 10:09:07 -0700945 segment_manager_dealloc_fifos (s->rx_fifo, s->tx_fifo);
946 session_free (s);
Florin Corasa27a46e2019-02-18 13:02:28 -0800947 return -1;
Florin Coras6534b7a2017-07-18 05:38:03 -0400948 }
Dave Barach68b0fb02017-02-28 15:15:56 -0500949
Florin Corasa27a46e2019-02-18 13:02:28 -0800950 return 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500951}
952
Florin Coras02000442021-11-16 12:45:43 -0800953typedef union session_switch_pool_reply_args_
954{
955 struct
956 {
957 u32 session_index;
958 u16 thread_index;
959 u8 is_closed;
960 };
961 u64 as_u64;
962} session_switch_pool_reply_args_t;
963
964STATIC_ASSERT (sizeof (session_switch_pool_reply_args_t) <= sizeof (uword),
965 "switch pool reply args size");
966
Florin Coras6d7552c2020-04-09 01:49:45 +0000967static void
968session_switch_pool_reply (void *arg)
969{
Florin Coras02000442021-11-16 12:45:43 -0800970 session_switch_pool_reply_args_t rargs;
Florin Coras6d7552c2020-04-09 01:49:45 +0000971 session_t *s;
972
Florin Coras02000442021-11-16 12:45:43 -0800973 rargs.as_u64 = pointer_to_uword (arg);
974 s = session_get_if_valid (rargs.session_index, rargs.thread_index);
Florin Coras6d7552c2020-04-09 01:49:45 +0000975 if (!s)
976 return;
977
Florin Coras02000442021-11-16 12:45:43 -0800978 /* Session closed during migration. Clean everything up */
979 if (rargs.is_closed)
980 {
981 transport_cleanup (session_get_transport_proto (s), s->connection_index,
982 s->thread_index);
983 segment_manager_dealloc_fifos (s->rx_fifo, s->tx_fifo);
984 session_free (s);
985 return;
986 }
987
Florin Coras6d7552c2020-04-09 01:49:45 +0000988 /* Notify app that it has data on the new session */
989 session_enqueue_notify (s);
990}
991
Florin Coras3cbc04b2017-10-02 00:18:51 -0700992typedef struct _session_switch_pool_args
993{
994 u32 session_index;
995 u32 thread_index;
996 u32 new_thread_index;
997 u32 new_session_index;
998} session_switch_pool_args_t;
999
Florin Coras49568af2019-07-31 16:46:24 -07001000/**
1001 * Notify old thread of the session pool switch
1002 */
Florin Coras3cbc04b2017-10-02 00:18:51 -07001003static void
1004session_switch_pool (void *cb_args)
1005{
1006 session_switch_pool_args_t *args = (session_switch_pool_args_t *) cb_args;
Florin Coras02000442021-11-16 12:45:43 -08001007 session_switch_pool_reply_args_t rargs;
Florin Coras6d7552c2020-04-09 01:49:45 +00001008 session_handle_t new_sh;
1009 segment_manager_t *sm;
Florin Coras49568af2019-07-31 16:46:24 -07001010 app_worker_t *app_wrk;
Florin Coras288eaab2019-02-03 15:26:14 -08001011 session_t *s;
Florin Coras49568af2019-07-31 16:46:24 -07001012
Florin Coras3cbc04b2017-10-02 00:18:51 -07001013 ASSERT (args->thread_index == vlib_get_thread_index ());
1014 s = session_get (args->session_index, args->thread_index);
Florin Coras6d7552c2020-04-09 01:49:45 +00001015
Florin Corasfc8d0c52021-11-17 11:41:10 -08001016 /* Check if session closed during migration */
1017 rargs.is_closed = s->session_state >= SESSION_STATE_TRANSPORT_CLOSING;
1018
Florin Coras1ee78302019-02-05 15:51:15 -08001019 transport_cleanup (session_get_transport_proto (s), s->connection_index,
1020 s->thread_index);
Florin Coras49568af2019-07-31 16:46:24 -07001021
1022 app_wrk = app_worker_get_if_valid (s->app_wrk_index);
1023 if (app_wrk)
1024 {
Florin Coras6d7552c2020-04-09 01:49:45 +00001025 /* Cleanup fifo segment slice state for fifos */
1026 sm = app_worker_get_connect_segment_manager (app_wrk);
Florin Coras0bc78d82021-01-09 14:34:01 -08001027 segment_manager_detach_fifo (sm, &s->rx_fifo);
1028 segment_manager_detach_fifo (sm, &s->tx_fifo);
Aloys Augustinb3392332019-08-06 16:09:01 +02001029
Florin Coras6d7552c2020-04-09 01:49:45 +00001030 /* Notify app, using old session, about the migration event */
Florin Coras02000442021-11-16 12:45:43 -08001031 if (!rargs.is_closed)
1032 {
1033 new_sh = session_make_handle (args->new_session_index,
1034 args->new_thread_index);
1035 app_worker_migrate_notify (app_wrk, s, new_sh);
1036 }
Florin Coras49568af2019-07-31 16:46:24 -07001037 }
1038
Florin Coras6d7552c2020-04-09 01:49:45 +00001039 /* Trigger app read and fifo updates on the new thread */
Florin Coras02000442021-11-16 12:45:43 -08001040 rargs.session_index = args->new_session_index;
1041 rargs.thread_index = args->new_thread_index;
Florin Coras6d7552c2020-04-09 01:49:45 +00001042 session_send_rpc_evt_to_thread (args->new_thread_index,
Florin Coras02000442021-11-16 12:45:43 -08001043 session_switch_pool_reply,
1044 uword_to_pointer (rargs.as_u64, void *));
Florin Coras6d7552c2020-04-09 01:49:45 +00001045
Florin Coras3cbc04b2017-10-02 00:18:51 -07001046 session_free (s);
1047 clib_mem_free (cb_args);
1048}
1049
1050/**
1051 * Move dgram session to the right thread
1052 */
1053int
1054session_dgram_connect_notify (transport_connection_t * tc,
Florin Coras288eaab2019-02-03 15:26:14 -08001055 u32 old_thread_index, session_t ** new_session)
Florin Coras3cbc04b2017-10-02 00:18:51 -07001056{
Florin Coras288eaab2019-02-03 15:26:14 -08001057 session_t *new_s;
Florin Coras3cbc04b2017-10-02 00:18:51 -07001058 session_switch_pool_args_t *rpc_args;
Florin Coras0bc78d82021-01-09 14:34:01 -08001059 segment_manager_t *sm;
1060 app_worker_t *app_wrk;
Florin Coras3cbc04b2017-10-02 00:18:51 -07001061
1062 /*
1063 * Clone half-open session to the right thread.
1064 */
1065 new_s = session_clone_safe (tc->s_index, old_thread_index);
1066 new_s->connection_index = tc->c_index;
Steven Luongd810a6e2022-10-25 13:09:11 -07001067 session_set_state (new_s, SESSION_STATE_READY);
Nathan Skrzypczakcaa7acf2019-10-02 10:02:05 +02001068 new_s->flags |= SESSION_F_IS_MIGRATING;
Florin Coras6d7552c2020-04-09 01:49:45 +00001069
Florin Coras0bc78d82021-01-09 14:34:01 -08001070 if (!(tc->flags & TRANSPORT_CONNECTION_F_NO_LOOKUP))
1071 session_lookup_add_connection (tc, session_handle (new_s));
1072
1073 app_wrk = app_worker_get_if_valid (new_s->app_wrk_index);
1074 if (app_wrk)
1075 {
1076 /* New set of fifos attached to the same shared memory */
1077 sm = app_worker_get_connect_segment_manager (app_wrk);
1078 segment_manager_attach_fifo (sm, &new_s->rx_fifo, new_s);
1079 segment_manager_attach_fifo (sm, &new_s->tx_fifo, new_s);
1080 }
Florin Coras3cbc04b2017-10-02 00:18:51 -07001081
1082 /*
1083 * Ask thread owning the old session to clean it up and make us the tx
1084 * fifo owner
1085 */
1086 rpc_args = clib_mem_alloc (sizeof (*rpc_args));
1087 rpc_args->new_session_index = new_s->session_index;
1088 rpc_args->new_thread_index = new_s->thread_index;
1089 rpc_args->session_index = tc->s_index;
1090 rpc_args->thread_index = old_thread_index;
1091 session_send_rpc_evt_to_thread (rpc_args->thread_index, session_switch_pool,
1092 rpc_args);
1093
1094 tc->s_index = new_s->session_index;
1095 new_s->connection_index = tc->c_index;
1096 *new_session = new_s;
1097 return 0;
1098}
1099
Dave Barach68b0fb02017-02-28 15:15:56 -05001100/**
1101 * Notification from transport that connection is being closed.
1102 *
1103 * A disconnect is sent to application but state is not removed. Once
1104 * disconnect is acknowledged by application, session disconnect is called.
1105 * Ultimately this leads to close being called on transport (passive close).
1106 */
1107void
Florin Coras5a2ec8f2018-12-27 11:53:11 -08001108session_transport_closing_notify (transport_connection_t * tc)
Dave Barach68b0fb02017-02-28 15:15:56 -05001109{
Florin Coras15531972018-08-12 23:50:53 -07001110 app_worker_t *app_wrk;
Florin Coras288eaab2019-02-03 15:26:14 -08001111 session_t *s;
Dave Barach68b0fb02017-02-28 15:15:56 -05001112
Florin Corascea194d2017-10-02 00:18:51 -07001113 s = session_get (tc->s_index, tc->thread_index);
Florin Coras400ded32018-10-03 01:00:57 -07001114 if (s->session_state >= SESSION_STATE_TRANSPORT_CLOSING)
1115 return;
Florin Corasdfd27bc2021-11-27 10:45:55 -08001116
1117 /* Wait for reply from app before sending notification as the
1118 * accept might be rejected */
1119 if (s->session_state == SESSION_STATE_ACCEPTING)
1120 {
Steven Luongd810a6e2022-10-25 13:09:11 -07001121 session_set_state (s, SESSION_STATE_TRANSPORT_CLOSING);
Florin Corasdfd27bc2021-11-27 10:45:55 -08001122 return;
1123 }
1124
Steven Luongd810a6e2022-10-25 13:09:11 -07001125 session_set_state (s, SESSION_STATE_TRANSPORT_CLOSING);
Florin Corasbf7ce2c2019-03-06 14:44:42 -08001126 app_wrk = app_worker_get (s->app_wrk_index);
1127 app_worker_close_notify (app_wrk, s);
Dave Barach68b0fb02017-02-28 15:15:56 -05001128}
1129
1130/**
Dave Barach68b0fb02017-02-28 15:15:56 -05001131 * Notification from transport that connection is being deleted
1132 *
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001133 * This removes the session if it is still valid. It should be called only on
1134 * previously fully established sessions. For instance failed connects should
1135 * call stream_session_connect_notify and indicate that the connect has
1136 * failed.
Dave Barach68b0fb02017-02-28 15:15:56 -05001137 */
1138void
Florin Coras5a2ec8f2018-12-27 11:53:11 -08001139session_transport_delete_notify (transport_connection_t * tc)
Dave Barach68b0fb02017-02-28 15:15:56 -05001140{
Florin Coras288eaab2019-02-03 15:26:14 -08001141 session_t *s;
Dave Barach68b0fb02017-02-28 15:15:56 -05001142
Florin Corase04c2992017-03-01 08:17:34 -08001143 /* App might've been removed already */
Florin Coras568ebc72018-09-18 16:12:50 -07001144 if (!(s = session_get_if_valid (tc->s_index, tc->thread_index)))
Florin Corasc87c91d2017-08-16 19:55:49 -07001145 return;
Florin Coras568ebc72018-09-18 16:12:50 -07001146
Florin Coras568ebc72018-09-18 16:12:50 -07001147 switch (s->session_state)
1148 {
Florin Coras222e1f412019-02-16 20:47:32 -08001149 case SESSION_STATE_CREATED:
1150 /* Session was created but accept notification was not yet sent to the
1151 * app. Cleanup everything. */
1152 session_lookup_del_session (s);
Zeyu Zhangcbbc4a22019-10-12 14:21:59 +08001153 segment_manager_dealloc_fifos (s->rx_fifo, s->tx_fifo);
1154 session_free (s);
Florin Coras222e1f412019-02-16 20:47:32 -08001155 break;
Florin Corasb0f662f2018-12-27 14:51:46 -08001156 case SESSION_STATE_ACCEPTING:
Florin Coras568ebc72018-09-18 16:12:50 -07001157 case SESSION_STATE_TRANSPORT_CLOSING:
Florin Coras692b9492019-07-12 15:01:53 -07001158 case SESSION_STATE_CLOSING:
Florin Coras5f066322019-07-22 19:03:03 -07001159 case SESSION_STATE_TRANSPORT_CLOSED:
Florin Coras568ebc72018-09-18 16:12:50 -07001160 /* If transport finishes or times out before we get a reply
Florin Coras5a2ec8f2018-12-27 11:53:11 -08001161 * from the app, mark transport as closed and wait for reply
1162 * before removing the session. Cleanup session table in advance
Florin Corasa9d5bea2018-12-17 08:24:19 -08001163 * because transport will soon be closed and closed sessions
1164 * are assumed to have been removed from the lookup table */
1165 session_lookup_del_session (s);
Steven Luongd810a6e2022-10-25 13:09:11 -07001166 session_set_state (s, SESSION_STATE_TRANSPORT_DELETED);
Florin Coras70f26d52019-07-08 11:47:18 -07001167 session_cleanup_notify (s, SESSION_CLEANUP_TRANSPORT);
1168 svm_fifo_dequeue_drop_all (s->tx_fifo);
Florin Coras568ebc72018-09-18 16:12:50 -07001169 break;
Florin Coras692b9492019-07-12 15:01:53 -07001170 case SESSION_STATE_APP_CLOSED:
Florin Coras5a2ec8f2018-12-27 11:53:11 -08001171 /* Cleanup lookup table as transport needs to still be valid.
1172 * Program transport close to ensure that all session events
1173 * have been cleaned up. Once transport close is called, the
1174 * session is just removed because both transport and app have
1175 * confirmed the close*/
Florin Coras568ebc72018-09-18 16:12:50 -07001176 session_lookup_del_session (s);
Steven Luongd810a6e2022-10-25 13:09:11 -07001177 session_set_state (s, SESSION_STATE_TRANSPORT_DELETED);
Florin Coras70f26d52019-07-08 11:47:18 -07001178 session_cleanup_notify (s, SESSION_CLEANUP_TRANSPORT);
1179 svm_fifo_dequeue_drop_all (s->tx_fifo);
Florin Corasdfb3b872019-08-16 17:48:44 -07001180 session_program_transport_ctrl_evt (s, SESSION_CTRL_EVT_CLOSE);
Florin Coras568ebc72018-09-18 16:12:50 -07001181 break;
Florin Coras5f066322019-07-22 19:03:03 -07001182 case SESSION_STATE_TRANSPORT_DELETED:
Florin Corasb0f662f2018-12-27 14:51:46 -08001183 break;
Florin Coras568ebc72018-09-18 16:12:50 -07001184 case SESSION_STATE_CLOSED:
Florin Coras70f26d52019-07-08 11:47:18 -07001185 session_cleanup_notify (s, SESSION_CLEANUP_TRANSPORT);
Florin Coras5a2ec8f2018-12-27 11:53:11 -08001186 session_delete (s);
Florin Coras568ebc72018-09-18 16:12:50 -07001187 break;
Florin Corasc01d5782018-10-17 14:53:11 -07001188 default:
Florin Corasb0f662f2018-12-27 14:51:46 -08001189 clib_warning ("session state %u", s->session_state);
Florin Coras70f26d52019-07-08 11:47:18 -07001190 session_cleanup_notify (s, SESSION_CLEANUP_TRANSPORT);
Florin Coras5a2ec8f2018-12-27 11:53:11 -08001191 session_delete (s);
Florin Corasc01d5782018-10-17 14:53:11 -07001192 break;
Florin Coras568ebc72018-09-18 16:12:50 -07001193 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001194}
1195
1196/**
Florin Coras692b9492019-07-12 15:01:53 -07001197 * Notification from transport that it is closed
Florin Coras54ddf432018-12-21 13:54:09 -08001198 *
Florin Coras692b9492019-07-12 15:01:53 -07001199 * Should be called by transport, prior to calling delete notify, once it
1200 * knows that no more data will be exchanged. This could serve as an
1201 * early acknowledgment of an active close especially if transport delete
1202 * can be delayed a long time, e.g., tcp time-wait.
Florin Coras54ddf432018-12-21 13:54:09 -08001203 */
1204void
Florin Coras5a2ec8f2018-12-27 11:53:11 -08001205session_transport_closed_notify (transport_connection_t * tc)
Florin Coras54ddf432018-12-21 13:54:09 -08001206{
Florin Coras692b9492019-07-12 15:01:53 -07001207 app_worker_t *app_wrk;
Florin Coras288eaab2019-02-03 15:26:14 -08001208 session_t *s;
Florin Coras54ddf432018-12-21 13:54:09 -08001209
1210 if (!(s = session_get_if_valid (tc->s_index, tc->thread_index)))
1211 return;
Florin Corasb0f662f2018-12-27 14:51:46 -08001212
Florin Coras06a2eec2019-05-23 22:28:16 -07001213 /* Transport thinks that app requested close but it actually didn't.
liuyacan534468e2021-05-09 03:50:40 +00001214 * Can happen for tcp:
1215 * 1)if fin and rst are received in close succession.
1216 * 2)if app shutdown the connection. */
Florin Coras06a2eec2019-05-23 22:28:16 -07001217 if (s->session_state == SESSION_STATE_READY)
1218 {
1219 session_transport_closing_notify (tc);
1220 svm_fifo_dequeue_drop_all (s->tx_fifo);
Steven Luongd810a6e2022-10-25 13:09:11 -07001221 session_set_state (s, SESSION_STATE_TRANSPORT_CLOSED);
Florin Coras06a2eec2019-05-23 22:28:16 -07001222 }
Florin Corasb0f662f2018-12-27 14:51:46 -08001223 /* If app close has not been received or has not yet resulted in
1224 * a transport close, only mark the session transport as closed */
Florin Coras06a2eec2019-05-23 22:28:16 -07001225 else if (s->session_state <= SESSION_STATE_CLOSING)
Steven Luongd810a6e2022-10-25 13:09:11 -07001226 session_set_state (s, SESSION_STATE_TRANSPORT_CLOSED);
Florin Coras5f066322019-07-22 19:03:03 -07001227 /* If app also closed, switch to closed */
1228 else if (s->session_state == SESSION_STATE_APP_CLOSED)
Steven Luongd810a6e2022-10-25 13:09:11 -07001229 session_set_state (s, SESSION_STATE_CLOSED);
Florin Coras692b9492019-07-12 15:01:53 -07001230
1231 app_wrk = app_worker_get_if_valid (s->app_wrk_index);
1232 if (app_wrk)
1233 app_worker_transport_closed_notify (app_wrk, s);
Florin Coras54ddf432018-12-21 13:54:09 -08001234}
1235
1236/**
Dave Barach68b0fb02017-02-28 15:15:56 -05001237 * Notify application that connection has been reset.
1238 */
1239void
Florin Coras5a2ec8f2018-12-27 11:53:11 -08001240session_transport_reset_notify (transport_connection_t * tc)
Dave Barach68b0fb02017-02-28 15:15:56 -05001241{
Florin Coras15531972018-08-12 23:50:53 -07001242 app_worker_t *app_wrk;
Florin Coras69b68ef2019-04-02 11:38:51 -07001243 session_t *s;
1244
Florin Corascea194d2017-10-02 00:18:51 -07001245 s = session_get (tc->s_index, tc->thread_index);
Florin Coras288eaab2019-02-03 15:26:14 -08001246 svm_fifo_dequeue_drop_all (s->tx_fifo);
Florin Coras3c7d4f92018-12-14 11:28:43 -08001247 if (s->session_state >= SESSION_STATE_TRANSPORT_CLOSING)
1248 return;
Florin Corasdfd27bc2021-11-27 10:45:55 -08001249 if (s->session_state == SESSION_STATE_ACCEPTING)
1250 {
Steven Luongd810a6e2022-10-25 13:09:11 -07001251 session_set_state (s, SESSION_STATE_TRANSPORT_CLOSING);
Florin Corasdfd27bc2021-11-27 10:45:55 -08001252 return;
1253 }
Steven Luongd810a6e2022-10-25 13:09:11 -07001254 session_set_state (s, SESSION_STATE_TRANSPORT_CLOSING);
Florin Coras15531972018-08-12 23:50:53 -07001255 app_wrk = app_worker_get (s->app_wrk_index);
Florin Coras69b68ef2019-04-02 11:38:51 -07001256 app_worker_reset_notify (app_wrk, s);
Dave Barach68b0fb02017-02-28 15:15:56 -05001257}
1258
Florin Corasa27a46e2019-02-18 13:02:28 -08001259int
1260session_stream_accept_notify (transport_connection_t * tc)
1261{
1262 app_worker_t *app_wrk;
1263 session_t *s;
1264
1265 s = session_get (tc->s_index, tc->thread_index);
1266 app_wrk = app_worker_get_if_valid (s->app_wrk_index);
1267 if (!app_wrk)
1268 return -1;
Florin Coras600d7a82021-04-29 11:55:23 -07001269 if (s->session_state != SESSION_STATE_CREATED)
1270 return 0;
Steven Luongd810a6e2022-10-25 13:09:11 -07001271 session_set_state (s, SESSION_STATE_ACCEPTING);
Florin Coras4dc10a42020-02-11 05:31:49 +00001272 if (app_worker_accept_notify (app_wrk, s))
1273 {
1274 /* On transport delete, no notifications should be sent. Unless, the
1275 * accept is retried and successful. */
Steven Luongd810a6e2022-10-25 13:09:11 -07001276 session_set_state (s, SESSION_STATE_CREATED);
Florin Coras4dc10a42020-02-11 05:31:49 +00001277 return -1;
1278 }
1279 return 0;
Florin Corasa27a46e2019-02-18 13:02:28 -08001280}
1281
Dave Barach68b0fb02017-02-28 15:15:56 -05001282/**
1283 * Accept a stream session. Optionally ping the server by callback.
1284 */
1285int
Florin Corasc9940fc2019-02-05 20:55:11 -08001286session_stream_accept (transport_connection_t * tc, u32 listener_index,
Nathan Skrzypczak2f0f96b2019-06-13 10:14:28 +02001287 u32 thread_index, u8 notify)
Dave Barach68b0fb02017-02-28 15:15:56 -05001288{
Florin Corasa27a46e2019-02-18 13:02:28 -08001289 session_t *s;
Dave Barach68b0fb02017-02-28 15:15:56 -05001290 int rv;
1291
Florin Corasa27a46e2019-02-18 13:02:28 -08001292 s = session_alloc_for_connection (tc);
Nathan Skrzypczak2f0f96b2019-06-13 10:14:28 +02001293 s->listener_handle = ((u64) thread_index << 32) | (u64) listener_index;
Steven Luongd810a6e2022-10-25 13:09:11 -07001294 session_set_state (s, SESSION_STATE_CREATED);
Dave Barach68b0fb02017-02-28 15:15:56 -05001295
Florin Corasa27a46e2019-02-18 13:02:28 -08001296 if ((rv = app_worker_init_accepted (s)))
Florin Coras12813d52020-04-09 20:59:20 +00001297 {
1298 session_free (s);
1299 return rv;
1300 }
Florin Corasa27a46e2019-02-18 13:02:28 -08001301
1302 session_lookup_add_connection (tc, session_handle (s));
1303
Dave Barach68b0fb02017-02-28 15:15:56 -05001304 /* Shoulder-tap the server */
1305 if (notify)
1306 {
Florin Corasa27a46e2019-02-18 13:02:28 -08001307 app_worker_t *app_wrk = app_worker_get (s->app_wrk_index);
Florin Coras12813d52020-04-09 20:59:20 +00001308 if ((rv = app_worker_accept_notify (app_wrk, s)))
1309 {
1310 session_lookup_del_session (s);
1311 segment_manager_dealloc_fifos (s->rx_fifo, s->tx_fifo);
1312 session_free (s);
1313 return rv;
1314 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001315 }
1316
1317 return 0;
1318}
1319
Florin Coras371ca502018-02-21 12:07:41 -08001320int
Florin Corase759bb52020-04-08 01:55:39 +00001321session_dgram_accept (transport_connection_t * tc, u32 listener_index,
1322 u32 thread_index)
1323{
1324 app_worker_t *app_wrk;
1325 session_t *s;
1326 int rv;
1327
1328 s = session_alloc_for_connection (tc);
1329 s->listener_handle = ((u64) thread_index << 32) | (u64) listener_index;
1330
1331 if ((rv = app_worker_init_accepted (s)))
1332 {
1333 session_free (s);
1334 return rv;
1335 }
1336
Florin Coras473556d2020-11-23 15:59:36 -08001337 session_lookup_add_connection (tc, session_handle (s));
Steven Luongd810a6e2022-10-25 13:09:11 -07001338 session_set_state (s, SESSION_STATE_ACCEPTING);
Florin Coras473556d2020-11-23 15:59:36 -08001339
Florin Corase759bb52020-04-08 01:55:39 +00001340 app_wrk = app_worker_get (s->app_wrk_index);
1341 if ((rv = app_worker_accept_notify (app_wrk, s)))
1342 {
Florin Coras473556d2020-11-23 15:59:36 -08001343 session_lookup_del_session (s);
Florin Coras12813d52020-04-09 20:59:20 +00001344 segment_manager_dealloc_fifos (s->rx_fifo, s->tx_fifo);
1345 session_free (s);
Florin Corase759bb52020-04-08 01:55:39 +00001346 return rv;
1347 }
1348
Florin Corase759bb52020-04-08 01:55:39 +00001349 return 0;
1350}
1351
1352int
Florin Coras89a9f612021-05-11 11:55:07 -07001353session_open_cl (session_endpoint_cfg_t *rmt, session_handle_t *rsh)
Florin Coras371ca502018-02-21 12:07:41 -08001354{
1355 transport_connection_t *tc;
Florin Coras5665ced2018-10-25 18:03:45 -07001356 transport_endpoint_cfg_t *tep;
Florin Coras15531972018-08-12 23:50:53 -07001357 app_worker_t *app_wrk;
Aloys Augustin20ab31e2019-03-25 11:29:17 +01001358 session_handle_t sh;
Florin Coras288eaab2019-02-03 15:26:14 -08001359 session_t *s;
Florin Coras371ca502018-02-21 12:07:41 -08001360 int rv;
1361
Florin Coras5665ced2018-10-25 18:03:45 -07001362 tep = session_endpoint_to_transport_cfg (rmt);
Florin Coras1ee78302019-02-05 15:51:15 -08001363 rv = transport_connect (rmt->transport_proto, tep);
Florin Coras371ca502018-02-21 12:07:41 -08001364 if (rv < 0)
1365 {
1366 SESSION_DBG ("Transport failed to open connection.");
Florin Coras57660d92020-04-04 22:45:34 +00001367 return rv;
Florin Coras371ca502018-02-21 12:07:41 -08001368 }
1369
Florin Coras1ee78302019-02-05 15:51:15 -08001370 tc = transport_get_half_open (rmt->transport_proto, (u32) rv);
Florin Coras371ca502018-02-21 12:07:41 -08001371
Florin Corasa27a46e2019-02-18 13:02:28 -08001372 /* For dgram type of service, allocate session and fifos now */
Florin Coras89a9f612021-05-11 11:55:07 -07001373 app_wrk = app_worker_get (rmt->app_wrk_index);
Florin Corasa27a46e2019-02-18 13:02:28 -08001374 s = session_alloc_for_connection (tc);
Florin Coras15531972018-08-12 23:50:53 -07001375 s->app_wrk_index = app_wrk->wrk_index;
Steven Luongd810a6e2022-10-25 13:09:11 -07001376 session_set_state (s, SESSION_STATE_OPENED);
Florin Corasa27a46e2019-02-18 13:02:28 -08001377 if (app_worker_init_connected (app_wrk, s))
1378 {
1379 session_free (s);
1380 return -1;
1381 }
Florin Coras371ca502018-02-21 12:07:41 -08001382
Aloys Augustin20ab31e2019-03-25 11:29:17 +01001383 sh = session_handle (s);
Florin Coras89a9f612021-05-11 11:55:07 -07001384 *rsh = sh;
1385
Aloys Augustin20ab31e2019-03-25 11:29:17 +01001386 session_lookup_add_connection (tc, sh);
Florin Coras89a9f612021-05-11 11:55:07 -07001387 return app_worker_connect_notify (app_wrk, s, SESSION_E_NONE, rmt->opaque);
Florin Coras371ca502018-02-21 12:07:41 -08001388}
1389
1390int
Florin Coras89a9f612021-05-11 11:55:07 -07001391session_open_vc (session_endpoint_cfg_t *rmt, session_handle_t *rsh)
Florin Coras371ca502018-02-21 12:07:41 -08001392{
1393 transport_connection_t *tc;
Florin Coras5665ced2018-10-25 18:03:45 -07001394 transport_endpoint_cfg_t *tep;
Florin Corasea727642021-05-07 19:39:43 -07001395 app_worker_t *app_wrk;
Florin Coras89a9f612021-05-11 11:55:07 -07001396 session_t *ho;
Florin Coras371ca502018-02-21 12:07:41 -08001397 int rv;
1398
Florin Coras5665ced2018-10-25 18:03:45 -07001399 tep = session_endpoint_to_transport_cfg (rmt);
Florin Coras1ee78302019-02-05 15:51:15 -08001400 rv = transport_connect (rmt->transport_proto, tep);
Florin Coras371ca502018-02-21 12:07:41 -08001401 if (rv < 0)
1402 {
1403 SESSION_DBG ("Transport failed to open connection.");
Florin Coras57660d92020-04-04 22:45:34 +00001404 return rv;
Florin Coras371ca502018-02-21 12:07:41 -08001405 }
1406
Florin Coras1ee78302019-02-05 15:51:15 -08001407 tc = transport_get_half_open (rmt->transport_proto, (u32) rv);
Florin Coras371ca502018-02-21 12:07:41 -08001408
Florin Coras89a9f612021-05-11 11:55:07 -07001409 app_wrk = app_worker_get (rmt->app_wrk_index);
Florin Coras371ca502018-02-21 12:07:41 -08001410
Florin Corasea727642021-05-07 19:39:43 -07001411 /* If transport offers a vc service, only allocate established
1412 * session once the connection has been established.
1413 * In the meantime allocate half-open session for tracking purposes
1414 * associate half-open connection to it and add session to app-worker
1415 * half-open table. These are needed to allocate the established
1416 * session on transport notification, and to cleanup the half-open
1417 * session if the app detaches before connection establishment.
Florin Coras371ca502018-02-21 12:07:41 -08001418 */
Florin Coras89a9f612021-05-11 11:55:07 -07001419 ho = session_alloc_for_half_open (tc);
1420 ho->app_wrk_index = app_wrk->wrk_index;
1421 ho->ho_index = app_worker_add_half_open (app_wrk, session_handle (ho));
1422 ho->opaque = rmt->opaque;
1423 *rsh = session_handle (ho);
Florin Corasd50ff7f2020-04-16 04:30:22 +00001424
Florin Coras2c876f92021-05-10 21:12:27 -07001425 if (!(tc->flags & TRANSPORT_CONNECTION_F_NO_LOOKUP))
1426 session_lookup_add_half_open (tc, tc->c_index);
Florin Coras4fde4ae2020-04-13 16:48:04 +00001427
Florin Coras371ca502018-02-21 12:07:41 -08001428 return 0;
1429}
1430
1431int
Florin Coras89a9f612021-05-11 11:55:07 -07001432session_open_app (session_endpoint_cfg_t *rmt, session_handle_t *rsh)
Florin Coras371ca502018-02-21 12:07:41 -08001433{
Florin Coras89a9f612021-05-11 11:55:07 -07001434 transport_endpoint_cfg_t *tep_cfg = session_endpoint_to_transport_cfg (rmt);
Florin Coras5665ced2018-10-25 18:03:45 -07001435
Florin Coras89a9f612021-05-11 11:55:07 -07001436 /* Not supported for now */
1437 *rsh = SESSION_INVALID_HANDLE;
Florin Coras1ee78302019-02-05 15:51:15 -08001438 return transport_connect (rmt->transport_proto, tep_cfg);
Florin Coras371ca502018-02-21 12:07:41 -08001439}
1440
Florin Coras89a9f612021-05-11 11:55:07 -07001441typedef int (*session_open_service_fn) (session_endpoint_cfg_t *,
1442 session_handle_t *);
Florin Coras371ca502018-02-21 12:07:41 -08001443
1444/* *INDENT-OFF* */
1445static session_open_service_fn session_open_srv_fns[TRANSPORT_N_SERVICES] = {
1446 session_open_vc,
1447 session_open_cl,
1448 session_open_app,
1449};
1450/* *INDENT-ON* */
1451
Florin Coras6cf30ad2017-04-04 23:08:23 -07001452/**
1453 * Ask transport to open connection to remote transport endpoint.
1454 *
1455 * Stores handle for matching request with reply since the call can be
1456 * asynchronous. For instance, for TCP the 3-way handshake must complete
1457 * before reply comes. Session is only created once connection is established.
1458 *
1459 * @param app_index Index of the application requesting the connect
1460 * @param st Session type requested.
1461 * @param tep Remote transport endpoint
Florin Coras3cbc04b2017-10-02 00:18:51 -07001462 * @param opaque Opaque data (typically, api_context) the application expects
1463 * on open completion.
Florin Coras6cf30ad2017-04-04 23:08:23 -07001464 */
Florin Corase04c2992017-03-01 08:17:34 -08001465int
Florin Coras89a9f612021-05-11 11:55:07 -07001466session_open (session_endpoint_cfg_t *rmt, session_handle_t *rsh)
Dave Barach68b0fb02017-02-28 15:15:56 -05001467{
Florin Coras1ee78302019-02-05 15:51:15 -08001468 transport_service_type_t tst;
1469 tst = transport_protocol_service_type (rmt->transport_proto);
Florin Coras89a9f612021-05-11 11:55:07 -07001470 return session_open_srv_fns[tst](rmt, rsh);
Florin Coras6cf30ad2017-04-04 23:08:23 -07001471}
1472
Florin Coras7fb0fe12018-04-09 09:24:52 -07001473/**
Florin Corasab2f6db2018-08-31 14:31:41 -07001474 * Ask transport to listen on session endpoint.
Florin Coras7fb0fe12018-04-09 09:24:52 -07001475 *
1476 * @param s Session for which listen will be called. Note that unlike
1477 * established sessions, listen sessions are not associated to a
1478 * thread.
Florin Corasab2f6db2018-08-31 14:31:41 -07001479 * @param sep Local endpoint to be listened on.
Florin Coras7fb0fe12018-04-09 09:24:52 -07001480 */
Florin Coras371ca502018-02-21 12:07:41 -08001481int
Florin Coras288eaab2019-02-03 15:26:14 -08001482session_listen (session_t * ls, session_endpoint_cfg_t * sep)
Florin Coras371ca502018-02-21 12:07:41 -08001483{
Florin Coras0bce71e2022-02-10 11:57:06 -08001484 transport_endpoint_cfg_t *tep;
Florin Corasa8c3b862020-04-07 22:31:06 +00001485 int tc_index;
1486 u32 s_index;
Florin Corasab2f6db2018-08-31 14:31:41 -07001487
1488 /* Transport bind/listen */
Florin Coras0bce71e2022-02-10 11:57:06 -08001489 tep = session_endpoint_to_transport_cfg (sep);
Florin Coras74cac882018-09-07 09:13:15 -07001490 s_index = ls->session_index;
Florin Corasd4295e62019-02-22 13:11:38 -08001491 tc_index = transport_start_listen (session_get_transport_proto (ls),
1492 s_index, tep);
Florin Corasab2f6db2018-08-31 14:31:41 -07001493
Florin Corasa8c3b862020-04-07 22:31:06 +00001494 if (tc_index < 0)
1495 return tc_index;
Florin Corasab2f6db2018-08-31 14:31:41 -07001496
Florin Corasd4295e62019-02-22 13:11:38 -08001497 /* Attach transport to session. Lookup tables are populated by the app
1498 * worker because local tables (for ct sessions) are not backed by a fib */
Florin Coras74cac882018-09-07 09:13:15 -07001499 ls = listen_session_get (s_index);
Florin Corasab2f6db2018-08-31 14:31:41 -07001500 ls->connection_index = tc_index;
Mohammed Hawari472d0da2022-10-17 17:55:35 +02001501 ls->opaque = sep->opaque;
Florin Corasab2f6db2018-08-31 14:31:41 -07001502
Florin Corasab2f6db2018-08-31 14:31:41 -07001503 return 0;
Florin Coras371ca502018-02-21 12:07:41 -08001504}
1505
Florin Coras6cf30ad2017-04-04 23:08:23 -07001506/**
1507 * Ask transport to stop listening on local transport endpoint.
1508 *
1509 * @param s Session to stop listening on. It must be in state LISTENING.
1510 */
1511int
Florin Coras288eaab2019-02-03 15:26:14 -08001512session_stop_listen (session_t * s)
Florin Coras6cf30ad2017-04-04 23:08:23 -07001513{
Florin Coras561af9b2017-12-09 10:19:43 -08001514 transport_proto_t tp = session_get_transport_proto (s);
Florin Coras6cf30ad2017-04-04 23:08:23 -07001515 transport_connection_t *tc;
Florin Coras6cf30ad2017-04-04 23:08:23 -07001516
Florin Coras1ee78302019-02-05 15:51:15 -08001517 if (s->session_state != SESSION_STATE_LISTENING)
Florin Corasa8c3b862020-04-07 22:31:06 +00001518 return SESSION_E_NOLISTEN;
Florin Coras1ee78302019-02-05 15:51:15 -08001519
1520 tc = transport_get_listener (tp, s->connection_index);
Florin Corasa8c3b862020-04-07 22:31:06 +00001521
1522 /* If no transport, assume everything was cleaned up already */
Florin Coras6cf30ad2017-04-04 23:08:23 -07001523 if (!tc)
Florin Corasa8c3b862020-04-07 22:31:06 +00001524 return SESSION_E_NONE;
Florin Coras6cf30ad2017-04-04 23:08:23 -07001525
Nathan Skrzypczak2eed1a12019-07-04 14:26:21 +02001526 if (!(tc->flags & TRANSPORT_CONNECTION_F_NO_LOOKUP))
1527 session_lookup_del_connection (tc);
Florin Corasa8c3b862020-04-07 22:31:06 +00001528
Florin Coras1ee78302019-02-05 15:51:15 -08001529 transport_stop_listen (tp, s->connection_index);
Florin Corase04c2992017-03-01 08:17:34 -08001530 return 0;
Dave Barach68b0fb02017-02-28 15:15:56 -05001531}
1532
1533/**
liuyacan534468e2021-05-09 03:50:40 +00001534 * Initialize session half-closing procedure.
1535 *
1536 * Note that half-closing will not change the state of the session.
1537 */
1538void
1539session_half_close (session_t *s)
1540{
1541 if (!s)
1542 return;
1543
1544 session_program_transport_ctrl_evt (s, SESSION_CTRL_EVT_HALF_CLOSE);
1545}
1546
1547/**
Florin Coras5a2ec8f2018-12-27 11:53:11 -08001548 * Initialize session closing procedure.
Florin Corasa5464812017-04-19 13:00:05 -07001549 *
Florin Coras2f8d8fa2018-01-26 06:36:04 -08001550 * Request is always sent to session node to ensure that all outstanding
1551 * requests are served before transport is notified.
Dave Barach68b0fb02017-02-28 15:15:56 -05001552 */
1553void
Florin Coras288eaab2019-02-03 15:26:14 -08001554session_close (session_t * s)
Dave Barach68b0fb02017-02-28 15:15:56 -05001555{
Florin Coras7c06b572023-02-20 15:14:04 -08001556 if (!s || (s->flags & SESSION_F_APP_CLOSED))
Florin Coras2f8d8fa2018-01-26 06:36:04 -08001557 return;
Florin Coras25579b42018-06-06 17:55:02 -07001558
Florin Coras7c06b572023-02-20 15:14:04 -08001559 /* Transports can close and delete their state independent of app closes
1560 * and transport initiated state transitions can hide app closes. Instead
1561 * of extending the state machine to support separate tracking of app and
1562 * transport initiated closes, use a flag. */
1563 s->flags |= SESSION_F_APP_CLOSED;
1564
Florin Coras25579b42018-06-06 17:55:02 -07001565 if (s->session_state >= SESSION_STATE_CLOSING)
1566 {
Florin Coras5a2ec8f2018-12-27 11:53:11 -08001567 /* Session will only be removed once both app and transport
1568 * acknowledge the close */
Florin Coras5f066322019-07-22 19:03:03 -07001569 if (s->session_state == SESSION_STATE_TRANSPORT_CLOSED
1570 || s->session_state == SESSION_STATE_TRANSPORT_DELETED)
Florin Corasdfb3b872019-08-16 17:48:44 -07001571 session_program_transport_ctrl_evt (s, SESSION_CTRL_EVT_CLOSE);
Florin Coras25579b42018-06-06 17:55:02 -07001572 return;
1573 }
1574
Dongya Zhang7a87c712022-11-03 15:22:34 +08001575 /* App closed so stop propagating dequeue notifications.
1576 * App might disconnect session before connected, in this case,
1577 * tx_fifo may not be setup yet, so clear only it's inited. */
1578 if (s->tx_fifo)
1579 svm_fifo_clear_deq_ntf (s->tx_fifo);
Steven Luongd810a6e2022-10-25 13:09:11 -07001580 session_set_state (s, SESSION_STATE_CLOSING);
Florin Corasdfb3b872019-08-16 17:48:44 -07001581 session_program_transport_ctrl_evt (s, SESSION_CTRL_EVT_CLOSE);
1582}
1583
1584/**
1585 * Force a close without waiting for data to be flushed
1586 */
1587void
1588session_reset (session_t * s)
1589{
1590 if (s->session_state >= SESSION_STATE_CLOSING)
1591 return;
Dongya Zhang7a87c712022-11-03 15:22:34 +08001592 /* Drop all outstanding tx data
1593 * App might disconnect session before connected, in this case,
1594 * tx_fifo may not be setup yet, so clear only it's inited. */
1595 if (s->tx_fifo)
1596 svm_fifo_dequeue_drop_all (s->tx_fifo);
Steven Luongd810a6e2022-10-25 13:09:11 -07001597 session_set_state (s, SESSION_STATE_CLOSING);
Florin Corasdfb3b872019-08-16 17:48:44 -07001598 session_program_transport_ctrl_evt (s, SESSION_CTRL_EVT_RESET);
Florin Coras2f8d8fa2018-01-26 06:36:04 -08001599}
1600
1601/**
liuyacan534468e2021-05-09 03:50:40 +00001602 * Notify transport the session can be half-disconnected.
1603 *
1604 * Must be called from the session's thread.
1605 */
1606void
1607session_transport_half_close (session_t *s)
1608{
1609 /* Only READY session can be half-closed */
1610 if (s->session_state != SESSION_STATE_READY)
1611 {
1612 return;
1613 }
1614
1615 transport_half_close (session_get_transport_proto (s), s->connection_index,
1616 s->thread_index);
1617}
1618
1619/**
Florin Coras2f8d8fa2018-01-26 06:36:04 -08001620 * Notify transport the session can be disconnected. This should eventually
1621 * result in a delete notification that allows us to cleanup session state.
1622 * Called for both active/passive disconnects.
1623 *
1624 * Must be called from the session's thread.
1625 */
1626void
Florin Coras288eaab2019-02-03 15:26:14 -08001627session_transport_close (session_t * s)
Florin Coras2f8d8fa2018-01-26 06:36:04 -08001628{
Florin Coras692b9492019-07-12 15:01:53 -07001629 if (s->session_state >= SESSION_STATE_APP_CLOSED)
Florin Coras568ebc72018-09-18 16:12:50 -07001630 {
Florin Coras5f066322019-07-22 19:03:03 -07001631 if (s->session_state == SESSION_STATE_TRANSPORT_CLOSED)
Steven Luongd810a6e2022-10-25 13:09:11 -07001632 session_set_state (s, SESSION_STATE_CLOSED);
Florin Coras5f066322019-07-22 19:03:03 -07001633 /* If transport is already deleted, just free the session */
1634 else if (s->session_state >= SESSION_STATE_TRANSPORT_DELETED)
Florin Coras692b9492019-07-12 15:01:53 -07001635 session_free_w_fifos (s);
Florin Coras568ebc72018-09-18 16:12:50 -07001636 return;
1637 }
Florin Coras78cc4b02018-12-20 18:24:49 -08001638
Florin Coras692b9492019-07-12 15:01:53 -07001639 /* If the tx queue wasn't drained, the transport can continue to try
1640 * sending the outstanding data (in closed state it cannot). It MUST however
1641 * at one point, either after sending everything or after a timeout, call
1642 * delete notify. This will finally lead to the complete cleanup of the
1643 * session.
Florin Coras78cc4b02018-12-20 18:24:49 -08001644 */
Steven Luongd810a6e2022-10-25 13:09:11 -07001645 session_set_state (s, SESSION_STATE_APP_CLOSED);
Florin Coras78cc4b02018-12-20 18:24:49 -08001646
Florin Coras1ee78302019-02-05 15:51:15 -08001647 transport_close (session_get_transport_proto (s), s->connection_index,
1648 s->thread_index);
Dave Barach68b0fb02017-02-28 15:15:56 -05001649}
1650
1651/**
Florin Corasdfb3b872019-08-16 17:48:44 -07001652 * Force transport close
1653 */
1654void
1655session_transport_reset (session_t * s)
1656{
1657 if (s->session_state >= SESSION_STATE_APP_CLOSED)
1658 {
1659 if (s->session_state == SESSION_STATE_TRANSPORT_CLOSED)
Steven Luongd810a6e2022-10-25 13:09:11 -07001660 session_set_state (s, SESSION_STATE_CLOSED);
Florin Corasdfb3b872019-08-16 17:48:44 -07001661 else if (s->session_state >= SESSION_STATE_TRANSPORT_DELETED)
1662 session_free_w_fifos (s);
1663 return;
1664 }
1665
Steven Luongd810a6e2022-10-25 13:09:11 -07001666 session_set_state (s, SESSION_STATE_APP_CLOSED);
Florin Corasdfb3b872019-08-16 17:48:44 -07001667 transport_reset (session_get_transport_proto (s), s->connection_index,
1668 s->thread_index);
1669}
1670
1671/**
Dave Barach68b0fb02017-02-28 15:15:56 -05001672 * Cleanup transport and session state.
Florin Corasd79b41e2017-03-04 05:37:52 -08001673 *
Florin Coras25579b42018-06-06 17:55:02 -07001674 * Notify transport of the cleanup and free the session. This should
1675 * be called only if transport reported some error and is already
1676 * closed.
Dave Barach68b0fb02017-02-28 15:15:56 -05001677 */
1678void
Florin Coras288eaab2019-02-03 15:26:14 -08001679session_transport_cleanup (session_t * s)
Dave Barach68b0fb02017-02-28 15:15:56 -05001680{
Florin Coras25579b42018-06-06 17:55:02 -07001681 /* Delete from main lookup table before we axe the the transport */
1682 session_lookup_del_session (s);
Florin Coras5afea122019-10-28 08:46:37 -07001683 if (s->session_state != SESSION_STATE_TRANSPORT_DELETED)
1684 transport_cleanup (session_get_transport_proto (s), s->connection_index,
1685 s->thread_index);
Florin Coras25579b42018-06-06 17:55:02 -07001686 /* Since we called cleanup, no delete notification will come. So, make
1687 * sure the session is properly freed. */
Florin Corasd3955fb2019-11-29 09:31:47 -08001688 segment_manager_dealloc_fifos (s->rx_fifo, s->tx_fifo);
1689 session_free (s);
Dave Barach68b0fb02017-02-28 15:15:56 -05001690}
1691
Florin Corasa5464812017-04-19 13:00:05 -07001692/**
Florin Coras8afe1b82021-11-17 23:38:54 -08001693 * Allocate worker mqs in share-able segment
Florin Corasb384b542018-01-15 01:08:33 -08001694 *
Florin Coras8afe1b82021-11-17 23:38:54 -08001695 * That can only be a newly created memfd segment, that must be mapped
1696 * by all apps/stack users unless private rx mqs are enabled.
Florin Corasa5464812017-04-19 13:00:05 -07001697 */
1698void
Florin Coras8afe1b82021-11-17 23:38:54 -08001699session_vpp_wrk_mqs_alloc (session_main_t *smm)
Florin Corasa5464812017-04-19 13:00:05 -07001700{
Florin Coras8afe1b82021-11-17 23:38:54 -08001701 u32 mq_q_length = 2048, evt_size = sizeof (session_event_t);
1702 fifo_segment_t *mqs_seg = &smm->wrk_mqs_segment;
1703 svm_msg_q_cfg_t _cfg, *cfg = &_cfg;
1704 uword mqs_seg_size;
Florin Corasb384b542018-01-15 01:08:33 -08001705 int i;
Florin Corasa5464812017-04-19 13:00:05 -07001706
Florin Coras8afe1b82021-11-17 23:38:54 -08001707 mq_q_length = clib_max (mq_q_length, smm->configured_wrk_mq_length);
Florin Corasb384b542018-01-15 01:08:33 -08001708
Florin Coras8afe1b82021-11-17 23:38:54 -08001709 svm_msg_q_ring_cfg_t rc[SESSION_MQ_N_RINGS] = {
1710 { mq_q_length, evt_size, 0 }, { mq_q_length >> 1, 256, 0 }
1711 };
1712 cfg->consumer_pid = 0;
1713 cfg->n_rings = 2;
1714 cfg->q_nitems = mq_q_length;
1715 cfg->ring_cfgs = rc;
Florin Coras6c10ab22020-11-08 16:50:39 -08001716
Florin Coras8afe1b82021-11-17 23:38:54 -08001717 /*
1718 * Compute mqs segment size based on rings config and leave space
1719 * for passing extended configuration messages, i.e., data allocated
1720 * outside of the rings. If provided with a config value, accept it
1721 * if larger than minimum size.
1722 */
1723 mqs_seg_size = svm_msg_q_size_to_alloc (cfg) * vec_len (smm->wrk);
Florin Corascf5c7742022-06-28 14:34:45 -07001724 mqs_seg_size = mqs_seg_size + (1 << 20);
Florin Coras8afe1b82021-11-17 23:38:54 -08001725 mqs_seg_size = clib_max (mqs_seg_size, smm->wrk_mqs_segment_size);
1726
1727 mqs_seg->ssvm.ssvm_size = mqs_seg_size;
1728 mqs_seg->ssvm.my_pid = getpid ();
1729 mqs_seg->ssvm.name = format (0, "%s%c", "session: wrk-mqs-segment", 0);
Florin Coras6c10ab22020-11-08 16:50:39 -08001730
Florin Coras8afe1b82021-11-17 23:38:54 -08001731 if (ssvm_server_init (&mqs_seg->ssvm, SSVM_SEGMENT_MEMFD))
Florin Corasa5464812017-04-19 13:00:05 -07001732 {
Florin Coras6c10ab22020-11-08 16:50:39 -08001733 clib_warning ("failed to initialize queue segment");
1734 return;
Florin Corasa5464812017-04-19 13:00:05 -07001735 }
Florin Corasb384b542018-01-15 01:08:33 -08001736
Florin Coras8afe1b82021-11-17 23:38:54 -08001737 fifo_segment_init (mqs_seg);
Florin Corasb384b542018-01-15 01:08:33 -08001738
Florin Corasb4624182020-12-11 13:58:12 -08001739 /* Special fifo segment that's filled only with mqs */
Florin Coras8afe1b82021-11-17 23:38:54 -08001740 mqs_seg->h->n_mqs = vec_len (smm->wrk);
Florin Corasb4624182020-12-11 13:58:12 -08001741
Florin Coras5a7ca7b2018-10-30 12:01:48 -07001742 for (i = 0; i < vec_len (smm->wrk); i++)
Florin Coras8afe1b82021-11-17 23:38:54 -08001743 smm->wrk[i].vpp_event_queue = fifo_segment_msg_q_alloc (mqs_seg, i, cfg);
Florin Corasb384b542018-01-15 01:08:33 -08001744}
1745
Florin Coras04943b42020-12-25 11:45:40 -08001746fifo_segment_t *
Florin Coras8afe1b82021-11-17 23:38:54 -08001747session_main_get_wrk_mqs_segment (void)
Florin Corasb384b542018-01-15 01:08:33 -08001748{
Florin Coras8afe1b82021-11-17 23:38:54 -08001749 return &session_main.wrk_mqs_segment;
Florin Corasa5464812017-04-19 13:00:05 -07001750}
1751
Florin Coras31c99552019-03-01 13:00:58 -08001752u64
1753session_segment_handle (session_t * s)
1754{
1755 svm_fifo_t *f;
1756
Aloys Augustincdb71702019-04-08 17:54:39 +02001757 if (!s->rx_fifo)
Florin Coras31c99552019-03-01 13:00:58 -08001758 return SESSION_INVALID_HANDLE;
1759
1760 f = s->rx_fifo;
1761 return segment_manager_make_segment_handle (f->segment_manager,
1762 f->segment_index);
1763}
1764
Florin Coras371ca502018-02-21 12:07:41 -08001765/* *INDENT-OFF* */
1766static session_fifo_rx_fn *session_tx_fns[TRANSPORT_TX_N_FNS] = {
1767 session_tx_fifo_peek_and_snd,
1768 session_tx_fifo_dequeue_and_snd,
Florin Coras7fb0fe12018-04-09 09:24:52 -07001769 session_tx_fifo_dequeue_internal,
1770 session_tx_fifo_dequeue_and_snd
Florin Coras371ca502018-02-21 12:07:41 -08001771};
1772/* *INDENT-ON* */
1773
Florin Coras561af9b2017-12-09 10:19:43 -08001774void
1775session_register_transport (transport_proto_t transport_proto,
1776 const transport_proto_vft_t * vft, u8 is_ip4,
1777 u32 output_node)
Dave Barach2c25a622017-06-26 11:35:07 -04001778{
Florin Coras31c99552019-03-01 13:00:58 -08001779 session_main_t *smm = &session_main;
Florin Coras561af9b2017-12-09 10:19:43 -08001780 session_type_t session_type;
1781 u32 next_index = ~0;
Dave Barach2c25a622017-06-26 11:35:07 -04001782
Florin Coras561af9b2017-12-09 10:19:43 -08001783 session_type = session_type_from_proto_and_ip (transport_proto, is_ip4);
1784
1785 vec_validate (smm->session_type_to_next, session_type);
Florin Coras561af9b2017-12-09 10:19:43 -08001786 vec_validate (smm->session_tx_fns, session_type);
1787
Florin Coras371ca502018-02-21 12:07:41 -08001788 if (output_node != ~0)
Florin Coraseb839cc2021-04-14 11:23:55 -07001789 next_index = vlib_node_add_next (vlib_get_main (),
1790 session_queue_node.index, output_node);
Florin Coras561af9b2017-12-09 10:19:43 -08001791
1792 smm->session_type_to_next[session_type] = next_index;
Nathan Skrzypczake971bc92019-06-19 13:42:37 +02001793 smm->session_tx_fns[session_type] =
1794 session_tx_fns[vft->transport_options.tx_type];
Dave Barach2c25a622017-06-26 11:35:07 -04001795}
1796
Florin Coras5384cca2022-01-20 18:10:26 -08001797void
1798session_register_update_time_fn (session_update_time_fn fn, u8 is_add)
1799{
1800 session_main_t *smm = &session_main;
1801 session_update_time_fn *fi;
1802 u32 fi_pos = ~0;
1803 u8 found = 0;
1804
1805 vec_foreach (fi, smm->update_time_fns)
1806 {
1807 if (*fi == fn)
1808 {
1809 fi_pos = fi - smm->update_time_fns;
1810 found = 1;
1811 break;
1812 }
1813 }
1814
1815 if (is_add)
1816 {
1817 if (found)
1818 {
1819 clib_warning ("update time fn %p already registered", fn);
1820 return;
1821 }
1822 vec_add1 (smm->update_time_fns, fn);
1823 }
1824 else
1825 {
1826 vec_del1 (smm->update_time_fns, fi_pos);
1827 }
1828}
1829
Florin Coras07063b82020-03-13 04:44:51 +00001830transport_proto_t
1831session_add_transport_proto (void)
1832{
1833 session_main_t *smm = &session_main;
1834 session_worker_t *wrk;
1835 u32 thread;
1836
1837 smm->last_transport_proto_type += 1;
1838
1839 for (thread = 0; thread < vec_len (smm->wrk); thread++)
1840 {
1841 wrk = session_main_get_worker (thread);
1842 vec_validate (wrk->session_to_enqueue, smm->last_transport_proto_type);
1843 }
1844
1845 return smm->last_transport_proto_type;
1846}
1847
Florin Coras3cbc04b2017-10-02 00:18:51 -07001848transport_connection_t *
Florin Coras288eaab2019-02-03 15:26:14 -08001849session_get_transport (session_t * s)
Florin Coras3cbc04b2017-10-02 00:18:51 -07001850{
Florin Corasade70e42017-10-14 18:56:41 -07001851 if (s->session_state != SESSION_STATE_LISTENING)
Florin Coras4edc37e2019-02-04 23:01:34 -08001852 return transport_get_connection (session_get_transport_proto (s),
1853 s->connection_index, s->thread_index);
1854 else
1855 return transport_get_listener (session_get_transport_proto (s),
1856 s->connection_index);
Florin Coras3cbc04b2017-10-02 00:18:51 -07001857}
1858
Aloys Augustincdb71702019-04-08 17:54:39 +02001859void
Florin Coras09d18c22019-04-24 11:10:02 -07001860session_get_endpoint (session_t * s, transport_endpoint_t * tep, u8 is_lcl)
Aloys Augustincdb71702019-04-08 17:54:39 +02001861{
1862 if (s->session_state != SESSION_STATE_LISTENING)
1863 return transport_get_endpoint (session_get_transport_proto (s),
Florin Coras09d18c22019-04-24 11:10:02 -07001864 s->connection_index, s->thread_index, tep,
1865 is_lcl);
Aloys Augustincdb71702019-04-08 17:54:39 +02001866 else
1867 return transport_get_listener_endpoint (session_get_transport_proto (s),
Florin Coras09d18c22019-04-24 11:10:02 -07001868 s->connection_index, tep, is_lcl);
Aloys Augustincdb71702019-04-08 17:54:39 +02001869}
1870
Florin Coras04ae8272021-04-12 19:55:37 -07001871int
1872session_transport_attribute (session_t *s, u8 is_get,
1873 transport_endpt_attr_t *attr)
1874{
1875 if (s->session_state < SESSION_STATE_READY)
1876 return -1;
1877
1878 return transport_connection_attribute (session_get_transport_proto (s),
1879 s->connection_index, s->thread_index,
1880 is_get, attr);
1881}
1882
Florin Coras3cbc04b2017-10-02 00:18:51 -07001883transport_connection_t *
Florin Coras288eaab2019-02-03 15:26:14 -08001884listen_session_get_transport (session_t * s)
Florin Coras3cbc04b2017-10-02 00:18:51 -07001885{
Florin Coras4edc37e2019-02-04 23:01:34 -08001886 return transport_get_listener (session_get_transport_proto (s),
1887 s->connection_index);
Florin Coras3cbc04b2017-10-02 00:18:51 -07001888}
1889
Florin Corasfd542f12018-05-16 19:28:24 -07001890void
Florin Coras5484daa2020-03-27 23:55:06 +00001891session_queue_run_on_main_thread (vlib_main_t * vm)
Florin Corasfd542f12018-05-16 19:28:24 -07001892{
1893 ASSERT (vlib_get_thread_index () == 0);
Florin Coras1d2e38b2021-03-17 21:52:49 -07001894 vlib_node_set_interrupt_pending (vm, session_queue_node.index);
Florin Corasfd542f12018-05-16 19:28:24 -07001895}
1896
Florin Corasd82f4712022-04-25 16:15:02 -07001897static void
1898session_stats_collector_fn (vlib_stats_collector_data_t *d)
1899{
1900 u32 i, n_workers, n_wrk_sessions, n_sessions = 0;
1901 session_main_t *smm = &session_main;
1902 session_worker_t *wrk;
1903 counter_t **counters;
1904 counter_t *cb;
1905
1906 n_workers = vec_len (smm->wrk);
1907 vlib_stats_validate (d->entry_index, 0, n_workers - 1);
1908 counters = d->entry->data;
1909 cb = counters[0];
1910
1911 for (i = 0; i < vec_len (smm->wrk); i++)
1912 {
1913 wrk = session_main_get_worker (i);
1914 n_wrk_sessions = pool_elts (wrk->sessions);
1915 cb[i] = n_wrk_sessions;
1916 n_sessions += n_wrk_sessions;
1917 }
1918
1919 vlib_stats_set_gauge (d->private_data, n_sessions);
1920}
1921
1922static void
1923session_stats_collector_init (void)
1924{
Florin Coras975e0df2022-04-26 19:32:11 -07001925 vlib_stats_collector_reg_t reg = {};
Florin Corasd82f4712022-04-25 16:15:02 -07001926
1927 reg.entry_index =
1928 vlib_stats_add_counter_vector ("/sys/session/sessions_per_worker");
1929 reg.private_data = vlib_stats_add_gauge ("/sys/session/sessions_total");
1930 reg.collect_fn = session_stats_collector_fn;
1931 vlib_stats_register_collector_fn (&reg);
1932 vlib_stats_validate (reg.entry_index, 0, vlib_get_n_threads ());
1933}
1934
Dave Barach68b0fb02017-02-28 15:15:56 -05001935static clib_error_t *
Florin Corase04c2992017-03-01 08:17:34 -08001936session_manager_main_enable (vlib_main_t * vm)
Dave Barach68b0fb02017-02-28 15:15:56 -05001937{
Florin Coras31c99552019-03-01 13:00:58 -08001938 session_main_t *smm = &session_main;
Florin Corase04c2992017-03-01 08:17:34 -08001939 vlib_thread_main_t *vtm = vlib_get_thread_main ();
Florin Coras371ca502018-02-21 12:07:41 -08001940 u32 num_threads, preallocated_sessions_per_worker;
Florin Coras31c99552019-03-01 13:00:58 -08001941 session_worker_t *wrk;
Florin Corasd5c604d2019-03-18 09:06:35 -07001942 int i;
Dave Barach68b0fb02017-02-28 15:15:56 -05001943
Florin Corase10da622020-10-25 14:00:29 -07001944 /* We only initialize once and do not de-initialized on disable */
1945 if (smm->is_initialized)
1946 goto done;
1947
Dave Barach68b0fb02017-02-28 15:15:56 -05001948 num_threads = 1 /* main thread */ + vtm->n_threads;
1949
1950 if (num_threads < 1)
1951 return clib_error_return (0, "n_thread_stacks not set");
1952
Florin Coras5f56d732018-10-30 10:21:59 -07001953 /* Allocate cache line aligned worker contexts */
Florin Coras5a7ca7b2018-10-30 12:01:48 -07001954 vec_validate_aligned (smm->wrk, num_threads - 1, CLIB_CACHE_LINE_BYTES);
Florin Coras53d8d4f2022-03-09 13:55:38 -08001955 clib_spinlock_init (&session_main.pool_realloc_lock);
Florin Coras3cbc04b2017-10-02 00:18:51 -07001956
Dave Barachacd2a6a2017-05-16 17:41:34 -04001957 for (i = 0; i < num_threads; i++)
1958 {
Florin Coras5a7ca7b2018-10-30 12:01:48 -07001959 wrk = &smm->wrk[i];
Florin Corasb0ffbee2019-07-21 19:23:46 -07001960 wrk->ctrl_head = clib_llist_make_head (wrk->event_elts, evt_list);
Florin Coras2062ec02019-07-15 13:15:18 -07001961 wrk->new_head = clib_llist_make_head (wrk->event_elts, evt_list);
Florin Coras60183db2019-07-20 15:53:16 -07001962 wrk->old_head = clib_llist_make_head (wrk->event_elts, evt_list);
Florin Corasaf972212021-05-05 09:54:00 -07001963 wrk->pending_connects = clib_llist_make_head (wrk->event_elts, evt_list);
Florin Corase09bd482022-03-21 10:38:01 -07001964 wrk->evts_pending_main =
1965 clib_llist_make_head (wrk->event_elts, evt_list);
Damjan Marion6ffb7c62021-03-26 13:06:13 +01001966 wrk->vm = vlib_get_main_by_index (i);
Dave Barach77d98382020-04-28 18:00:21 -04001967 wrk->last_vlib_time = vlib_time_now (vm);
Florin Corasa8e71c82019-10-22 19:01:39 -07001968 wrk->last_vlib_us_time = wrk->last_vlib_time * CLIB_US_TIME_FREQ;
Florin Corasa48dffe2021-05-16 10:58:53 -07001969 wrk->timerfd = -1;
Florin Coras07063b82020-03-13 04:44:51 +00001970 vec_validate (wrk->session_to_enqueue, smm->last_transport_proto_type);
Florin Corasd67f1122018-05-21 17:47:40 -07001971
Florin Coras7da88292021-03-18 15:04:34 -07001972 if (!smm->no_adaptive && smm->use_private_rx_mqs)
1973 session_wrk_enable_adaptive_mode (wrk);
Dave Barachacd2a6a2017-05-16 17:41:34 -04001974 }
1975
Florin Corasb384b542018-01-15 01:08:33 -08001976 /* Allocate vpp event queues segment and queue */
Florin Coras8afe1b82021-11-17 23:38:54 -08001977 session_vpp_wrk_mqs_alloc (smm);
Florin Corasb384b542018-01-15 01:08:33 -08001978
Florin Coras9a45bd82020-12-28 16:28:07 -08001979 /* Initialize segment manager properties */
1980 segment_manager_main_init ();
Florin Coras6cf30ad2017-04-04 23:08:23 -07001981
Florin Coras66b11312017-07-31 17:18:03 -07001982 /* Preallocate sessions */
Dave Barachb7f1faa2017-08-29 11:43:37 -04001983 if (smm->preallocated_sessions)
Dave Barach68b0fb02017-02-28 15:15:56 -05001984 {
Dave Barachb7f1faa2017-08-29 11:43:37 -04001985 if (num_threads == 1)
Florin Coras66b11312017-07-31 17:18:03 -07001986 {
Florin Coras5a7ca7b2018-10-30 12:01:48 -07001987 pool_init_fixed (smm->wrk[0].sessions, smm->preallocated_sessions);
Florin Coras66b11312017-07-31 17:18:03 -07001988 }
Dave Barachb7f1faa2017-08-29 11:43:37 -04001989 else
Florin Coras66b11312017-07-31 17:18:03 -07001990 {
Dave Barachb7f1faa2017-08-29 11:43:37 -04001991 int j;
1992 preallocated_sessions_per_worker =
1993 (1.1 * (f64) smm->preallocated_sessions /
1994 (f64) (num_threads - 1));
1995
1996 for (j = 1; j < num_threads; j++)
Florin Coras66b11312017-07-31 17:18:03 -07001997 {
Florin Coras5a7ca7b2018-10-30 12:01:48 -07001998 pool_init_fixed (smm->wrk[j].sessions,
Dave Barachb7f1faa2017-08-29 11:43:37 -04001999 preallocated_sessions_per_worker);
Florin Coras66b11312017-07-31 17:18:03 -07002000 }
Florin Coras66b11312017-07-31 17:18:03 -07002001 }
2002 }
Dave Barach68b0fb02017-02-28 15:15:56 -05002003
Florin Coras04e53442017-07-16 17:12:15 -07002004 session_lookup_init ();
Florin Corascea194d2017-10-02 00:18:51 -07002005 app_namespaces_init ();
Florin Coras3cbc04b2017-10-02 00:18:51 -07002006 transport_init ();
Florin Corasd82f4712022-04-25 16:15:02 -07002007 session_stats_collector_init ();
Florin Corase10da622020-10-25 14:00:29 -07002008 smm->is_initialized = 1;
2009
2010done:
Dave Barach68b0fb02017-02-28 15:15:56 -05002011
Florin Corase04c2992017-03-01 08:17:34 -08002012 smm->is_enabled = 1;
2013
Florin Coras561af9b2017-12-09 10:19:43 -08002014 /* Enable transports */
2015 transport_enable_disable (vm, 1);
Florin Coras11166672020-04-13 01:20:25 +00002016 session_debug_init ();
Srikanth Akula73570432020-04-06 19:19:49 -07002017
Dave Barach68b0fb02017-02-28 15:15:56 -05002018 return 0;
2019}
2020
Florin Corase10da622020-10-25 14:00:29 -07002021static void
2022session_manager_main_disable (vlib_main_t * vm)
2023{
2024 transport_enable_disable (vm, 0 /* is_en */ );
2025}
2026
Marvin Liu06542422022-08-16 06:49:09 +00002027/* in this new callback, cookie hint the index */
2028void
2029session_dma_completion_cb (vlib_main_t *vm, struct vlib_dma_batch *batch)
2030{
2031 session_worker_t *wrk;
2032 wrk = session_main_get_worker (vm->thread_index);
2033 session_dma_transfer *dma_transfer;
2034
2035 dma_transfer = &wrk->dma_trans[wrk->trans_head];
2036 vec_add (wrk->pending_tx_buffers, dma_transfer->pending_tx_buffers,
2037 vec_len (dma_transfer->pending_tx_buffers));
2038 vec_add (wrk->pending_tx_nexts, dma_transfer->pending_tx_nexts,
2039 vec_len (dma_transfer->pending_tx_nexts));
2040 vec_reset_length (dma_transfer->pending_tx_buffers);
2041 vec_reset_length (dma_transfer->pending_tx_nexts);
2042 wrk->trans_head++;
2043 if (wrk->trans_head == wrk->trans_size)
2044 wrk->trans_head = 0;
2045 return;
2046}
2047
2048static void
2049session_prepare_dma_args (vlib_dma_config_t *args)
2050{
Marvin Liu48d2e152023-03-14 23:56:31 +08002051 args->max_batches = 16;
Marvin Liu06542422022-08-16 06:49:09 +00002052 args->max_transfers = DMA_TRANS_SIZE;
2053 args->max_transfer_size = 65536;
2054 args->features = 0;
2055 args->sw_fallback = 1;
2056 args->barrier_before_last = 1;
2057 args->callback_fn = session_dma_completion_cb;
2058}
2059
2060static void
2061session_node_enable_dma (u8 is_en, int n_vlibs)
2062{
2063 vlib_dma_config_t args;
2064 session_prepare_dma_args (&args);
2065 session_worker_t *wrk;
2066 vlib_main_t *vm;
2067
2068 int config_index = -1;
2069
2070 if (is_en)
2071 {
2072 vm = vlib_get_main_by_index (0);
2073 config_index = vlib_dma_config_add (vm, &args);
2074 }
2075 else
2076 {
2077 vm = vlib_get_main_by_index (0);
2078 wrk = session_main_get_worker (0);
2079 if (wrk->config_index >= 0)
2080 vlib_dma_config_del (vm, wrk->config_index);
2081 }
2082 int i;
2083 for (i = 0; i < n_vlibs; i++)
2084 {
2085 vm = vlib_get_main_by_index (i);
2086 wrk = session_main_get_worker (vm->thread_index);
2087 wrk->config_index = config_index;
2088 if (is_en)
2089 {
2090 if (config_index >= 0)
2091 wrk->dma_enabled = true;
2092 wrk->dma_trans = (session_dma_transfer *) clib_mem_alloc (
2093 sizeof (session_dma_transfer) * DMA_TRANS_SIZE);
2094 bzero (wrk->dma_trans,
2095 sizeof (session_dma_transfer) * DMA_TRANS_SIZE);
2096 }
2097 else
2098 {
2099 if (wrk->dma_trans)
2100 clib_mem_free (wrk->dma_trans);
2101 }
2102 wrk->trans_head = 0;
2103 wrk->trans_tail = 0;
2104 wrk->trans_size = DMA_TRANS_SIZE;
2105 }
2106}
2107
Florin Corasa5464812017-04-19 13:00:05 -07002108void
2109session_node_enable_disable (u8 is_en)
2110{
Florin Coras1d2e38b2021-03-17 21:52:49 -07002111 u8 mstate = is_en ? VLIB_NODE_STATE_INTERRUPT : VLIB_NODE_STATE_DISABLED;
Florin Corasa5464812017-04-19 13:00:05 -07002112 u8 state = is_en ? VLIB_NODE_STATE_POLLING : VLIB_NODE_STATE_DISABLED;
Florin Coras1d2e38b2021-03-17 21:52:49 -07002113 session_main_t *sm = &session_main;
2114 vlib_main_t *vm;
2115 vlib_node_t *n;
2116 int n_vlibs, i;
Florin Corasfd542f12018-05-16 19:28:24 -07002117
Florin Coras1d2e38b2021-03-17 21:52:49 -07002118 n_vlibs = vlib_get_n_threads ();
2119 for (i = 0; i < n_vlibs; i++)
Damjan Marion92ccf9b2021-03-26 11:38:01 +01002120 {
Florin Coras1d2e38b2021-03-17 21:52:49 -07002121 vm = vlib_get_main_by_index (i);
2122 /* main thread with workers and not polling */
2123 if (i == 0 && n_vlibs > 1)
Damjan Marion92ccf9b2021-03-26 11:38:01 +01002124 {
Florin Coras1d2e38b2021-03-17 21:52:49 -07002125 vlib_node_set_state (vm, session_queue_node.index, mstate);
Damjan Marion92ccf9b2021-03-26 11:38:01 +01002126 if (is_en)
2127 {
Florin Corasb32af352021-05-19 07:58:49 -07002128 session_main_get_worker (0)->state = SESSION_WRK_INTERRUPT;
Florin Coras1d2e38b2021-03-17 21:52:49 -07002129 vlib_node_set_state (vm, session_queue_process_node.index,
2130 state);
2131 n = vlib_get_node (vm, session_queue_process_node.index);
2132 vlib_start_process (vm, n->runtime_index);
Damjan Marion92ccf9b2021-03-26 11:38:01 +01002133 }
2134 else
2135 {
Florin Coras1d2e38b2021-03-17 21:52:49 -07002136 vlib_process_signal_event_mt (vm,
Damjan Marion92ccf9b2021-03-26 11:38:01 +01002137 session_queue_process_node.index,
2138 SESSION_Q_PROCESS_STOP, 0);
2139 }
Florin Coras1d2e38b2021-03-17 21:52:49 -07002140 if (!sm->poll_main)
Damjan Marion92ccf9b2021-03-26 11:38:01 +01002141 continue;
2142 }
Florin Coras1d2e38b2021-03-17 21:52:49 -07002143 vlib_node_set_state (vm, session_queue_node.index, state);
Damjan Marion92ccf9b2021-03-26 11:38:01 +01002144 }
Florin Coras41d5f542021-01-15 13:49:33 -08002145
Florin Coras1d2e38b2021-03-17 21:52:49 -07002146 if (sm->use_private_rx_mqs)
Florin Coras41d5f542021-01-15 13:49:33 -08002147 application_enable_rx_mqs_nodes (is_en);
Marvin Liu06542422022-08-16 06:49:09 +00002148
2149 if (sm->dma_enabled)
2150 session_node_enable_dma (is_en, n_vlibs);
Florin Corasa5464812017-04-19 13:00:05 -07002151}
2152
Florin Corase04c2992017-03-01 08:17:34 -08002153clib_error_t *
2154vnet_session_enable_disable (vlib_main_t * vm, u8 is_en)
2155{
Florin Corascea194d2017-10-02 00:18:51 -07002156 clib_error_t *error = 0;
Florin Corase04c2992017-03-01 08:17:34 -08002157 if (is_en)
2158 {
Florin Coras31c99552019-03-01 13:00:58 -08002159 if (session_main.is_enabled)
Florin Corase04c2992017-03-01 08:17:34 -08002160 return 0;
Dave Barach68b0fb02017-02-28 15:15:56 -05002161
Florin Corascea194d2017-10-02 00:18:51 -07002162 error = session_manager_main_enable (vm);
Vladimir Kropyleva2e44512019-07-16 21:32:41 +03002163 session_node_enable_disable (is_en);
Florin Corase04c2992017-03-01 08:17:34 -08002164 }
2165 else
2166 {
Florin Coras31c99552019-03-01 13:00:58 -08002167 session_main.is_enabled = 0;
Florin Corase10da622020-10-25 14:00:29 -07002168 session_manager_main_disable (vm);
Florin Corasa5464812017-04-19 13:00:05 -07002169 session_node_enable_disable (is_en);
Florin Corase04c2992017-03-01 08:17:34 -08002170 }
2171
Florin Corascea194d2017-10-02 00:18:51 -07002172 return error;
Florin Corase04c2992017-03-01 08:17:34 -08002173}
2174
Florin Corase04c2992017-03-01 08:17:34 -08002175clib_error_t *
Florin Corase33c0022020-04-03 17:23:42 +00002176session_main_init (vlib_main_t * vm)
Florin Corase04c2992017-03-01 08:17:34 -08002177{
Florin Coras31c99552019-03-01 13:00:58 -08002178 session_main_t *smm = &session_main;
Florin Corase33c0022020-04-03 17:23:42 +00002179
2180 smm->is_enabled = 0;
2181 smm->session_enable_asap = 0;
Florin Corase92c9462020-11-25 08:44:16 -08002182 smm->poll_main = 0;
Florin Coras41d5f542021-01-15 13:49:33 -08002183 smm->use_private_rx_mqs = 0;
Florin Coras7da88292021-03-18 15:04:34 -07002184 smm->no_adaptive = 0;
Florin Coras0b656212022-01-13 11:59:44 -08002185 smm->last_transport_proto_type = TRANSPORT_PROTO_HTTP;
Florin Corase33c0022020-04-03 17:23:42 +00002186
Nathan Skrzypczak31bd0352019-11-07 17:55:01 +01002187 return 0;
2188}
2189
2190static clib_error_t *
Florin Corase33c0022020-04-03 17:23:42 +00002191session_main_loop_init (vlib_main_t * vm)
Nathan Skrzypczak31bd0352019-11-07 17:55:01 +01002192{
2193 session_main_t *smm = &session_main;
2194 if (smm->session_enable_asap)
2195 {
2196 vlib_worker_thread_barrier_sync (vm);
2197 vnet_session_enable_disable (vm, 1 /* is_en */ );
2198 vlib_worker_thread_barrier_release (vm);
2199 }
Florin Corase04c2992017-03-01 08:17:34 -08002200 return 0;
2201}
2202
Florin Corase33c0022020-04-03 17:23:42 +00002203VLIB_INIT_FUNCTION (session_main_init);
2204VLIB_MAIN_LOOP_ENTER_FUNCTION (session_main_loop_init);
Dave Barach2c25a622017-06-26 11:35:07 -04002205
2206static clib_error_t *
2207session_config_fn (vlib_main_t * vm, unformat_input_t * input)
Dave Barach10d8cc62017-05-30 09:30:07 -04002208{
Florin Coras31c99552019-03-01 13:00:58 -08002209 session_main_t *smm = &session_main;
Dave Barach10d8cc62017-05-30 09:30:07 -04002210 u32 nitems;
Florin Coras66b11312017-07-31 17:18:03 -07002211 uword tmp;
Dave Barach10d8cc62017-05-30 09:30:07 -04002212
2213 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2214 {
Florin Coras8afe1b82021-11-17 23:38:54 -08002215 if (unformat (input, "wrk-mq-length %d", &nitems))
Dave Barach10d8cc62017-05-30 09:30:07 -04002216 {
2217 if (nitems >= 2048)
Florin Coras8afe1b82021-11-17 23:38:54 -08002218 smm->configured_wrk_mq_length = nitems;
Dave Barach10d8cc62017-05-30 09:30:07 -04002219 else
2220 clib_warning ("event queue length %d too small, ignored", nitems);
2221 }
Florin Corascf5c7742022-06-28 14:34:45 -07002222 else if (unformat (input, "wrk-mqs-segment-size %U",
2223 unformat_memory_size, &smm->wrk_mqs_segment_size))
2224 ;
Florin Coras66b11312017-07-31 17:18:03 -07002225 else if (unformat (input, "preallocated-sessions %d",
2226 &smm->preallocated_sessions))
Dave Barach2c25a622017-06-26 11:35:07 -04002227 ;
Florin Coras66b11312017-07-31 17:18:03 -07002228 else if (unformat (input, "v4-session-table-buckets %d",
2229 &smm->configured_v4_session_table_buckets))
2230 ;
2231 else if (unformat (input, "v4-halfopen-table-buckets %d",
2232 &smm->configured_v4_halfopen_table_buckets))
2233 ;
2234 else if (unformat (input, "v6-session-table-buckets %d",
2235 &smm->configured_v6_session_table_buckets))
2236 ;
2237 else if (unformat (input, "v6-halfopen-table-buckets %d",
2238 &smm->configured_v6_halfopen_table_buckets))
2239 ;
2240 else if (unformat (input, "v4-session-table-memory %U",
2241 unformat_memory_size, &tmp))
2242 {
2243 if (tmp >= 0x100000000)
2244 return clib_error_return (0, "memory size %llx (%lld) too large",
2245 tmp, tmp);
2246 smm->configured_v4_session_table_memory = tmp;
2247 }
2248 else if (unformat (input, "v4-halfopen-table-memory %U",
2249 unformat_memory_size, &tmp))
2250 {
2251 if (tmp >= 0x100000000)
2252 return clib_error_return (0, "memory size %llx (%lld) too large",
2253 tmp, tmp);
2254 smm->configured_v4_halfopen_table_memory = tmp;
2255 }
2256 else if (unformat (input, "v6-session-table-memory %U",
2257 unformat_memory_size, &tmp))
2258 {
2259 if (tmp >= 0x100000000)
2260 return clib_error_return (0, "memory size %llx (%lld) too large",
2261 tmp, tmp);
2262 smm->configured_v6_session_table_memory = tmp;
2263 }
2264 else if (unformat (input, "v6-halfopen-table-memory %U",
2265 unformat_memory_size, &tmp))
2266 {
2267 if (tmp >= 0x100000000)
2268 return clib_error_return (0, "memory size %llx (%lld) too large",
2269 tmp, tmp);
2270 smm->configured_v6_halfopen_table_memory = tmp;
2271 }
Florin Coras93e65802017-11-29 00:07:11 -05002272 else if (unformat (input, "local-endpoints-table-memory %U",
2273 unformat_memory_size, &tmp))
2274 {
2275 if (tmp >= 0x100000000)
2276 return clib_error_return (0, "memory size %llx (%lld) too large",
2277 tmp, tmp);
2278 smm->local_endpoints_table_memory = tmp;
2279 }
2280 else if (unformat (input, "local-endpoints-table-buckets %d",
2281 &smm->local_endpoints_table_buckets))
2282 ;
Nathan Skrzypczak1292d192019-09-13 15:44:54 +02002283 else if (unformat (input, "enable"))
Nathan Skrzypczak31bd0352019-11-07 17:55:01 +01002284 smm->session_enable_asap = 1;
Florin Coras61ae0562020-09-02 19:10:28 -07002285 else if (unformat (input, "use-app-socket-api"))
Nathan Skrzypczak7b3a3df2021-07-28 14:09:50 +02002286 (void) appns_sapi_enable_disable (1 /* is_enable */);
Florin Corase92c9462020-11-25 08:44:16 -08002287 else if (unformat (input, "poll-main"))
2288 smm->poll_main = 1;
Florin Coras41d5f542021-01-15 13:49:33 -08002289 else if (unformat (input, "use-private-rx-mqs"))
2290 smm->use_private_rx_mqs = 1;
Florin Coras7da88292021-03-18 15:04:34 -07002291 else if (unformat (input, "no-adaptive"))
2292 smm->no_adaptive = 1;
Marvin Liu06542422022-08-16 06:49:09 +00002293 else if (unformat (input, "use-dma"))
2294 smm->dma_enabled = 1;
Florin Coras8afe1b82021-11-17 23:38:54 -08002295 /*
2296 * Deprecated but maintained for compatibility
2297 */
2298 else if (unformat (input, "evt_qs_memfd_seg"))
2299 ;
Florin Corasef048032021-11-17 23:57:30 -08002300 else if (unformat (input, "segment-baseva 0x%lx", &tmp))
2301 ;
Florin Coras8afe1b82021-11-17 23:38:54 -08002302 else if (unformat (input, "evt_qs_seg_size %U", unformat_memory_size,
Florin Corascf5c7742022-06-28 14:34:45 -07002303 &smm->wrk_mqs_segment_size))
Florin Coras8afe1b82021-11-17 23:38:54 -08002304 ;
2305 else if (unformat (input, "event-queue-length %d", &nitems))
2306 {
2307 if (nitems >= 2048)
2308 smm->configured_wrk_mq_length = nitems;
2309 else
2310 clib_warning ("event queue length %d too small, ignored", nitems);
2311 }
Dave Barach10d8cc62017-05-30 09:30:07 -04002312 else
2313 return clib_error_return (0, "unknown input `%U'",
2314 format_unformat_error, input);
2315 }
2316 return 0;
2317}
2318
2319VLIB_CONFIG_FUNCTION (session_config_fn, "session");
2320
Dave Barach68b0fb02017-02-28 15:15:56 -05002321/*
2322 * fd.io coding-style-patch-verification: ON
2323 *
2324 * Local Variables:
2325 * eval: (c-set-style "gnu")
2326 * End:
2327 */