blob: b0dbb9ddfe8246261b5d512777260a5d010a95b9 [file] [log] [blame]
Dave Barach68b0fb02017-02-28 15:15:56 -05001/*
Florin Coras288eaab2019-02-03 15:26:14 -08002 * Copyright (c) 2017-2019 Cisco and/or its affiliates.
Dave Barach68b0fb02017-02-28 15:15:56 -05003 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15/**
16 * @file
17 * @brief Session and session manager
18 */
19
20#include <vnet/session/session.h>
Florin Coras04e53442017-07-16 17:12:15 -070021#include <vnet/session/application.h>
Dave Barach68b0fb02017-02-28 15:15:56 -050022#include <vnet/dpo/load_balance.h>
23#include <vnet/fib/ip4_fib.h>
Florin Corasd82f4712022-04-25 16:15:02 -070024#include <vlib/stats/stats.h>
Marvin Liu06542422022-08-16 06:49:09 +000025#include <vlib/dma/dma.h>
Dave Barach68b0fb02017-02-28 15:15:56 -050026
Florin Coras31c99552019-03-01 13:00:58 -080027session_main_t session_main;
Florin Coras3eb50622017-07-13 01:24:57 -040028
Florin Coras3c2fed52018-07-04 04:15:05 -070029static inline int
30session_send_evt_to_thread (void *data, void *args, u32 thread_index,
31 session_evt_type_t evt_type)
Florin Coras3cbc04b2017-10-02 00:18:51 -070032{
Florin Coras7da88292021-03-18 15:04:34 -070033 session_worker_t *wrk = session_main_get_worker (thread_index);
Florin Coras52207f12018-07-12 14:48:06 -070034 session_event_t *evt;
Florin Coras3c2fed52018-07-04 04:15:05 -070035 svm_msg_q_msg_t msg;
36 svm_msg_q_t *mq;
Florin Coras3cbc04b2017-10-02 00:18:51 -070037
Florin Coras7da88292021-03-18 15:04:34 -070038 mq = wrk->vpp_event_queue;
Nathan Skrzypczakb4c67752019-06-20 09:58:37 +020039 if (PREDICT_FALSE (svm_msg_q_lock (mq)))
40 return -1;
Florin Coras80d100c2022-04-19 18:57:24 -070041 if (PREDICT_FALSE (svm_msg_q_or_ring_is_full (mq, SESSION_MQ_IO_EVT_RING)))
Florin Coras3c2fed52018-07-04 04:15:05 -070042 {
43 svm_msg_q_unlock (mq);
44 return -2;
45 }
Florin Coras3c2fed52018-07-04 04:15:05 -070046 switch (evt_type)
47 {
Florin Corasf6c43132019-03-01 12:41:21 -080048 case SESSION_CTRL_EVT_RPC:
Nathan Skrzypczak1afa7af2019-09-13 17:14:57 +020049 msg = svm_msg_q_alloc_msg_w_ring (mq, SESSION_MQ_IO_EVT_RING);
50 evt = (session_event_t *) svm_msg_q_msg_data (mq, &msg);
Florin Coras3c2fed52018-07-04 04:15:05 -070051 evt->rpc_args.fp = data;
52 evt->rpc_args.arg = args;
53 break;
Aloys Augustin0c7f54d2019-07-03 16:59:43 +020054 case SESSION_IO_EVT_RX:
Florin Corasf6c43132019-03-01 12:41:21 -080055 case SESSION_IO_EVT_TX:
Florin Coras844a36d2018-12-20 09:50:50 -080056 case SESSION_IO_EVT_TX_FLUSH:
Florin Corasf6c43132019-03-01 12:41:21 -080057 case SESSION_IO_EVT_BUILTIN_RX:
Nathan Skrzypczak1afa7af2019-09-13 17:14:57 +020058 msg = svm_msg_q_alloc_msg_w_ring (mq, SESSION_MQ_IO_EVT_RING);
59 evt = (session_event_t *) svm_msg_q_msg_data (mq, &msg);
Florin Corasc0737e92019-03-04 14:19:39 -080060 evt->session_index = *(u32 *) data;
Florin Coras3c2fed52018-07-04 04:15:05 -070061 break;
Florin Corasf6c43132019-03-01 12:41:21 -080062 case SESSION_IO_EVT_BUILTIN_TX:
63 case SESSION_CTRL_EVT_CLOSE:
Florin Coras73cad332019-08-28 17:12:32 -070064 case SESSION_CTRL_EVT_RESET:
Nathan Skrzypczak1afa7af2019-09-13 17:14:57 +020065 msg = svm_msg_q_alloc_msg_w_ring (mq, SESSION_MQ_IO_EVT_RING);
66 evt = (session_event_t *) svm_msg_q_msg_data (mq, &msg);
Florin Coras288eaab2019-02-03 15:26:14 -080067 evt->session_handle = session_handle ((session_t *) data);
Florin Coras3c2fed52018-07-04 04:15:05 -070068 break;
69 default:
70 clib_warning ("evt unhandled!");
71 svm_msg_q_unlock (mq);
72 return -1;
73 }
Nathan Skrzypczak1afa7af2019-09-13 17:14:57 +020074 evt->event_type = evt_type;
Florin Coras3c2fed52018-07-04 04:15:05 -070075
Florin Coras52207f12018-07-12 14:48:06 -070076 svm_msg_q_add_and_unlock (mq, &msg);
Florin Coras7da88292021-03-18 15:04:34 -070077
78 if (PREDICT_FALSE (wrk->state == SESSION_WRK_INTERRUPT))
79 vlib_node_set_interrupt_pending (wrk->vm, session_queue_node.index);
80
Florin Coras3c2fed52018-07-04 04:15:05 -070081 return 0;
Florin Coras3cbc04b2017-10-02 00:18:51 -070082}
83
Florin Coras3c2fed52018-07-04 04:15:05 -070084int
85session_send_io_evt_to_thread (svm_fifo_t * f, session_evt_type_t evt_type)
Florin Coras3cbc04b2017-10-02 00:18:51 -070086{
Florin Corasc547e912020-12-08 17:50:45 -080087 return session_send_evt_to_thread (&f->shr->master_session_index, 0,
Florin Corasc0737e92019-03-04 14:19:39 -080088 f->master_thread_index, evt_type);
Florin Coras3c2fed52018-07-04 04:15:05 -070089}
90
91int
Florin Corasef915342018-09-29 10:23:06 -070092session_send_io_evt_to_thread_custom (void *data, u32 thread_index,
Florin Coras3c2fed52018-07-04 04:15:05 -070093 session_evt_type_t evt_type)
94{
Florin Corasef915342018-09-29 10:23:06 -070095 return session_send_evt_to_thread (data, 0, thread_index, evt_type);
Florin Coras3c2fed52018-07-04 04:15:05 -070096}
97
98int
Florin Coras288eaab2019-02-03 15:26:14 -080099session_send_ctrl_evt_to_thread (session_t * s, session_evt_type_t evt_type)
Florin Coras3c2fed52018-07-04 04:15:05 -0700100{
liuyacan534468e2021-05-09 03:50:40 +0000101 /* only events supported are disconnect, shutdown and reset */
102 ASSERT (evt_type == SESSION_CTRL_EVT_CLOSE ||
103 evt_type == SESSION_CTRL_EVT_HALF_CLOSE ||
104 evt_type == SESSION_CTRL_EVT_RESET);
Florin Coras458089b2019-08-21 16:20:44 -0700105 return session_send_evt_to_thread (s, 0, s->thread_index, evt_type);
Florin Coras3cbc04b2017-10-02 00:18:51 -0700106}
107
108void
Nathan Skrzypczak60f3e652019-03-19 13:57:31 +0100109session_send_rpc_evt_to_thread_force (u32 thread_index, void *fp,
110 void *rpc_args)
111{
112 session_send_evt_to_thread (fp, rpc_args, thread_index,
113 SESSION_CTRL_EVT_RPC);
114}
115
116void
Florin Coras3cbc04b2017-10-02 00:18:51 -0700117session_send_rpc_evt_to_thread (u32 thread_index, void *fp, void *rpc_args)
118{
119 if (thread_index != vlib_get_thread_index ())
Nathan Skrzypczak60f3e652019-03-19 13:57:31 +0100120 session_send_rpc_evt_to_thread_force (thread_index, fp, rpc_args);
Florin Coras3cbc04b2017-10-02 00:18:51 -0700121 else
122 {
123 void (*fnp) (void *) = fp;
124 fnp (rpc_args);
125 }
126}
127
Florin Coras26dd6de2019-07-23 23:54:47 -0700128void
129session_add_self_custom_tx_evt (transport_connection_t * tc, u8 has_prio)
130{
Florin Coras7da88292021-03-18 15:04:34 -0700131 session_t *s = session_get (tc->s_index, tc->thread_index);
Florin Coras26dd6de2019-07-23 23:54:47 -0700132
Florin Coras26dd6de2019-07-23 23:54:47 -0700133 ASSERT (s->thread_index == vlib_get_thread_index ());
Florin Corasbe237bf2019-09-27 08:16:40 -0700134 ASSERT (s->session_state != SESSION_STATE_TRANSPORT_DELETED);
Florin Coras7da88292021-03-18 15:04:34 -0700135
Florin Coras26dd6de2019-07-23 23:54:47 -0700136 if (!(s->flags & SESSION_F_CUSTOM_TX))
137 {
138 s->flags |= SESSION_F_CUSTOM_TX;
Florin Coras6080e0d2020-03-13 20:39:43 +0000139 if (svm_fifo_set_event (s->tx_fifo)
140 || transport_connection_is_descheduled (tc))
Florin Coras26dd6de2019-07-23 23:54:47 -0700141 {
Florin Coras26dd6de2019-07-23 23:54:47 -0700142 session_evt_elt_t *elt;
Florin Coras7da88292021-03-18 15:04:34 -0700143 session_worker_t *wrk;
144
Florin Coras26dd6de2019-07-23 23:54:47 -0700145 wrk = session_main_get_worker (tc->thread_index);
146 if (has_prio)
147 elt = session_evt_alloc_new (wrk);
148 else
149 elt = session_evt_alloc_old (wrk);
150 elt->evt.session_index = tc->s_index;
151 elt->evt.event_type = SESSION_IO_EVT_TX;
Florin Coras6080e0d2020-03-13 20:39:43 +0000152 tc->flags &= ~TRANSPORT_CONNECTION_F_DESCHED;
Florin Coras7da88292021-03-18 15:04:34 -0700153
154 if (PREDICT_FALSE (wrk->state == SESSION_WRK_INTERRUPT))
155 vlib_node_set_interrupt_pending (wrk->vm,
156 session_queue_node.index);
Florin Coras26dd6de2019-07-23 23:54:47 -0700157 }
158 }
159}
160
Florin Coras70f879d2020-03-13 17:54:42 +0000161void
162sesssion_reschedule_tx (transport_connection_t * tc)
163{
164 session_worker_t *wrk = session_main_get_worker (tc->thread_index);
165 session_evt_elt_t *elt;
166
167 ASSERT (tc->thread_index == vlib_get_thread_index ());
168
169 elt = session_evt_alloc_new (wrk);
170 elt->evt.session_index = tc->s_index;
171 elt->evt.event_type = SESSION_IO_EVT_TX;
Florin Coras7da88292021-03-18 15:04:34 -0700172
173 if (PREDICT_FALSE (wrk->state == SESSION_WRK_INTERRUPT))
174 vlib_node_set_interrupt_pending (wrk->vm, session_queue_node.index);
Florin Coras70f879d2020-03-13 17:54:42 +0000175}
176
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800177static void
Florin Corasdfb3b872019-08-16 17:48:44 -0700178session_program_transport_ctrl_evt (session_t * s, session_evt_type_t evt)
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800179{
180 u32 thread_index = vlib_get_thread_index ();
Florin Coras2062ec02019-07-15 13:15:18 -0700181 session_evt_elt_t *elt;
Florin Coras31c99552019-03-01 13:00:58 -0800182 session_worker_t *wrk;
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800183
184 /* If we are in the handler thread, or being called with the worker barrier
185 * held, just append a new event to pending disconnects vector. */
186 if (vlib_thread_is_main_w_barrier () || thread_index == s->thread_index)
187 {
Florin Coras31c99552019-03-01 13:00:58 -0800188 wrk = session_main_get_worker (s->thread_index);
Florin Corasb0ffbee2019-07-21 19:23:46 -0700189 elt = session_evt_alloc_ctrl (wrk);
Florin Coras2062ec02019-07-15 13:15:18 -0700190 clib_memset (&elt->evt, 0, sizeof (session_event_t));
191 elt->evt.session_handle = session_handle (s);
Florin Corasdfb3b872019-08-16 17:48:44 -0700192 elt->evt.event_type = evt;
Florin Coras7da88292021-03-18 15:04:34 -0700193
194 if (PREDICT_FALSE (wrk->state == SESSION_WRK_INTERRUPT))
195 vlib_node_set_interrupt_pending (wrk->vm, session_queue_node.index);
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800196 }
197 else
Florin Corasdfb3b872019-08-16 17:48:44 -0700198 session_send_ctrl_evt_to_thread (s, evt);
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800199}
200
Florin Coras288eaab2019-02-03 15:26:14 -0800201session_t *
Florin Coras3cbc04b2017-10-02 00:18:51 -0700202session_alloc (u32 thread_index)
Dave Barach68b0fb02017-02-28 15:15:56 -0500203{
Florin Coras31c99552019-03-01 13:00:58 -0800204 session_worker_t *wrk = &session_main.wrk[thread_index];
Florin Coras288eaab2019-02-03 15:26:14 -0800205 session_t *s;
Damjan Marion66d4cb52022-03-17 18:59:46 +0100206
Florin Corasd3915dc2022-03-31 15:42:17 -0700207 pool_get_aligned_safe (wrk->sessions, s, CLIB_CACHE_LINE_BYTES);
Dave Barachb7b92992018-10-17 10:38:51 -0400208 clib_memset (s, 0, sizeof (*s));
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700209 s->session_index = s - wrk->sessions;
Florin Coras3cbc04b2017-10-02 00:18:51 -0700210 s->thread_index = thread_index;
Nathan Skrzypczak2f0f96b2019-06-13 10:14:28 +0200211 s->app_index = APP_INVALID_INDEX;
Florin Coras6bd8d3f2022-03-14 21:17:25 -0700212
Florin Coras3cbc04b2017-10-02 00:18:51 -0700213 return s;
214}
215
Florin Coras371ca502018-02-21 12:07:41 -0800216void
Florin Coras288eaab2019-02-03 15:26:14 -0800217session_free (session_t * s)
Florin Coras3cbc04b2017-10-02 00:18:51 -0700218{
Steven Luong20de85b2022-10-17 10:39:06 -0700219 session_worker_t *wrk = &session_main.wrk[s->thread_index];
220
Florin Corascca96182019-07-19 07:34:13 -0700221 SESSION_EVT (SESSION_EVT_FREE, s);
Steven Luong20de85b2022-10-17 10:39:06 -0700222 if (CLIB_DEBUG)
223 clib_memset (s, 0xFA, sizeof (*s));
224 pool_put (wrk->sessions, s);
Florin Coras3cbc04b2017-10-02 00:18:51 -0700225}
226
Srikanth Akulae140d5d2019-11-18 11:49:58 -0800227u8
228session_is_valid (u32 si, u8 thread_index)
229{
230 session_t *s;
231 transport_connection_t *tc;
232
233 s = pool_elt_at_index (session_main.wrk[thread_index].sessions, si);
234
Srikanth Akulae140d5d2019-11-18 11:49:58 -0800235 if (s->thread_index != thread_index || s->session_index != si)
236 return 0;
237
238 if (s->session_state == SESSION_STATE_TRANSPORT_DELETED
239 || s->session_state <= SESSION_STATE_LISTENING)
240 return 1;
241
Florin Corasea727642021-05-07 19:39:43 -0700242 if (s->session_state == SESSION_STATE_CONNECTING &&
243 (s->flags & SESSION_F_HALF_OPEN))
244 return 1;
245
Srikanth Akulae140d5d2019-11-18 11:49:58 -0800246 tc = session_get_transport (s);
247 if (s->connection_index != tc->c_index
248 || s->thread_index != tc->thread_index || tc->s_index != si)
249 return 0;
250
251 return 1;
252}
253
Florin Coras70f26d52019-07-08 11:47:18 -0700254static void
255session_cleanup_notify (session_t * s, session_cleanup_ntf_t ntf)
256{
257 app_worker_t *app_wrk;
258
259 app_wrk = app_worker_get_if_valid (s->app_wrk_index);
260 if (!app_wrk)
261 return;
262 app_worker_cleanup_notify (app_wrk, s, ntf);
263}
264
Florin Coraseb97e5f2018-10-15 21:35:42 -0700265void
Florin Coras288eaab2019-02-03 15:26:14 -0800266session_free_w_fifos (session_t * s)
Florin Coras568ebc72018-09-18 16:12:50 -0700267{
Florin Coras70f26d52019-07-08 11:47:18 -0700268 session_cleanup_notify (s, SESSION_CLEANUP_SESSION);
Florin Coras19223e02019-03-03 14:56:05 -0800269 segment_manager_dealloc_fifos (s->rx_fifo, s->tx_fifo);
Florin Coras568ebc72018-09-18 16:12:50 -0700270 session_free (s);
271}
272
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800273/**
274 * Cleans up session and lookup table.
275 *
276 * Transport connection must still be valid.
277 */
278static void
Florin Coras288eaab2019-02-03 15:26:14 -0800279session_delete (session_t * s)
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800280{
281 int rv;
282
283 /* Delete from the main lookup table. */
284 if ((rv = session_lookup_del_session (s)))
Florin Corasd09236d2019-08-08 17:38:26 -0700285 clib_warning ("session %u hash delete rv %d", s->session_index, rv);
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800286
287 session_free_w_fifos (s);
288}
289
Florin Corasd50ff7f2020-04-16 04:30:22 +0000290void
Florin Corasea727642021-05-07 19:39:43 -0700291session_cleanup_half_open (session_handle_t ho_handle)
Florin Corasd50ff7f2020-04-16 04:30:22 +0000292{
Florin Corasbab6c522021-05-13 08:36:56 -0700293 session_t *ho = session_get_from_handle (ho_handle);
294
295 /* App transports can migrate their half-opens */
296 if (ho->flags & SESSION_F_IS_MIGRATING)
297 {
298 /* Session still migrating, move to closed state to signal that the
299 * session should be removed. */
300 if (ho->connection_index == ~0)
301 {
Steven Luongd810a6e2022-10-25 13:09:11 -0700302 session_set_state (ho, SESSION_STATE_CLOSED);
Florin Corasbab6c522021-05-13 08:36:56 -0700303 return;
304 }
305 /* Migrated transports are no longer half-opens */
306 transport_cleanup (session_get_transport_proto (ho),
307 ho->connection_index, ho->app_index /* overloaded */);
Florin Corasbab6c522021-05-13 08:36:56 -0700308 }
Florin Coras374df7a2021-05-13 11:37:43 -0700309 else
310 transport_cleanup_half_open (session_get_transport_proto (ho),
311 ho->connection_index);
312 session_free (ho);
Florin Corasbab6c522021-05-13 08:36:56 -0700313}
314
315static void
Florin Coras374df7a2021-05-13 11:37:43 -0700316session_half_open_free (session_t *ho)
Florin Corasbab6c522021-05-13 08:36:56 -0700317{
318 app_worker_t *app_wrk;
Florin Corasbab6c522021-05-13 08:36:56 -0700319
320 ASSERT (vlib_get_thread_index () <= 1);
Florin Corasbab6c522021-05-13 08:36:56 -0700321 app_wrk = app_worker_get (ho->app_wrk_index);
322 app_worker_del_half_open (app_wrk, ho);
323 session_free (ho);
324}
325
326static void
327session_half_open_free_rpc (void *args)
328{
Florin Coras374df7a2021-05-13 11:37:43 -0700329 session_t *ho = ho_session_get (pointer_to_uword (args));
330 session_half_open_free (ho);
Florin Corasd50ff7f2020-04-16 04:30:22 +0000331}
332
333void
Florin Corasea727642021-05-07 19:39:43 -0700334session_half_open_delete_notify (transport_connection_t *tc)
Florin Corasd50ff7f2020-04-16 04:30:22 +0000335{
Florin Coras374df7a2021-05-13 11:37:43 -0700336 /* Notification from ctrl thread accepted without rpc */
Florin Coras309f7aa2022-03-18 08:33:08 -0700337 if (tc->thread_index == transport_cl_thread ())
Florin Coras374df7a2021-05-13 11:37:43 -0700338 {
339 session_half_open_free (ho_session_get (tc->s_index));
340 }
341 else
342 {
343 void *args = uword_to_pointer ((uword) tc->s_index, void *);
Florin Coras309f7aa2022-03-18 08:33:08 -0700344 session_send_rpc_evt_to_thread_force (transport_cl_thread (),
345 session_half_open_free_rpc, args);
Florin Coras374df7a2021-05-13 11:37:43 -0700346 }
Florin Corasbab6c522021-05-13 08:36:56 -0700347}
Florin Corasea727642021-05-07 19:39:43 -0700348
Florin Corasbab6c522021-05-13 08:36:56 -0700349void
350session_half_open_migrate_notify (transport_connection_t *tc)
351{
352 session_t *ho;
353
354 ho = ho_session_get (tc->s_index);
355 ho->flags |= SESSION_F_IS_MIGRATING;
356 ho->connection_index = ~0;
357}
358
359int
360session_half_open_migrated_notify (transport_connection_t *tc)
361{
362 session_t *ho;
363
364 ho = ho_session_get (tc->s_index);
365
366 /* App probably detached so the half-open must be cleaned up */
367 if (ho->session_state == SESSION_STATE_CLOSED)
368 {
369 session_half_open_delete_notify (tc);
370 return -1;
371 }
372 ho->connection_index = tc->c_index;
373 /* Overload app index for half-open with new thread */
374 ho->app_index = tc->thread_index;
375 return 0;
Florin Corasd50ff7f2020-04-16 04:30:22 +0000376}
377
Andreas Schultz1a8c4372020-03-20 09:39:59 +0100378session_t *
Florin Coras3cbc04b2017-10-02 00:18:51 -0700379session_alloc_for_connection (transport_connection_t * tc)
380{
Florin Coras288eaab2019-02-03 15:26:14 -0800381 session_t *s;
Florin Coras3cbc04b2017-10-02 00:18:51 -0700382 u32 thread_index = tc->thread_index;
383
Florin Coras40903ac2018-06-10 14:41:23 -0700384 ASSERT (thread_index == vlib_get_thread_index ()
385 || transport_protocol_is_cl (tc->proto));
Dave Barach2c25a622017-06-26 11:35:07 -0400386
Florin Coras3cbc04b2017-10-02 00:18:51 -0700387 s = session_alloc (thread_index);
388 s->session_type = session_type_from_proto_and_ip (tc->proto, tc->is_ip4);
Steven Luongd810a6e2022-10-25 13:09:11 -0700389 session_set_state (s, SESSION_STATE_CLOSED);
Dave Barach68b0fb02017-02-28 15:15:56 -0500390
Florin Coras3cbc04b2017-10-02 00:18:51 -0700391 /* Attach transport to session and vice versa */
Dave Barach68b0fb02017-02-28 15:15:56 -0500392 s->connection_index = tc->c_index;
Dave Barach68b0fb02017-02-28 15:15:56 -0500393 tc->s_index = s->session_index;
Florin Coras3cbc04b2017-10-02 00:18:51 -0700394 return s;
395}
396
Florin Coras2c876f92021-05-10 21:12:27 -0700397session_t *
Florin Corasea727642021-05-07 19:39:43 -0700398session_alloc_for_half_open (transport_connection_t *tc)
399{
400 session_t *s;
401
402 s = ho_session_alloc ();
403 s->session_type = session_type_from_proto_and_ip (tc->proto, tc->is_ip4);
404 s->connection_index = tc->c_index;
405 tc->s_index = s->session_index;
406 return s;
407}
408
Florin Coras1f152cd2017-08-18 19:28:03 -0700409/**
410 * Discards bytes from buffer chain
411 *
412 * It discards n_bytes_to_drop starting at first buffer after chain_b
413 */
414always_inline void
415session_enqueue_discard_chain_bytes (vlib_main_t * vm, vlib_buffer_t * b,
416 vlib_buffer_t ** chain_b,
417 u32 n_bytes_to_drop)
418{
419 vlib_buffer_t *next = *chain_b;
420 u32 to_drop = n_bytes_to_drop;
421 ASSERT (b->flags & VLIB_BUFFER_NEXT_PRESENT);
422 while (to_drop && (next->flags & VLIB_BUFFER_NEXT_PRESENT))
423 {
424 next = vlib_get_buffer (vm, next->next_buffer);
425 if (next->current_length > to_drop)
426 {
427 vlib_buffer_advance (next, to_drop);
428 to_drop = 0;
429 }
430 else
431 {
432 to_drop -= next->current_length;
433 next->current_length = 0;
434 }
435 }
436 *chain_b = next;
437
438 if (to_drop == 0)
439 b->total_length_not_including_first_buffer -= n_bytes_to_drop;
440}
441
442/**
443 * Enqueue buffer chain tail
444 */
Florin Corasf6d68ed2017-05-07 19:12:02 -0700445always_inline int
Florin Coras288eaab2019-02-03 15:26:14 -0800446session_enqueue_chain_tail (session_t * s, vlib_buffer_t * b,
Florin Corasf6d68ed2017-05-07 19:12:02 -0700447 u32 offset, u8 is_in_order)
448{
449 vlib_buffer_t *chain_b;
Florin Coras1f152cd2017-08-18 19:28:03 -0700450 u32 chain_bi, len, diff;
Florin Corasf6d68ed2017-05-07 19:12:02 -0700451 vlib_main_t *vm = vlib_get_main ();
Florin Corasb2215d62017-08-01 16:56:58 -0700452 u8 *data;
Florin Coras1f152cd2017-08-18 19:28:03 -0700453 u32 written = 0;
Florin Corasf6d68ed2017-05-07 19:12:02 -0700454 int rv = 0;
455
Florin Coras1f152cd2017-08-18 19:28:03 -0700456 if (is_in_order && offset)
457 {
458 diff = offset - b->current_length;
459 if (diff > b->total_length_not_including_first_buffer)
460 return 0;
461 chain_b = b;
462 session_enqueue_discard_chain_bytes (vm, b, &chain_b, diff);
463 chain_bi = vlib_get_buffer_index (vm, chain_b);
464 }
465 else
466 chain_bi = b->next_buffer;
467
Florin Corasf6d68ed2017-05-07 19:12:02 -0700468 do
469 {
470 chain_b = vlib_get_buffer (vm, chain_bi);
471 data = vlib_buffer_get_current (chain_b);
472 len = chain_b->current_length;
Florin Coras1f152cd2017-08-18 19:28:03 -0700473 if (!len)
474 continue;
Florin Corasf6d68ed2017-05-07 19:12:02 -0700475 if (is_in_order)
476 {
Florin Coras87b15ce2019-04-28 21:16:30 -0700477 rv = svm_fifo_enqueue (s->rx_fifo, len, data);
Florin Coras1f152cd2017-08-18 19:28:03 -0700478 if (rv == len)
479 {
480 written += rv;
481 }
482 else if (rv < len)
Florin Corasf6d68ed2017-05-07 19:12:02 -0700483 {
484 return (rv > 0) ? (written + rv) : written;
485 }
Florin Coras1f152cd2017-08-18 19:28:03 -0700486 else if (rv > len)
487 {
488 written += rv;
489
490 /* written more than what was left in chain */
491 if (written > b->total_length_not_including_first_buffer)
492 return written;
493
494 /* drop the bytes that have already been delivered */
495 session_enqueue_discard_chain_bytes (vm, b, &chain_b, rv - len);
496 }
Florin Corasf6d68ed2017-05-07 19:12:02 -0700497 }
498 else
499 {
Florin Coras288eaab2019-02-03 15:26:14 -0800500 rv = svm_fifo_enqueue_with_offset (s->rx_fifo, offset, len, data);
Florin Corasf6d68ed2017-05-07 19:12:02 -0700501 if (rv)
Florin Coras1f152cd2017-08-18 19:28:03 -0700502 {
503 clib_warning ("failed to enqueue multi-buffer seg");
504 return -1;
505 }
Florin Corasf6d68ed2017-05-07 19:12:02 -0700506 offset += len;
507 }
508 }
509 while ((chain_bi = (chain_b->flags & VLIB_BUFFER_NEXT_PRESENT)
510 ? chain_b->next_buffer : 0));
511
512 if (is_in_order)
513 return written;
514
515 return 0;
516}
517
Ryujiro Shibuyad8f48e22020-01-22 12:11:42 +0000518void
519session_fifo_tuning (session_t * s, svm_fifo_t * f,
520 session_ft_action_t act, u32 len)
521{
522 if (s->flags & SESSION_F_CUSTOM_FIFO_TUNING)
523 {
524 app_worker_t *app_wrk = app_worker_get (s->app_wrk_index);
525 app_worker_session_fifo_tuning (app_wrk, s, f, act, len);
526 if (CLIB_ASSERT_ENABLE)
527 {
528 segment_manager_t *sm;
529 sm = segment_manager_get (f->segment_manager);
Florin Corasc547e912020-12-08 17:50:45 -0800530 ASSERT (f->shr->size >= 4096);
531 ASSERT (f->shr->size <= sm->max_fifo_size);
Ryujiro Shibuyad8f48e22020-01-22 12:11:42 +0000532 }
533 }
534}
535
Dave Barach68b0fb02017-02-28 15:15:56 -0500536/*
537 * Enqueue data for delivery to session peer. Does not notify peer of enqueue
538 * event but on request can queue notification events for later delivery by
539 * calling stream_server_flush_enqueue_events().
540 *
541 * @param tc Transport connection which is to be enqueued data
Florin Corasf6d68ed2017-05-07 19:12:02 -0700542 * @param b Buffer to be enqueued
543 * @param offset Offset at which to start enqueueing if out-of-order
Dave Barach68b0fb02017-02-28 15:15:56 -0500544 * @param queue_event Flag to indicate if peer is to be notified or if event
545 * is to be queued. The former is useful when more data is
546 * enqueued and only one event is to be generated.
Florin Corasf6d68ed2017-05-07 19:12:02 -0700547 * @param is_in_order Flag to indicate if data is in order
Dave Barach68b0fb02017-02-28 15:15:56 -0500548 * @return Number of bytes enqueued or a negative value if enqueueing failed.
549 */
550int
Florin Coras3cbc04b2017-10-02 00:18:51 -0700551session_enqueue_stream_connection (transport_connection_t * tc,
552 vlib_buffer_t * b, u32 offset,
553 u8 queue_event, u8 is_in_order)
Dave Barach68b0fb02017-02-28 15:15:56 -0500554{
Florin Coras288eaab2019-02-03 15:26:14 -0800555 session_t *s;
Florin Coras1f152cd2017-08-18 19:28:03 -0700556 int enqueued = 0, rv, in_order_off;
Dave Barach68b0fb02017-02-28 15:15:56 -0500557
Florin Corascea194d2017-10-02 00:18:51 -0700558 s = session_get (tc->s_index, tc->thread_index);
Dave Barach68b0fb02017-02-28 15:15:56 -0500559
Florin Corasf6d68ed2017-05-07 19:12:02 -0700560 if (is_in_order)
561 {
Florin Coras87b15ce2019-04-28 21:16:30 -0700562 enqueued = svm_fifo_enqueue (s->rx_fifo,
563 b->current_length,
564 vlib_buffer_get_current (b));
Florin Coras1f152cd2017-08-18 19:28:03 -0700565 if (PREDICT_FALSE ((b->flags & VLIB_BUFFER_NEXT_PRESENT)
566 && enqueued >= 0))
Florin Corasf6d68ed2017-05-07 19:12:02 -0700567 {
Florin Coras1f152cd2017-08-18 19:28:03 -0700568 in_order_off = enqueued > b->current_length ? enqueued : 0;
569 rv = session_enqueue_chain_tail (s, b, in_order_off, 1);
570 if (rv > 0)
571 enqueued += rv;
Florin Corasf6d68ed2017-05-07 19:12:02 -0700572 }
573 }
574 else
575 {
Florin Coras288eaab2019-02-03 15:26:14 -0800576 rv = svm_fifo_enqueue_with_offset (s->rx_fifo, offset,
Florin Corasf6d68ed2017-05-07 19:12:02 -0700577 b->current_length,
578 vlib_buffer_get_current (b));
579 if (PREDICT_FALSE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) && !rv))
Florin Coras1f152cd2017-08-18 19:28:03 -0700580 session_enqueue_chain_tail (s, b, offset + b->current_length, 0);
581 /* if something was enqueued, report even this as success for ooo
582 * segment handling */
583 return rv;
Florin Corasf6d68ed2017-05-07 19:12:02 -0700584 }
Dave Barach68b0fb02017-02-28 15:15:56 -0500585
586 if (queue_event)
587 {
588 /* Queue RX event on this fifo. Eventually these will need to be flushed
589 * by calling stream_server_flush_enqueue_events () */
Florin Coras31c99552019-03-01 13:00:58 -0800590 session_worker_t *wrk;
Dave Barach68b0fb02017-02-28 15:15:56 -0500591
Florin Coras31c99552019-03-01 13:00:58 -0800592 wrk = session_main_get_worker (s->thread_index);
Florin Corasd5c604d2019-03-18 09:06:35 -0700593 if (!(s->flags & SESSION_F_RX_EVT))
Dave Barach68b0fb02017-02-28 15:15:56 -0500594 {
Florin Corasd5c604d2019-03-18 09:06:35 -0700595 s->flags |= SESSION_F_RX_EVT;
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700596 vec_add1 (wrk->session_to_enqueue[tc->proto], s->session_index);
Dave Barach68b0fb02017-02-28 15:15:56 -0500597 }
Ryujiro Shibuyad8f48e22020-01-22 12:11:42 +0000598
599 session_fifo_tuning (s, s->rx_fifo, SESSION_FT_ACTION_ENQUEUED, 0);
Dave Barach68b0fb02017-02-28 15:15:56 -0500600 }
601
Chris Lukeb2bcad62017-09-18 08:51:22 -0400602 return enqueued;
Dave Barach68b0fb02017-02-28 15:15:56 -0500603}
604
Florin Coras3cbc04b2017-10-02 00:18:51 -0700605int
Florin Coras288eaab2019-02-03 15:26:14 -0800606session_enqueue_dgram_connection (session_t * s,
Florin Coras7fb0fe12018-04-09 09:24:52 -0700607 session_dgram_hdr_t * hdr,
608 vlib_buffer_t * b, u8 proto, u8 queue_event)
Florin Coras3cbc04b2017-10-02 00:18:51 -0700609{
Florin Corasc95cfa22020-11-24 08:41:17 -0800610 int rv;
Florin Coras3cbc04b2017-10-02 00:18:51 -0700611
Sirshak Das28aa5392019-02-05 01:33:33 -0600612 ASSERT (svm_fifo_max_enqueue_prod (s->rx_fifo)
Florin Coras7fb0fe12018-04-09 09:24:52 -0700613 >= b->current_length + sizeof (*hdr));
614
Florin Corasc95cfa22020-11-24 08:41:17 -0800615 if (PREDICT_TRUE (!(b->flags & VLIB_BUFFER_NEXT_PRESENT)))
Florin Coras3cbc04b2017-10-02 00:18:51 -0700616 {
Florin Corasc95cfa22020-11-24 08:41:17 -0800617 /* *INDENT-OFF* */
618 svm_fifo_seg_t segs[2] = {
619 { (u8 *) hdr, sizeof (*hdr) },
620 { vlib_buffer_get_current (b), b->current_length }
621 };
622 /* *INDENT-ON* */
623
624 rv = svm_fifo_enqueue_segments (s->rx_fifo, segs, 2,
625 0 /* allow_partial */ );
Florin Coras3cbc04b2017-10-02 00:18:51 -0700626 }
Florin Corasc95cfa22020-11-24 08:41:17 -0800627 else
628 {
629 vlib_main_t *vm = vlib_get_main ();
630 svm_fifo_seg_t *segs = 0, *seg;
631 vlib_buffer_t *it = b;
632 u32 n_segs = 1;
633
634 vec_add2 (segs, seg, 1);
635 seg->data = (u8 *) hdr;
636 seg->len = sizeof (*hdr);
637 while (it)
638 {
639 vec_add2 (segs, seg, 1);
640 seg->data = vlib_buffer_get_current (it);
641 seg->len = it->current_length;
642 n_segs++;
643 if (!(it->flags & VLIB_BUFFER_NEXT_PRESENT))
644 break;
645 it = vlib_get_buffer (vm, it->next_buffer);
646 }
647 rv = svm_fifo_enqueue_segments (s->rx_fifo, segs, n_segs,
648 0 /* allow partial */ );
649 vec_free (segs);
650 }
651
652 if (queue_event && rv > 0)
Florin Coras3cbc04b2017-10-02 00:18:51 -0700653 {
654 /* Queue RX event on this fifo. Eventually these will need to be flushed
655 * by calling stream_server_flush_enqueue_events () */
Florin Coras31c99552019-03-01 13:00:58 -0800656 session_worker_t *wrk;
Florin Coras3cbc04b2017-10-02 00:18:51 -0700657
Florin Coras31c99552019-03-01 13:00:58 -0800658 wrk = session_main_get_worker (s->thread_index);
Florin Corasd5c604d2019-03-18 09:06:35 -0700659 if (!(s->flags & SESSION_F_RX_EVT))
Florin Coras3cbc04b2017-10-02 00:18:51 -0700660 {
Florin Corasd5c604d2019-03-18 09:06:35 -0700661 s->flags |= SESSION_F_RX_EVT;
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700662 vec_add1 (wrk->session_to_enqueue[proto], s->session_index);
Florin Coras3cbc04b2017-10-02 00:18:51 -0700663 }
Ryujiro Shibuyad8f48e22020-01-22 12:11:42 +0000664
665 session_fifo_tuning (s, s->rx_fifo, SESSION_FT_ACTION_ENQUEUED, 0);
Florin Coras3cbc04b2017-10-02 00:18:51 -0700666 }
Florin Corasc95cfa22020-11-24 08:41:17 -0800667 return rv > 0 ? rv : 0;
Florin Coras3cbc04b2017-10-02 00:18:51 -0700668}
669
Florin Coras93992a92017-05-24 18:03:56 -0700670int
Florin Coras31c99552019-03-01 13:00:58 -0800671session_tx_fifo_peek_bytes (transport_connection_t * tc, u8 * buffer,
672 u32 offset, u32 max_bytes)
Dave Barach68b0fb02017-02-28 15:15:56 -0500673{
Florin Coras288eaab2019-02-03 15:26:14 -0800674 session_t *s = session_get (tc->s_index, tc->thread_index);
675 return svm_fifo_peek (s->tx_fifo, offset, max_bytes, buffer);
Dave Barach68b0fb02017-02-28 15:15:56 -0500676}
677
678u32
Florin Coras31c99552019-03-01 13:00:58 -0800679session_tx_fifo_dequeue_drop (transport_connection_t * tc, u32 max_bytes)
Dave Barach68b0fb02017-02-28 15:15:56 -0500680{
Florin Coras288eaab2019-02-03 15:26:14 -0800681 session_t *s = session_get (tc->s_index, tc->thread_index);
Vladimir Kropylevf867cf12019-06-17 21:38:00 +0300682 u32 rv;
683
684 rv = svm_fifo_dequeue_drop (s->tx_fifo, max_bytes);
Ryujiro Shibuyad8f48e22020-01-22 12:11:42 +0000685 session_fifo_tuning (s, s->tx_fifo, SESSION_FT_ACTION_DEQUEUED, rv);
Florin Coras0e573f52019-05-07 16:28:16 -0700686
Florin Coras2d379d82019-06-28 12:45:12 -0700687 if (svm_fifo_needs_deq_ntf (s->tx_fifo, max_bytes))
Florin Coras0e573f52019-05-07 16:28:16 -0700688 session_dequeue_notify (s);
689
Vladimir Kropylevf867cf12019-06-17 21:38:00 +0300690 return rv;
Dave Barach68b0fb02017-02-28 15:15:56 -0500691}
692
Florin Coras72b04282019-01-14 17:23:11 -0800693static inline int
Florin Coras288eaab2019-02-03 15:26:14 -0800694session_notify_subscribers (u32 app_index, session_t * s,
Florin Coras72b04282019-01-14 17:23:11 -0800695 svm_fifo_t * f, session_evt_type_t evt_type)
696{
697 app_worker_t *app_wrk;
698 application_t *app;
699 int i;
700
701 app = application_get (app_index);
702 if (!app)
703 return -1;
704
Florin Corasc547e912020-12-08 17:50:45 -0800705 for (i = 0; i < f->shr->n_subscribers; i++)
Florin Coras72b04282019-01-14 17:23:11 -0800706 {
Florin Corasc547e912020-12-08 17:50:45 -0800707 app_wrk = application_get_worker (app, f->shr->subscribers[i]);
Florin Coras72b04282019-01-14 17:23:11 -0800708 if (!app_wrk)
709 continue;
710 if (app_worker_lock_and_send_event (app_wrk, s, evt_type))
711 return -1;
712 }
713
714 return 0;
715}
716
Dave Barach68b0fb02017-02-28 15:15:56 -0500717/**
718 * Notify session peer that new data has been enqueued.
719 *
Florin Coras3c2fed52018-07-04 04:15:05 -0700720 * @param s Stream session for which the event is to be generated.
721 * @param lock Flag to indicate if call should lock message queue.
Dave Barach68b0fb02017-02-28 15:15:56 -0500722 *
Florin Coras3c2fed52018-07-04 04:15:05 -0700723 * @return 0 on success or negative number if failed to send notification.
Dave Barach68b0fb02017-02-28 15:15:56 -0500724 */
Florin Coras3c2fed52018-07-04 04:15:05 -0700725static inline int
Florin Coras653e43f2019-03-04 10:56:23 -0800726session_enqueue_notify_inline (session_t * s)
Dave Barach68b0fb02017-02-28 15:15:56 -0500727{
Florin Coras72b04282019-01-14 17:23:11 -0800728 app_worker_t *app_wrk;
Aloys Augustinb3b267c2019-04-09 11:25:56 +0200729 u32 session_index;
730 u8 n_subscribers;
731
732 session_index = s->session_index;
733 n_subscribers = svm_fifo_n_subscribers (s->rx_fifo);
Dave Barach68b0fb02017-02-28 15:15:56 -0500734
Florin Coras72b04282019-01-14 17:23:11 -0800735 app_wrk = app_worker_get_if_valid (s->app_wrk_index);
736 if (PREDICT_FALSE (!app_wrk))
Dave Barach818eb542017-08-02 13:56:13 -0400737 {
Florin Coras15531972018-08-12 23:50:53 -0700738 SESSION_DBG ("invalid s->app_index = %d", s->app_wrk_index);
Dave Barach818eb542017-08-02 13:56:13 -0400739 return 0;
740 }
Dave Barach68b0fb02017-02-28 15:15:56 -0500741
Florin Corascca96182019-07-19 07:34:13 -0700742 SESSION_EVT (SESSION_EVT_ENQ, s, svm_fifo_max_dequeue_prod (s->rx_fifo));
Dave Barach68b0fb02017-02-28 15:15:56 -0500743
Florin Corasd5c604d2019-03-18 09:06:35 -0700744 s->flags &= ~SESSION_F_RX_EVT;
Florin Corasda302e42020-04-19 22:41:55 +0000745
746 /* Application didn't confirm accept yet */
747 if (PREDICT_FALSE (s->session_state == SESSION_STATE_ACCEPTING))
748 return 0;
749
Florin Coras72b04282019-01-14 17:23:11 -0800750 if (PREDICT_FALSE (app_worker_lock_and_send_event (app_wrk, s,
Florin Corasf6c43132019-03-01 12:41:21 -0800751 SESSION_IO_EVT_RX)))
Florin Coras72b04282019-01-14 17:23:11 -0800752 return -1;
753
Aloys Augustinb3b267c2019-04-09 11:25:56 +0200754 if (PREDICT_FALSE (n_subscribers))
755 {
756 s = session_get (session_index, vlib_get_thread_index ());
757 return session_notify_subscribers (app_wrk->app_index, s,
758 s->rx_fifo, SESSION_IO_EVT_RX);
759 }
Florin Coras72b04282019-01-14 17:23:11 -0800760
761 return 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500762}
763
Florin Coras0d60a0f2018-06-29 02:02:08 -0700764int
Florin Coras653e43f2019-03-04 10:56:23 -0800765session_enqueue_notify (session_t * s)
766{
767 return session_enqueue_notify_inline (s);
768}
769
Aloys Augustinb3392332019-08-06 16:09:01 +0200770static void
771session_enqueue_notify_rpc (void *arg)
772{
Florin Coras5d8a8062019-08-13 08:35:39 -0700773 u32 session_index = pointer_to_uword (arg);
Aloys Augustinb3392332019-08-06 16:09:01 +0200774 session_t *s;
775
Florin Coras5d8a8062019-08-13 08:35:39 -0700776 s = session_get_if_valid (session_index, vlib_get_thread_index ());
Aloys Augustinb3392332019-08-06 16:09:01 +0200777 if (!s)
778 return;
779
780 session_enqueue_notify (s);
781}
782
783/**
784 * Like session_enqueue_notify, but can be called from a thread that does not
785 * own the session.
786 */
787void
788session_enqueue_notify_thread (session_handle_t sh)
789{
790 u32 thread_index = session_thread_from_handle (sh);
Florin Coras5d8a8062019-08-13 08:35:39 -0700791 u32 session_index = session_index_from_handle (sh);
792
793 /*
794 * Pass session index (u32) as opposed to handle (u64) in case pointers
795 * are not 64-bit.
796 */
Aloys Augustinb3392332019-08-06 16:09:01 +0200797 session_send_rpc_evt_to_thread (thread_index,
Florin Coras5d8a8062019-08-13 08:35:39 -0700798 session_enqueue_notify_rpc,
799 uword_to_pointer (session_index, void *));
Aloys Augustinb3392332019-08-06 16:09:01 +0200800}
801
Florin Coras653e43f2019-03-04 10:56:23 -0800802int
Florin Coras288eaab2019-02-03 15:26:14 -0800803session_dequeue_notify (session_t * s)
Florin Coras0d60a0f2018-06-29 02:02:08 -0700804{
Florin Coras72b04282019-01-14 17:23:11 -0800805 app_worker_t *app_wrk;
Florin Coras0d60a0f2018-06-29 02:02:08 -0700806
Vladimir Kropylev5c89fbf2019-08-30 13:04:06 +0300807 svm_fifo_clear_deq_ntf (s->tx_fifo);
808
Florin Coras72b04282019-01-14 17:23:11 -0800809 app_wrk = app_worker_get_if_valid (s->app_wrk_index);
810 if (PREDICT_FALSE (!app_wrk))
Florin Coras208daa12018-07-02 01:30:51 -0700811 return -1;
812
Florin Coras72b04282019-01-14 17:23:11 -0800813 if (PREDICT_FALSE (app_worker_lock_and_send_event (app_wrk, s,
Florin Corasf6c43132019-03-01 12:41:21 -0800814 SESSION_IO_EVT_TX)))
Florin Coras1bcad5c2019-01-09 20:04:38 -0800815 return -1;
816
Florin Corasc547e912020-12-08 17:50:45 -0800817 if (PREDICT_FALSE (s->tx_fifo->shr->n_subscribers))
Florin Coras72b04282019-01-14 17:23:11 -0800818 return session_notify_subscribers (app_wrk->app_index, s,
Florin Corasf6c43132019-03-01 12:41:21 -0800819 s->tx_fifo, SESSION_IO_EVT_TX);
Florin Coras72b04282019-01-14 17:23:11 -0800820
Florin Coras1bcad5c2019-01-09 20:04:38 -0800821 return 0;
Florin Coras0d60a0f2018-06-29 02:02:08 -0700822}
823
Dave Barach68b0fb02017-02-28 15:15:56 -0500824/**
825 * Flushes queue of sessions that are to be notified of new data
826 * enqueued events.
827 *
828 * @param thread_index Thread index for which the flush is to be performed.
829 * @return 0 on success or a positive number indicating the number of
830 * failures due to API queue being full.
831 */
832int
Florin Coras31c99552019-03-01 13:00:58 -0800833session_main_flush_enqueue_events (u8 transport_proto, u32 thread_index)
Dave Barach68b0fb02017-02-28 15:15:56 -0500834{
Florin Coras31c99552019-03-01 13:00:58 -0800835 session_worker_t *wrk = session_main_get_worker (thread_index);
Florin Coras288eaab2019-02-03 15:26:14 -0800836 session_t *s;
Florin Coras21795132018-09-09 09:40:51 -0700837 int i, errors = 0;
Florin Coras3c2fed52018-07-04 04:15:05 -0700838 u32 *indices;
Dave Barach68b0fb02017-02-28 15:15:56 -0500839
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700840 indices = wrk->session_to_enqueue[transport_proto];
Dave Barach68b0fb02017-02-28 15:15:56 -0500841
Florin Coras3cbc04b2017-10-02 00:18:51 -0700842 for (i = 0; i < vec_len (indices); i++)
Dave Barach68b0fb02017-02-28 15:15:56 -0500843 {
Florin Coras3cbc04b2017-10-02 00:18:51 -0700844 s = session_get_if_valid (indices[i], thread_index);
Florin Coras3c2fed52018-07-04 04:15:05 -0700845 if (PREDICT_FALSE (!s))
846 {
847 errors++;
848 continue;
849 }
Florin Coras653e43f2019-03-04 10:56:23 -0800850
Ryujiro Shibuyad8f48e22020-01-22 12:11:42 +0000851 session_fifo_tuning (s, s->rx_fifo, SESSION_FT_ACTION_ENQUEUED,
852 0 /* TODO/not needed */ );
853
Florin Coras653e43f2019-03-04 10:56:23 -0800854 if (PREDICT_FALSE (session_enqueue_notify_inline (s)))
Florin Coras3cbc04b2017-10-02 00:18:51 -0700855 errors++;
Dave Barach68b0fb02017-02-28 15:15:56 -0500856 }
857
Florin Coras3cbc04b2017-10-02 00:18:51 -0700858 vec_reset_length (indices);
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700859 wrk->session_to_enqueue[transport_proto] = indices;
Dave Barach68b0fb02017-02-28 15:15:56 -0500860
861 return errors;
862}
863
Florin Coras7fb0fe12018-04-09 09:24:52 -0700864int
Florin Coras31c99552019-03-01 13:00:58 -0800865session_main_flush_all_enqueue_events (u8 transport_proto)
Florin Coras7fb0fe12018-04-09 09:24:52 -0700866{
867 vlib_thread_main_t *vtm = vlib_get_thread_main ();
868 int i, errors = 0;
869 for (i = 0; i < 1 + vtm->n_threads; i++)
Florin Coras31c99552019-03-01 13:00:58 -0800870 errors += session_main_flush_enqueue_events (transport_proto, i);
Florin Coras7fb0fe12018-04-09 09:24:52 -0700871 return errors;
872}
873
Florin Coras4fde4ae2020-04-13 16:48:04 +0000874int
875session_stream_connect_notify (transport_connection_t * tc,
876 session_error_t err)
Dave Barach68b0fb02017-02-28 15:15:56 -0500877{
Florin Coras371ca502018-02-21 12:07:41 -0800878 u32 opaque = 0, new_ti, new_si;
Florin Coras15531972018-08-12 23:50:53 -0700879 app_worker_t *app_wrk;
Florin Corasea727642021-05-07 19:39:43 -0700880 session_t *s = 0, *ho;
Dave Barach68b0fb02017-02-28 15:15:56 -0500881
Florin Coras3cbc04b2017-10-02 00:18:51 -0700882 /*
Florin Corasea727642021-05-07 19:39:43 -0700883 * Cleanup half-open table
Florin Coras3cbc04b2017-10-02 00:18:51 -0700884 */
Florin Corascea194d2017-10-02 00:18:51 -0700885 session_lookup_del_half_open (tc);
Florin Coras4eeeaaf2017-09-05 14:03:37 -0400886
Florin Corasea727642021-05-07 19:39:43 -0700887 ho = ho_session_get (tc->s_index);
888 opaque = ho->opaque;
889 app_wrk = app_worker_get_if_valid (ho->app_wrk_index);
Florin Coras15531972018-08-12 23:50:53 -0700890 if (!app_wrk)
Florin Corasc87c91d2017-08-16 19:55:49 -0700891 return -1;
Florin Corasa27a46e2019-02-18 13:02:28 -0800892
Florin Coras00e01d32019-10-21 16:07:46 -0700893 if (err)
894 return app_worker_connect_notify (app_wrk, s, err, opaque);
Florin Corasa27a46e2019-02-18 13:02:28 -0800895
896 s = session_alloc_for_connection (tc);
Steven Luongd810a6e2022-10-25 13:09:11 -0700897 session_set_state (s, SESSION_STATE_CONNECTING);
Florin Corasa27a46e2019-02-18 13:02:28 -0800898 s->app_wrk_index = app_wrk->wrk_index;
899 new_si = s->session_index;
900 new_ti = s->thread_index;
901
Florin Coras00e01d32019-10-21 16:07:46 -0700902 if ((err = app_worker_init_connected (app_wrk, s)))
Dave Barach68b0fb02017-02-28 15:15:56 -0500903 {
Florin Corasa27a46e2019-02-18 13:02:28 -0800904 session_free (s);
Florin Coras00e01d32019-10-21 16:07:46 -0700905 app_worker_connect_notify (app_wrk, 0, err, opaque);
Florin Corasa27a46e2019-02-18 13:02:28 -0800906 return -1;
Dave Barach68b0fb02017-02-28 15:15:56 -0500907 }
908
Nathan Skrzypczaka26349d2019-06-26 13:53:08 +0200909 s = session_get (new_si, new_ti);
Steven Luongd810a6e2022-10-25 13:09:11 -0700910 session_set_state (s, SESSION_STATE_READY);
Nathan Skrzypczaka26349d2019-06-26 13:53:08 +0200911 session_lookup_add_connection (tc, session_handle (s));
912
Florin Coras00e01d32019-10-21 16:07:46 -0700913 if (app_worker_connect_notify (app_wrk, s, SESSION_E_NONE, opaque))
Florin Coras6534b7a2017-07-18 05:38:03 -0400914 {
Florin Corasc7fd24e2020-07-24 10:09:07 -0700915 session_lookup_del_connection (tc);
916 /* Avoid notifying app about rejected session cleanup */
Florin Corasa27a46e2019-02-18 13:02:28 -0800917 s = session_get (new_si, new_ti);
Florin Corasc7fd24e2020-07-24 10:09:07 -0700918 segment_manager_dealloc_fifos (s->rx_fifo, s->tx_fifo);
919 session_free (s);
Florin Corasa27a46e2019-02-18 13:02:28 -0800920 return -1;
Florin Coras6534b7a2017-07-18 05:38:03 -0400921 }
Dave Barach68b0fb02017-02-28 15:15:56 -0500922
Florin Corasa27a46e2019-02-18 13:02:28 -0800923 return 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500924}
925
Florin Coras02000442021-11-16 12:45:43 -0800926typedef union session_switch_pool_reply_args_
927{
928 struct
929 {
930 u32 session_index;
931 u16 thread_index;
932 u8 is_closed;
933 };
934 u64 as_u64;
935} session_switch_pool_reply_args_t;
936
937STATIC_ASSERT (sizeof (session_switch_pool_reply_args_t) <= sizeof (uword),
938 "switch pool reply args size");
939
Florin Coras6d7552c2020-04-09 01:49:45 +0000940static void
941session_switch_pool_reply (void *arg)
942{
Florin Coras02000442021-11-16 12:45:43 -0800943 session_switch_pool_reply_args_t rargs;
Florin Coras6d7552c2020-04-09 01:49:45 +0000944 session_t *s;
945
Florin Coras02000442021-11-16 12:45:43 -0800946 rargs.as_u64 = pointer_to_uword (arg);
947 s = session_get_if_valid (rargs.session_index, rargs.thread_index);
Florin Coras6d7552c2020-04-09 01:49:45 +0000948 if (!s)
949 return;
950
Florin Coras02000442021-11-16 12:45:43 -0800951 /* Session closed during migration. Clean everything up */
952 if (rargs.is_closed)
953 {
954 transport_cleanup (session_get_transport_proto (s), s->connection_index,
955 s->thread_index);
956 segment_manager_dealloc_fifos (s->rx_fifo, s->tx_fifo);
957 session_free (s);
958 return;
959 }
960
Florin Coras6d7552c2020-04-09 01:49:45 +0000961 /* Notify app that it has data on the new session */
962 session_enqueue_notify (s);
963}
964
Florin Coras3cbc04b2017-10-02 00:18:51 -0700965typedef struct _session_switch_pool_args
966{
967 u32 session_index;
968 u32 thread_index;
969 u32 new_thread_index;
970 u32 new_session_index;
971} session_switch_pool_args_t;
972
Florin Coras49568af2019-07-31 16:46:24 -0700973/**
974 * Notify old thread of the session pool switch
975 */
Florin Coras3cbc04b2017-10-02 00:18:51 -0700976static void
977session_switch_pool (void *cb_args)
978{
979 session_switch_pool_args_t *args = (session_switch_pool_args_t *) cb_args;
Florin Coras02000442021-11-16 12:45:43 -0800980 session_switch_pool_reply_args_t rargs;
Florin Coras6d7552c2020-04-09 01:49:45 +0000981 session_handle_t new_sh;
982 segment_manager_t *sm;
Florin Coras49568af2019-07-31 16:46:24 -0700983 app_worker_t *app_wrk;
Florin Coras288eaab2019-02-03 15:26:14 -0800984 session_t *s;
Florin Coras49568af2019-07-31 16:46:24 -0700985
Florin Coras3cbc04b2017-10-02 00:18:51 -0700986 ASSERT (args->thread_index == vlib_get_thread_index ());
987 s = session_get (args->session_index, args->thread_index);
Florin Coras6d7552c2020-04-09 01:49:45 +0000988
Florin Corasfc8d0c52021-11-17 11:41:10 -0800989 /* Check if session closed during migration */
990 rargs.is_closed = s->session_state >= SESSION_STATE_TRANSPORT_CLOSING;
991
Florin Coras1ee78302019-02-05 15:51:15 -0800992 transport_cleanup (session_get_transport_proto (s), s->connection_index,
993 s->thread_index);
Florin Coras49568af2019-07-31 16:46:24 -0700994
995 app_wrk = app_worker_get_if_valid (s->app_wrk_index);
996 if (app_wrk)
997 {
Florin Coras6d7552c2020-04-09 01:49:45 +0000998 /* Cleanup fifo segment slice state for fifos */
999 sm = app_worker_get_connect_segment_manager (app_wrk);
Florin Coras0bc78d82021-01-09 14:34:01 -08001000 segment_manager_detach_fifo (sm, &s->rx_fifo);
1001 segment_manager_detach_fifo (sm, &s->tx_fifo);
Aloys Augustinb3392332019-08-06 16:09:01 +02001002
Florin Coras6d7552c2020-04-09 01:49:45 +00001003 /* Notify app, using old session, about the migration event */
Florin Coras02000442021-11-16 12:45:43 -08001004 if (!rargs.is_closed)
1005 {
1006 new_sh = session_make_handle (args->new_session_index,
1007 args->new_thread_index);
1008 app_worker_migrate_notify (app_wrk, s, new_sh);
1009 }
Florin Coras49568af2019-07-31 16:46:24 -07001010 }
1011
Florin Coras6d7552c2020-04-09 01:49:45 +00001012 /* Trigger app read and fifo updates on the new thread */
Florin Coras02000442021-11-16 12:45:43 -08001013 rargs.session_index = args->new_session_index;
1014 rargs.thread_index = args->new_thread_index;
Florin Coras6d7552c2020-04-09 01:49:45 +00001015 session_send_rpc_evt_to_thread (args->new_thread_index,
Florin Coras02000442021-11-16 12:45:43 -08001016 session_switch_pool_reply,
1017 uword_to_pointer (rargs.as_u64, void *));
Florin Coras6d7552c2020-04-09 01:49:45 +00001018
Florin Coras3cbc04b2017-10-02 00:18:51 -07001019 session_free (s);
1020 clib_mem_free (cb_args);
1021}
1022
1023/**
1024 * Move dgram session to the right thread
1025 */
1026int
1027session_dgram_connect_notify (transport_connection_t * tc,
Florin Coras288eaab2019-02-03 15:26:14 -08001028 u32 old_thread_index, session_t ** new_session)
Florin Coras3cbc04b2017-10-02 00:18:51 -07001029{
Florin Coras288eaab2019-02-03 15:26:14 -08001030 session_t *new_s;
Florin Coras3cbc04b2017-10-02 00:18:51 -07001031 session_switch_pool_args_t *rpc_args;
Florin Coras0bc78d82021-01-09 14:34:01 -08001032 segment_manager_t *sm;
1033 app_worker_t *app_wrk;
Florin Coras3cbc04b2017-10-02 00:18:51 -07001034
1035 /*
1036 * Clone half-open session to the right thread.
1037 */
1038 new_s = session_clone_safe (tc->s_index, old_thread_index);
1039 new_s->connection_index = tc->c_index;
Steven Luongd810a6e2022-10-25 13:09:11 -07001040 session_set_state (new_s, SESSION_STATE_READY);
Nathan Skrzypczakcaa7acf2019-10-02 10:02:05 +02001041 new_s->flags |= SESSION_F_IS_MIGRATING;
Florin Coras6d7552c2020-04-09 01:49:45 +00001042
Florin Coras0bc78d82021-01-09 14:34:01 -08001043 if (!(tc->flags & TRANSPORT_CONNECTION_F_NO_LOOKUP))
1044 session_lookup_add_connection (tc, session_handle (new_s));
1045
1046 app_wrk = app_worker_get_if_valid (new_s->app_wrk_index);
1047 if (app_wrk)
1048 {
1049 /* New set of fifos attached to the same shared memory */
1050 sm = app_worker_get_connect_segment_manager (app_wrk);
1051 segment_manager_attach_fifo (sm, &new_s->rx_fifo, new_s);
1052 segment_manager_attach_fifo (sm, &new_s->tx_fifo, new_s);
1053 }
Florin Coras3cbc04b2017-10-02 00:18:51 -07001054
1055 /*
1056 * Ask thread owning the old session to clean it up and make us the tx
1057 * fifo owner
1058 */
1059 rpc_args = clib_mem_alloc (sizeof (*rpc_args));
1060 rpc_args->new_session_index = new_s->session_index;
1061 rpc_args->new_thread_index = new_s->thread_index;
1062 rpc_args->session_index = tc->s_index;
1063 rpc_args->thread_index = old_thread_index;
1064 session_send_rpc_evt_to_thread (rpc_args->thread_index, session_switch_pool,
1065 rpc_args);
1066
1067 tc->s_index = new_s->session_index;
1068 new_s->connection_index = tc->c_index;
1069 *new_session = new_s;
1070 return 0;
1071}
1072
Dave Barach68b0fb02017-02-28 15:15:56 -05001073/**
1074 * Notification from transport that connection is being closed.
1075 *
1076 * A disconnect is sent to application but state is not removed. Once
1077 * disconnect is acknowledged by application, session disconnect is called.
1078 * Ultimately this leads to close being called on transport (passive close).
1079 */
1080void
Florin Coras5a2ec8f2018-12-27 11:53:11 -08001081session_transport_closing_notify (transport_connection_t * tc)
Dave Barach68b0fb02017-02-28 15:15:56 -05001082{
Florin Coras15531972018-08-12 23:50:53 -07001083 app_worker_t *app_wrk;
Florin Coras288eaab2019-02-03 15:26:14 -08001084 session_t *s;
Dave Barach68b0fb02017-02-28 15:15:56 -05001085
Florin Corascea194d2017-10-02 00:18:51 -07001086 s = session_get (tc->s_index, tc->thread_index);
Florin Coras400ded32018-10-03 01:00:57 -07001087 if (s->session_state >= SESSION_STATE_TRANSPORT_CLOSING)
1088 return;
Florin Corasdfd27bc2021-11-27 10:45:55 -08001089
1090 /* Wait for reply from app before sending notification as the
1091 * accept might be rejected */
1092 if (s->session_state == SESSION_STATE_ACCEPTING)
1093 {
Steven Luongd810a6e2022-10-25 13:09:11 -07001094 session_set_state (s, SESSION_STATE_TRANSPORT_CLOSING);
Florin Corasdfd27bc2021-11-27 10:45:55 -08001095 return;
1096 }
1097
Steven Luongd810a6e2022-10-25 13:09:11 -07001098 session_set_state (s, SESSION_STATE_TRANSPORT_CLOSING);
Florin Corasbf7ce2c2019-03-06 14:44:42 -08001099 app_wrk = app_worker_get (s->app_wrk_index);
1100 app_worker_close_notify (app_wrk, s);
Dave Barach68b0fb02017-02-28 15:15:56 -05001101}
1102
1103/**
Dave Barach68b0fb02017-02-28 15:15:56 -05001104 * Notification from transport that connection is being deleted
1105 *
Florin Coras4eeeaaf2017-09-05 14:03:37 -04001106 * This removes the session if it is still valid. It should be called only on
1107 * previously fully established sessions. For instance failed connects should
1108 * call stream_session_connect_notify and indicate that the connect has
1109 * failed.
Dave Barach68b0fb02017-02-28 15:15:56 -05001110 */
1111void
Florin Coras5a2ec8f2018-12-27 11:53:11 -08001112session_transport_delete_notify (transport_connection_t * tc)
Dave Barach68b0fb02017-02-28 15:15:56 -05001113{
Florin Coras288eaab2019-02-03 15:26:14 -08001114 session_t *s;
Dave Barach68b0fb02017-02-28 15:15:56 -05001115
Florin Corase04c2992017-03-01 08:17:34 -08001116 /* App might've been removed already */
Florin Coras568ebc72018-09-18 16:12:50 -07001117 if (!(s = session_get_if_valid (tc->s_index, tc->thread_index)))
Florin Corasc87c91d2017-08-16 19:55:49 -07001118 return;
Florin Coras568ebc72018-09-18 16:12:50 -07001119
Florin Coras568ebc72018-09-18 16:12:50 -07001120 switch (s->session_state)
1121 {
Florin Coras222e1f412019-02-16 20:47:32 -08001122 case SESSION_STATE_CREATED:
1123 /* Session was created but accept notification was not yet sent to the
1124 * app. Cleanup everything. */
1125 session_lookup_del_session (s);
Zeyu Zhangcbbc4a22019-10-12 14:21:59 +08001126 segment_manager_dealloc_fifos (s->rx_fifo, s->tx_fifo);
1127 session_free (s);
Florin Coras222e1f412019-02-16 20:47:32 -08001128 break;
Florin Corasb0f662f2018-12-27 14:51:46 -08001129 case SESSION_STATE_ACCEPTING:
Florin Coras568ebc72018-09-18 16:12:50 -07001130 case SESSION_STATE_TRANSPORT_CLOSING:
Florin Coras692b9492019-07-12 15:01:53 -07001131 case SESSION_STATE_CLOSING:
Florin Coras5f066322019-07-22 19:03:03 -07001132 case SESSION_STATE_TRANSPORT_CLOSED:
Florin Coras568ebc72018-09-18 16:12:50 -07001133 /* If transport finishes or times out before we get a reply
Florin Coras5a2ec8f2018-12-27 11:53:11 -08001134 * from the app, mark transport as closed and wait for reply
1135 * before removing the session. Cleanup session table in advance
Florin Corasa9d5bea2018-12-17 08:24:19 -08001136 * because transport will soon be closed and closed sessions
1137 * are assumed to have been removed from the lookup table */
1138 session_lookup_del_session (s);
Steven Luongd810a6e2022-10-25 13:09:11 -07001139 session_set_state (s, SESSION_STATE_TRANSPORT_DELETED);
Florin Coras70f26d52019-07-08 11:47:18 -07001140 session_cleanup_notify (s, SESSION_CLEANUP_TRANSPORT);
1141 svm_fifo_dequeue_drop_all (s->tx_fifo);
Florin Coras568ebc72018-09-18 16:12:50 -07001142 break;
Florin Coras692b9492019-07-12 15:01:53 -07001143 case SESSION_STATE_APP_CLOSED:
Florin Coras5a2ec8f2018-12-27 11:53:11 -08001144 /* Cleanup lookup table as transport needs to still be valid.
1145 * Program transport close to ensure that all session events
1146 * have been cleaned up. Once transport close is called, the
1147 * session is just removed because both transport and app have
1148 * confirmed the close*/
Florin Coras568ebc72018-09-18 16:12:50 -07001149 session_lookup_del_session (s);
Steven Luongd810a6e2022-10-25 13:09:11 -07001150 session_set_state (s, SESSION_STATE_TRANSPORT_DELETED);
Florin Coras70f26d52019-07-08 11:47:18 -07001151 session_cleanup_notify (s, SESSION_CLEANUP_TRANSPORT);
1152 svm_fifo_dequeue_drop_all (s->tx_fifo);
Florin Corasdfb3b872019-08-16 17:48:44 -07001153 session_program_transport_ctrl_evt (s, SESSION_CTRL_EVT_CLOSE);
Florin Coras568ebc72018-09-18 16:12:50 -07001154 break;
Florin Coras5f066322019-07-22 19:03:03 -07001155 case SESSION_STATE_TRANSPORT_DELETED:
Florin Corasb0f662f2018-12-27 14:51:46 -08001156 break;
Florin Coras568ebc72018-09-18 16:12:50 -07001157 case SESSION_STATE_CLOSED:
Florin Coras70f26d52019-07-08 11:47:18 -07001158 session_cleanup_notify (s, SESSION_CLEANUP_TRANSPORT);
Florin Coras5a2ec8f2018-12-27 11:53:11 -08001159 session_delete (s);
Florin Coras568ebc72018-09-18 16:12:50 -07001160 break;
Florin Corasc01d5782018-10-17 14:53:11 -07001161 default:
Florin Corasb0f662f2018-12-27 14:51:46 -08001162 clib_warning ("session state %u", s->session_state);
Florin Coras70f26d52019-07-08 11:47:18 -07001163 session_cleanup_notify (s, SESSION_CLEANUP_TRANSPORT);
Florin Coras5a2ec8f2018-12-27 11:53:11 -08001164 session_delete (s);
Florin Corasc01d5782018-10-17 14:53:11 -07001165 break;
Florin Coras568ebc72018-09-18 16:12:50 -07001166 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001167}
1168
1169/**
Florin Coras692b9492019-07-12 15:01:53 -07001170 * Notification from transport that it is closed
Florin Coras54ddf432018-12-21 13:54:09 -08001171 *
Florin Coras692b9492019-07-12 15:01:53 -07001172 * Should be called by transport, prior to calling delete notify, once it
1173 * knows that no more data will be exchanged. This could serve as an
1174 * early acknowledgment of an active close especially if transport delete
1175 * can be delayed a long time, e.g., tcp time-wait.
Florin Coras54ddf432018-12-21 13:54:09 -08001176 */
1177void
Florin Coras5a2ec8f2018-12-27 11:53:11 -08001178session_transport_closed_notify (transport_connection_t * tc)
Florin Coras54ddf432018-12-21 13:54:09 -08001179{
Florin Coras692b9492019-07-12 15:01:53 -07001180 app_worker_t *app_wrk;
Florin Coras288eaab2019-02-03 15:26:14 -08001181 session_t *s;
Florin Coras54ddf432018-12-21 13:54:09 -08001182
1183 if (!(s = session_get_if_valid (tc->s_index, tc->thread_index)))
1184 return;
Florin Corasb0f662f2018-12-27 14:51:46 -08001185
Florin Coras06a2eec2019-05-23 22:28:16 -07001186 /* Transport thinks that app requested close but it actually didn't.
liuyacan534468e2021-05-09 03:50:40 +00001187 * Can happen for tcp:
1188 * 1)if fin and rst are received in close succession.
1189 * 2)if app shutdown the connection. */
Florin Coras06a2eec2019-05-23 22:28:16 -07001190 if (s->session_state == SESSION_STATE_READY)
1191 {
1192 session_transport_closing_notify (tc);
1193 svm_fifo_dequeue_drop_all (s->tx_fifo);
Steven Luongd810a6e2022-10-25 13:09:11 -07001194 session_set_state (s, SESSION_STATE_TRANSPORT_CLOSED);
Florin Coras06a2eec2019-05-23 22:28:16 -07001195 }
Florin Corasb0f662f2018-12-27 14:51:46 -08001196 /* If app close has not been received or has not yet resulted in
1197 * a transport close, only mark the session transport as closed */
Florin Coras06a2eec2019-05-23 22:28:16 -07001198 else if (s->session_state <= SESSION_STATE_CLOSING)
Steven Luongd810a6e2022-10-25 13:09:11 -07001199 session_set_state (s, SESSION_STATE_TRANSPORT_CLOSED);
Florin Coras5f066322019-07-22 19:03:03 -07001200 /* If app also closed, switch to closed */
1201 else if (s->session_state == SESSION_STATE_APP_CLOSED)
Steven Luongd810a6e2022-10-25 13:09:11 -07001202 session_set_state (s, SESSION_STATE_CLOSED);
Florin Coras692b9492019-07-12 15:01:53 -07001203
1204 app_wrk = app_worker_get_if_valid (s->app_wrk_index);
1205 if (app_wrk)
1206 app_worker_transport_closed_notify (app_wrk, s);
Florin Coras54ddf432018-12-21 13:54:09 -08001207}
1208
1209/**
Dave Barach68b0fb02017-02-28 15:15:56 -05001210 * Notify application that connection has been reset.
1211 */
1212void
Florin Coras5a2ec8f2018-12-27 11:53:11 -08001213session_transport_reset_notify (transport_connection_t * tc)
Dave Barach68b0fb02017-02-28 15:15:56 -05001214{
Florin Coras15531972018-08-12 23:50:53 -07001215 app_worker_t *app_wrk;
Florin Coras69b68ef2019-04-02 11:38:51 -07001216 session_t *s;
1217
Florin Corascea194d2017-10-02 00:18:51 -07001218 s = session_get (tc->s_index, tc->thread_index);
Florin Coras288eaab2019-02-03 15:26:14 -08001219 svm_fifo_dequeue_drop_all (s->tx_fifo);
Florin Coras3c7d4f92018-12-14 11:28:43 -08001220 if (s->session_state >= SESSION_STATE_TRANSPORT_CLOSING)
1221 return;
Florin Corasdfd27bc2021-11-27 10:45:55 -08001222 if (s->session_state == SESSION_STATE_ACCEPTING)
1223 {
Steven Luongd810a6e2022-10-25 13:09:11 -07001224 session_set_state (s, SESSION_STATE_TRANSPORT_CLOSING);
Florin Corasdfd27bc2021-11-27 10:45:55 -08001225 return;
1226 }
Steven Luongd810a6e2022-10-25 13:09:11 -07001227 session_set_state (s, SESSION_STATE_TRANSPORT_CLOSING);
Florin Coras15531972018-08-12 23:50:53 -07001228 app_wrk = app_worker_get (s->app_wrk_index);
Florin Coras69b68ef2019-04-02 11:38:51 -07001229 app_worker_reset_notify (app_wrk, s);
Dave Barach68b0fb02017-02-28 15:15:56 -05001230}
1231
Florin Corasa27a46e2019-02-18 13:02:28 -08001232int
1233session_stream_accept_notify (transport_connection_t * tc)
1234{
1235 app_worker_t *app_wrk;
1236 session_t *s;
1237
1238 s = session_get (tc->s_index, tc->thread_index);
1239 app_wrk = app_worker_get_if_valid (s->app_wrk_index);
1240 if (!app_wrk)
1241 return -1;
Florin Coras600d7a82021-04-29 11:55:23 -07001242 if (s->session_state != SESSION_STATE_CREATED)
1243 return 0;
Steven Luongd810a6e2022-10-25 13:09:11 -07001244 session_set_state (s, SESSION_STATE_ACCEPTING);
Florin Coras4dc10a42020-02-11 05:31:49 +00001245 if (app_worker_accept_notify (app_wrk, s))
1246 {
1247 /* On transport delete, no notifications should be sent. Unless, the
1248 * accept is retried and successful. */
Steven Luongd810a6e2022-10-25 13:09:11 -07001249 session_set_state (s, SESSION_STATE_CREATED);
Florin Coras4dc10a42020-02-11 05:31:49 +00001250 return -1;
1251 }
1252 return 0;
Florin Corasa27a46e2019-02-18 13:02:28 -08001253}
1254
Dave Barach68b0fb02017-02-28 15:15:56 -05001255/**
1256 * Accept a stream session. Optionally ping the server by callback.
1257 */
1258int
Florin Corasc9940fc2019-02-05 20:55:11 -08001259session_stream_accept (transport_connection_t * tc, u32 listener_index,
Nathan Skrzypczak2f0f96b2019-06-13 10:14:28 +02001260 u32 thread_index, u8 notify)
Dave Barach68b0fb02017-02-28 15:15:56 -05001261{
Florin Corasa27a46e2019-02-18 13:02:28 -08001262 session_t *s;
Dave Barach68b0fb02017-02-28 15:15:56 -05001263 int rv;
1264
Florin Corasa27a46e2019-02-18 13:02:28 -08001265 s = session_alloc_for_connection (tc);
Nathan Skrzypczak2f0f96b2019-06-13 10:14:28 +02001266 s->listener_handle = ((u64) thread_index << 32) | (u64) listener_index;
Steven Luongd810a6e2022-10-25 13:09:11 -07001267 session_set_state (s, SESSION_STATE_CREATED);
Dave Barach68b0fb02017-02-28 15:15:56 -05001268
Florin Corasa27a46e2019-02-18 13:02:28 -08001269 if ((rv = app_worker_init_accepted (s)))
Florin Coras12813d52020-04-09 20:59:20 +00001270 {
1271 session_free (s);
1272 return rv;
1273 }
Florin Corasa27a46e2019-02-18 13:02:28 -08001274
1275 session_lookup_add_connection (tc, session_handle (s));
1276
Dave Barach68b0fb02017-02-28 15:15:56 -05001277 /* Shoulder-tap the server */
1278 if (notify)
1279 {
Florin Corasa27a46e2019-02-18 13:02:28 -08001280 app_worker_t *app_wrk = app_worker_get (s->app_wrk_index);
Florin Coras12813d52020-04-09 20:59:20 +00001281 if ((rv = app_worker_accept_notify (app_wrk, s)))
1282 {
1283 session_lookup_del_session (s);
1284 segment_manager_dealloc_fifos (s->rx_fifo, s->tx_fifo);
1285 session_free (s);
1286 return rv;
1287 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001288 }
1289
1290 return 0;
1291}
1292
Florin Coras371ca502018-02-21 12:07:41 -08001293int
Florin Corase759bb52020-04-08 01:55:39 +00001294session_dgram_accept (transport_connection_t * tc, u32 listener_index,
1295 u32 thread_index)
1296{
1297 app_worker_t *app_wrk;
1298 session_t *s;
1299 int rv;
1300
1301 s = session_alloc_for_connection (tc);
1302 s->listener_handle = ((u64) thread_index << 32) | (u64) listener_index;
1303
1304 if ((rv = app_worker_init_accepted (s)))
1305 {
1306 session_free (s);
1307 return rv;
1308 }
1309
Florin Coras473556d2020-11-23 15:59:36 -08001310 session_lookup_add_connection (tc, session_handle (s));
Steven Luongd810a6e2022-10-25 13:09:11 -07001311 session_set_state (s, SESSION_STATE_ACCEPTING);
Florin Coras473556d2020-11-23 15:59:36 -08001312
Florin Corase759bb52020-04-08 01:55:39 +00001313 app_wrk = app_worker_get (s->app_wrk_index);
1314 if ((rv = app_worker_accept_notify (app_wrk, s)))
1315 {
Florin Coras473556d2020-11-23 15:59:36 -08001316 session_lookup_del_session (s);
Florin Coras12813d52020-04-09 20:59:20 +00001317 segment_manager_dealloc_fifos (s->rx_fifo, s->tx_fifo);
1318 session_free (s);
Florin Corase759bb52020-04-08 01:55:39 +00001319 return rv;
1320 }
1321
Florin Corase759bb52020-04-08 01:55:39 +00001322 return 0;
1323}
1324
1325int
Florin Coras89a9f612021-05-11 11:55:07 -07001326session_open_cl (session_endpoint_cfg_t *rmt, session_handle_t *rsh)
Florin Coras371ca502018-02-21 12:07:41 -08001327{
1328 transport_connection_t *tc;
Florin Coras5665ced2018-10-25 18:03:45 -07001329 transport_endpoint_cfg_t *tep;
Florin Coras15531972018-08-12 23:50:53 -07001330 app_worker_t *app_wrk;
Aloys Augustin20ab31e2019-03-25 11:29:17 +01001331 session_handle_t sh;
Florin Coras288eaab2019-02-03 15:26:14 -08001332 session_t *s;
Florin Coras371ca502018-02-21 12:07:41 -08001333 int rv;
1334
Florin Coras5665ced2018-10-25 18:03:45 -07001335 tep = session_endpoint_to_transport_cfg (rmt);
Florin Coras1ee78302019-02-05 15:51:15 -08001336 rv = transport_connect (rmt->transport_proto, tep);
Florin Coras371ca502018-02-21 12:07:41 -08001337 if (rv < 0)
1338 {
1339 SESSION_DBG ("Transport failed to open connection.");
Florin Coras57660d92020-04-04 22:45:34 +00001340 return rv;
Florin Coras371ca502018-02-21 12:07:41 -08001341 }
1342
Florin Coras1ee78302019-02-05 15:51:15 -08001343 tc = transport_get_half_open (rmt->transport_proto, (u32) rv);
Florin Coras371ca502018-02-21 12:07:41 -08001344
Florin Corasa27a46e2019-02-18 13:02:28 -08001345 /* For dgram type of service, allocate session and fifos now */
Florin Coras89a9f612021-05-11 11:55:07 -07001346 app_wrk = app_worker_get (rmt->app_wrk_index);
Florin Corasa27a46e2019-02-18 13:02:28 -08001347 s = session_alloc_for_connection (tc);
Florin Coras15531972018-08-12 23:50:53 -07001348 s->app_wrk_index = app_wrk->wrk_index;
Steven Luongd810a6e2022-10-25 13:09:11 -07001349 session_set_state (s, SESSION_STATE_OPENED);
Florin Corasa27a46e2019-02-18 13:02:28 -08001350 if (app_worker_init_connected (app_wrk, s))
1351 {
1352 session_free (s);
1353 return -1;
1354 }
Florin Coras371ca502018-02-21 12:07:41 -08001355
Aloys Augustin20ab31e2019-03-25 11:29:17 +01001356 sh = session_handle (s);
Florin Coras89a9f612021-05-11 11:55:07 -07001357 *rsh = sh;
1358
Aloys Augustin20ab31e2019-03-25 11:29:17 +01001359 session_lookup_add_connection (tc, sh);
Florin Coras89a9f612021-05-11 11:55:07 -07001360 return app_worker_connect_notify (app_wrk, s, SESSION_E_NONE, rmt->opaque);
Florin Coras371ca502018-02-21 12:07:41 -08001361}
1362
1363int
Florin Coras89a9f612021-05-11 11:55:07 -07001364session_open_vc (session_endpoint_cfg_t *rmt, session_handle_t *rsh)
Florin Coras371ca502018-02-21 12:07:41 -08001365{
1366 transport_connection_t *tc;
Florin Coras5665ced2018-10-25 18:03:45 -07001367 transport_endpoint_cfg_t *tep;
Florin Corasea727642021-05-07 19:39:43 -07001368 app_worker_t *app_wrk;
Florin Coras89a9f612021-05-11 11:55:07 -07001369 session_t *ho;
Florin Coras371ca502018-02-21 12:07:41 -08001370 int rv;
1371
Florin Coras5665ced2018-10-25 18:03:45 -07001372 tep = session_endpoint_to_transport_cfg (rmt);
Florin Coras1ee78302019-02-05 15:51:15 -08001373 rv = transport_connect (rmt->transport_proto, tep);
Florin Coras371ca502018-02-21 12:07:41 -08001374 if (rv < 0)
1375 {
1376 SESSION_DBG ("Transport failed to open connection.");
Florin Coras57660d92020-04-04 22:45:34 +00001377 return rv;
Florin Coras371ca502018-02-21 12:07:41 -08001378 }
1379
Florin Coras1ee78302019-02-05 15:51:15 -08001380 tc = transport_get_half_open (rmt->transport_proto, (u32) rv);
Florin Coras371ca502018-02-21 12:07:41 -08001381
Florin Coras89a9f612021-05-11 11:55:07 -07001382 app_wrk = app_worker_get (rmt->app_wrk_index);
Florin Coras371ca502018-02-21 12:07:41 -08001383
Florin Corasea727642021-05-07 19:39:43 -07001384 /* If transport offers a vc service, only allocate established
1385 * session once the connection has been established.
1386 * In the meantime allocate half-open session for tracking purposes
1387 * associate half-open connection to it and add session to app-worker
1388 * half-open table. These are needed to allocate the established
1389 * session on transport notification, and to cleanup the half-open
1390 * session if the app detaches before connection establishment.
Florin Coras371ca502018-02-21 12:07:41 -08001391 */
Florin Coras89a9f612021-05-11 11:55:07 -07001392 ho = session_alloc_for_half_open (tc);
1393 ho->app_wrk_index = app_wrk->wrk_index;
1394 ho->ho_index = app_worker_add_half_open (app_wrk, session_handle (ho));
1395 ho->opaque = rmt->opaque;
1396 *rsh = session_handle (ho);
Florin Corasd50ff7f2020-04-16 04:30:22 +00001397
Florin Coras2c876f92021-05-10 21:12:27 -07001398 if (!(tc->flags & TRANSPORT_CONNECTION_F_NO_LOOKUP))
1399 session_lookup_add_half_open (tc, tc->c_index);
Florin Coras4fde4ae2020-04-13 16:48:04 +00001400
Florin Coras371ca502018-02-21 12:07:41 -08001401 return 0;
1402}
1403
1404int
Florin Coras89a9f612021-05-11 11:55:07 -07001405session_open_app (session_endpoint_cfg_t *rmt, session_handle_t *rsh)
Florin Coras371ca502018-02-21 12:07:41 -08001406{
Florin Coras89a9f612021-05-11 11:55:07 -07001407 transport_endpoint_cfg_t *tep_cfg = session_endpoint_to_transport_cfg (rmt);
Florin Coras5665ced2018-10-25 18:03:45 -07001408
Florin Coras89a9f612021-05-11 11:55:07 -07001409 /* Not supported for now */
1410 *rsh = SESSION_INVALID_HANDLE;
Florin Coras1ee78302019-02-05 15:51:15 -08001411 return transport_connect (rmt->transport_proto, tep_cfg);
Florin Coras371ca502018-02-21 12:07:41 -08001412}
1413
Florin Coras89a9f612021-05-11 11:55:07 -07001414typedef int (*session_open_service_fn) (session_endpoint_cfg_t *,
1415 session_handle_t *);
Florin Coras371ca502018-02-21 12:07:41 -08001416
1417/* *INDENT-OFF* */
1418static session_open_service_fn session_open_srv_fns[TRANSPORT_N_SERVICES] = {
1419 session_open_vc,
1420 session_open_cl,
1421 session_open_app,
1422};
1423/* *INDENT-ON* */
1424
Florin Coras6cf30ad2017-04-04 23:08:23 -07001425/**
1426 * Ask transport to open connection to remote transport endpoint.
1427 *
1428 * Stores handle for matching request with reply since the call can be
1429 * asynchronous. For instance, for TCP the 3-way handshake must complete
1430 * before reply comes. Session is only created once connection is established.
1431 *
1432 * @param app_index Index of the application requesting the connect
1433 * @param st Session type requested.
1434 * @param tep Remote transport endpoint
Florin Coras3cbc04b2017-10-02 00:18:51 -07001435 * @param opaque Opaque data (typically, api_context) the application expects
1436 * on open completion.
Florin Coras6cf30ad2017-04-04 23:08:23 -07001437 */
Florin Corase04c2992017-03-01 08:17:34 -08001438int
Florin Coras89a9f612021-05-11 11:55:07 -07001439session_open (session_endpoint_cfg_t *rmt, session_handle_t *rsh)
Dave Barach68b0fb02017-02-28 15:15:56 -05001440{
Florin Coras1ee78302019-02-05 15:51:15 -08001441 transport_service_type_t tst;
1442 tst = transport_protocol_service_type (rmt->transport_proto);
Florin Coras89a9f612021-05-11 11:55:07 -07001443 return session_open_srv_fns[tst](rmt, rsh);
Florin Coras6cf30ad2017-04-04 23:08:23 -07001444}
1445
Florin Coras7fb0fe12018-04-09 09:24:52 -07001446/**
Florin Corasab2f6db2018-08-31 14:31:41 -07001447 * Ask transport to listen on session endpoint.
Florin Coras7fb0fe12018-04-09 09:24:52 -07001448 *
1449 * @param s Session for which listen will be called. Note that unlike
1450 * established sessions, listen sessions are not associated to a
1451 * thread.
Florin Corasab2f6db2018-08-31 14:31:41 -07001452 * @param sep Local endpoint to be listened on.
Florin Coras7fb0fe12018-04-09 09:24:52 -07001453 */
Florin Coras371ca502018-02-21 12:07:41 -08001454int
Florin Coras288eaab2019-02-03 15:26:14 -08001455session_listen (session_t * ls, session_endpoint_cfg_t * sep)
Florin Coras371ca502018-02-21 12:07:41 -08001456{
Florin Coras0bce71e2022-02-10 11:57:06 -08001457 transport_endpoint_cfg_t *tep;
Florin Corasa8c3b862020-04-07 22:31:06 +00001458 int tc_index;
1459 u32 s_index;
Florin Corasab2f6db2018-08-31 14:31:41 -07001460
1461 /* Transport bind/listen */
Florin Coras0bce71e2022-02-10 11:57:06 -08001462 tep = session_endpoint_to_transport_cfg (sep);
Florin Coras74cac882018-09-07 09:13:15 -07001463 s_index = ls->session_index;
Florin Corasd4295e62019-02-22 13:11:38 -08001464 tc_index = transport_start_listen (session_get_transport_proto (ls),
1465 s_index, tep);
Florin Corasab2f6db2018-08-31 14:31:41 -07001466
Florin Corasa8c3b862020-04-07 22:31:06 +00001467 if (tc_index < 0)
1468 return tc_index;
Florin Corasab2f6db2018-08-31 14:31:41 -07001469
Florin Corasd4295e62019-02-22 13:11:38 -08001470 /* Attach transport to session. Lookup tables are populated by the app
1471 * worker because local tables (for ct sessions) are not backed by a fib */
Florin Coras74cac882018-09-07 09:13:15 -07001472 ls = listen_session_get (s_index);
Florin Corasab2f6db2018-08-31 14:31:41 -07001473 ls->connection_index = tc_index;
Mohammed Hawari472d0da2022-10-17 17:55:35 +02001474 ls->opaque = sep->opaque;
Florin Corasab2f6db2018-08-31 14:31:41 -07001475
Florin Corasab2f6db2018-08-31 14:31:41 -07001476 return 0;
Florin Coras371ca502018-02-21 12:07:41 -08001477}
1478
Florin Coras6cf30ad2017-04-04 23:08:23 -07001479/**
1480 * Ask transport to stop listening on local transport endpoint.
1481 *
1482 * @param s Session to stop listening on. It must be in state LISTENING.
1483 */
1484int
Florin Coras288eaab2019-02-03 15:26:14 -08001485session_stop_listen (session_t * s)
Florin Coras6cf30ad2017-04-04 23:08:23 -07001486{
Florin Coras561af9b2017-12-09 10:19:43 -08001487 transport_proto_t tp = session_get_transport_proto (s);
Florin Coras6cf30ad2017-04-04 23:08:23 -07001488 transport_connection_t *tc;
Florin Coras6cf30ad2017-04-04 23:08:23 -07001489
Florin Coras1ee78302019-02-05 15:51:15 -08001490 if (s->session_state != SESSION_STATE_LISTENING)
Florin Corasa8c3b862020-04-07 22:31:06 +00001491 return SESSION_E_NOLISTEN;
Florin Coras1ee78302019-02-05 15:51:15 -08001492
1493 tc = transport_get_listener (tp, s->connection_index);
Florin Corasa8c3b862020-04-07 22:31:06 +00001494
1495 /* If no transport, assume everything was cleaned up already */
Florin Coras6cf30ad2017-04-04 23:08:23 -07001496 if (!tc)
Florin Corasa8c3b862020-04-07 22:31:06 +00001497 return SESSION_E_NONE;
Florin Coras6cf30ad2017-04-04 23:08:23 -07001498
Nathan Skrzypczak2eed1a12019-07-04 14:26:21 +02001499 if (!(tc->flags & TRANSPORT_CONNECTION_F_NO_LOOKUP))
1500 session_lookup_del_connection (tc);
Florin Corasa8c3b862020-04-07 22:31:06 +00001501
Florin Coras1ee78302019-02-05 15:51:15 -08001502 transport_stop_listen (tp, s->connection_index);
Florin Corase04c2992017-03-01 08:17:34 -08001503 return 0;
Dave Barach68b0fb02017-02-28 15:15:56 -05001504}
1505
1506/**
liuyacan534468e2021-05-09 03:50:40 +00001507 * Initialize session half-closing procedure.
1508 *
1509 * Note that half-closing will not change the state of the session.
1510 */
1511void
1512session_half_close (session_t *s)
1513{
1514 if (!s)
1515 return;
1516
1517 session_program_transport_ctrl_evt (s, SESSION_CTRL_EVT_HALF_CLOSE);
1518}
1519
1520/**
Florin Coras5a2ec8f2018-12-27 11:53:11 -08001521 * Initialize session closing procedure.
Florin Corasa5464812017-04-19 13:00:05 -07001522 *
Florin Coras2f8d8fa2018-01-26 06:36:04 -08001523 * Request is always sent to session node to ensure that all outstanding
1524 * requests are served before transport is notified.
Dave Barach68b0fb02017-02-28 15:15:56 -05001525 */
1526void
Florin Coras288eaab2019-02-03 15:26:14 -08001527session_close (session_t * s)
Dave Barach68b0fb02017-02-28 15:15:56 -05001528{
Florin Coras7c06b572023-02-20 15:14:04 -08001529 if (!s || (s->flags & SESSION_F_APP_CLOSED))
Florin Coras2f8d8fa2018-01-26 06:36:04 -08001530 return;
Florin Coras25579b42018-06-06 17:55:02 -07001531
Florin Coras7c06b572023-02-20 15:14:04 -08001532 /* Transports can close and delete their state independent of app closes
1533 * and transport initiated state transitions can hide app closes. Instead
1534 * of extending the state machine to support separate tracking of app and
1535 * transport initiated closes, use a flag. */
1536 s->flags |= SESSION_F_APP_CLOSED;
1537
Florin Coras25579b42018-06-06 17:55:02 -07001538 if (s->session_state >= SESSION_STATE_CLOSING)
1539 {
Florin Coras5a2ec8f2018-12-27 11:53:11 -08001540 /* Session will only be removed once both app and transport
1541 * acknowledge the close */
Florin Coras5f066322019-07-22 19:03:03 -07001542 if (s->session_state == SESSION_STATE_TRANSPORT_CLOSED
1543 || s->session_state == SESSION_STATE_TRANSPORT_DELETED)
Florin Corasdfb3b872019-08-16 17:48:44 -07001544 session_program_transport_ctrl_evt (s, SESSION_CTRL_EVT_CLOSE);
Florin Coras25579b42018-06-06 17:55:02 -07001545 return;
1546 }
1547
Dongya Zhang7a87c712022-11-03 15:22:34 +08001548 /* App closed so stop propagating dequeue notifications.
1549 * App might disconnect session before connected, in this case,
1550 * tx_fifo may not be setup yet, so clear only it's inited. */
1551 if (s->tx_fifo)
1552 svm_fifo_clear_deq_ntf (s->tx_fifo);
Steven Luongd810a6e2022-10-25 13:09:11 -07001553 session_set_state (s, SESSION_STATE_CLOSING);
Florin Corasdfb3b872019-08-16 17:48:44 -07001554 session_program_transport_ctrl_evt (s, SESSION_CTRL_EVT_CLOSE);
1555}
1556
1557/**
1558 * Force a close without waiting for data to be flushed
1559 */
1560void
1561session_reset (session_t * s)
1562{
1563 if (s->session_state >= SESSION_STATE_CLOSING)
1564 return;
Dongya Zhang7a87c712022-11-03 15:22:34 +08001565 /* Drop all outstanding tx data
1566 * App might disconnect session before connected, in this case,
1567 * tx_fifo may not be setup yet, so clear only it's inited. */
1568 if (s->tx_fifo)
1569 svm_fifo_dequeue_drop_all (s->tx_fifo);
Steven Luongd810a6e2022-10-25 13:09:11 -07001570 session_set_state (s, SESSION_STATE_CLOSING);
Florin Corasdfb3b872019-08-16 17:48:44 -07001571 session_program_transport_ctrl_evt (s, SESSION_CTRL_EVT_RESET);
Florin Coras2f8d8fa2018-01-26 06:36:04 -08001572}
1573
1574/**
liuyacan534468e2021-05-09 03:50:40 +00001575 * Notify transport the session can be half-disconnected.
1576 *
1577 * Must be called from the session's thread.
1578 */
1579void
1580session_transport_half_close (session_t *s)
1581{
1582 /* Only READY session can be half-closed */
1583 if (s->session_state != SESSION_STATE_READY)
1584 {
1585 return;
1586 }
1587
1588 transport_half_close (session_get_transport_proto (s), s->connection_index,
1589 s->thread_index);
1590}
1591
1592/**
Florin Coras2f8d8fa2018-01-26 06:36:04 -08001593 * Notify transport the session can be disconnected. This should eventually
1594 * result in a delete notification that allows us to cleanup session state.
1595 * Called for both active/passive disconnects.
1596 *
1597 * Must be called from the session's thread.
1598 */
1599void
Florin Coras288eaab2019-02-03 15:26:14 -08001600session_transport_close (session_t * s)
Florin Coras2f8d8fa2018-01-26 06:36:04 -08001601{
Florin Coras692b9492019-07-12 15:01:53 -07001602 if (s->session_state >= SESSION_STATE_APP_CLOSED)
Florin Coras568ebc72018-09-18 16:12:50 -07001603 {
Florin Coras5f066322019-07-22 19:03:03 -07001604 if (s->session_state == SESSION_STATE_TRANSPORT_CLOSED)
Steven Luongd810a6e2022-10-25 13:09:11 -07001605 session_set_state (s, SESSION_STATE_CLOSED);
Florin Coras5f066322019-07-22 19:03:03 -07001606 /* If transport is already deleted, just free the session */
1607 else if (s->session_state >= SESSION_STATE_TRANSPORT_DELETED)
Florin Coras692b9492019-07-12 15:01:53 -07001608 session_free_w_fifos (s);
Florin Coras568ebc72018-09-18 16:12:50 -07001609 return;
1610 }
Florin Coras78cc4b02018-12-20 18:24:49 -08001611
Florin Coras692b9492019-07-12 15:01:53 -07001612 /* If the tx queue wasn't drained, the transport can continue to try
1613 * sending the outstanding data (in closed state it cannot). It MUST however
1614 * at one point, either after sending everything or after a timeout, call
1615 * delete notify. This will finally lead to the complete cleanup of the
1616 * session.
Florin Coras78cc4b02018-12-20 18:24:49 -08001617 */
Steven Luongd810a6e2022-10-25 13:09:11 -07001618 session_set_state (s, SESSION_STATE_APP_CLOSED);
Florin Coras78cc4b02018-12-20 18:24:49 -08001619
Florin Coras1ee78302019-02-05 15:51:15 -08001620 transport_close (session_get_transport_proto (s), s->connection_index,
1621 s->thread_index);
Dave Barach68b0fb02017-02-28 15:15:56 -05001622}
1623
1624/**
Florin Corasdfb3b872019-08-16 17:48:44 -07001625 * Force transport close
1626 */
1627void
1628session_transport_reset (session_t * s)
1629{
1630 if (s->session_state >= SESSION_STATE_APP_CLOSED)
1631 {
1632 if (s->session_state == SESSION_STATE_TRANSPORT_CLOSED)
Steven Luongd810a6e2022-10-25 13:09:11 -07001633 session_set_state (s, SESSION_STATE_CLOSED);
Florin Corasdfb3b872019-08-16 17:48:44 -07001634 else if (s->session_state >= SESSION_STATE_TRANSPORT_DELETED)
1635 session_free_w_fifos (s);
1636 return;
1637 }
1638
Steven Luongd810a6e2022-10-25 13:09:11 -07001639 session_set_state (s, SESSION_STATE_APP_CLOSED);
Florin Corasdfb3b872019-08-16 17:48:44 -07001640 transport_reset (session_get_transport_proto (s), s->connection_index,
1641 s->thread_index);
1642}
1643
1644/**
Dave Barach68b0fb02017-02-28 15:15:56 -05001645 * Cleanup transport and session state.
Florin Corasd79b41e2017-03-04 05:37:52 -08001646 *
Florin Coras25579b42018-06-06 17:55:02 -07001647 * Notify transport of the cleanup and free the session. This should
1648 * be called only if transport reported some error and is already
1649 * closed.
Dave Barach68b0fb02017-02-28 15:15:56 -05001650 */
1651void
Florin Coras288eaab2019-02-03 15:26:14 -08001652session_transport_cleanup (session_t * s)
Dave Barach68b0fb02017-02-28 15:15:56 -05001653{
Florin Coras25579b42018-06-06 17:55:02 -07001654 /* Delete from main lookup table before we axe the the transport */
1655 session_lookup_del_session (s);
Florin Coras5afea122019-10-28 08:46:37 -07001656 if (s->session_state != SESSION_STATE_TRANSPORT_DELETED)
1657 transport_cleanup (session_get_transport_proto (s), s->connection_index,
1658 s->thread_index);
Florin Coras25579b42018-06-06 17:55:02 -07001659 /* Since we called cleanup, no delete notification will come. So, make
1660 * sure the session is properly freed. */
Florin Corasd3955fb2019-11-29 09:31:47 -08001661 segment_manager_dealloc_fifos (s->rx_fifo, s->tx_fifo);
1662 session_free (s);
Dave Barach68b0fb02017-02-28 15:15:56 -05001663}
1664
Florin Corasa5464812017-04-19 13:00:05 -07001665/**
Florin Coras8afe1b82021-11-17 23:38:54 -08001666 * Allocate worker mqs in share-able segment
Florin Corasb384b542018-01-15 01:08:33 -08001667 *
Florin Coras8afe1b82021-11-17 23:38:54 -08001668 * That can only be a newly created memfd segment, that must be mapped
1669 * by all apps/stack users unless private rx mqs are enabled.
Florin Corasa5464812017-04-19 13:00:05 -07001670 */
1671void
Florin Coras8afe1b82021-11-17 23:38:54 -08001672session_vpp_wrk_mqs_alloc (session_main_t *smm)
Florin Corasa5464812017-04-19 13:00:05 -07001673{
Florin Coras8afe1b82021-11-17 23:38:54 -08001674 u32 mq_q_length = 2048, evt_size = sizeof (session_event_t);
1675 fifo_segment_t *mqs_seg = &smm->wrk_mqs_segment;
1676 svm_msg_q_cfg_t _cfg, *cfg = &_cfg;
1677 uword mqs_seg_size;
Florin Corasb384b542018-01-15 01:08:33 -08001678 int i;
Florin Corasa5464812017-04-19 13:00:05 -07001679
Florin Coras8afe1b82021-11-17 23:38:54 -08001680 mq_q_length = clib_max (mq_q_length, smm->configured_wrk_mq_length);
Florin Corasb384b542018-01-15 01:08:33 -08001681
Florin Coras8afe1b82021-11-17 23:38:54 -08001682 svm_msg_q_ring_cfg_t rc[SESSION_MQ_N_RINGS] = {
1683 { mq_q_length, evt_size, 0 }, { mq_q_length >> 1, 256, 0 }
1684 };
1685 cfg->consumer_pid = 0;
1686 cfg->n_rings = 2;
1687 cfg->q_nitems = mq_q_length;
1688 cfg->ring_cfgs = rc;
Florin Coras6c10ab22020-11-08 16:50:39 -08001689
Florin Coras8afe1b82021-11-17 23:38:54 -08001690 /*
1691 * Compute mqs segment size based on rings config and leave space
1692 * for passing extended configuration messages, i.e., data allocated
1693 * outside of the rings. If provided with a config value, accept it
1694 * if larger than minimum size.
1695 */
1696 mqs_seg_size = svm_msg_q_size_to_alloc (cfg) * vec_len (smm->wrk);
Florin Corascf5c7742022-06-28 14:34:45 -07001697 mqs_seg_size = mqs_seg_size + (1 << 20);
Florin Coras8afe1b82021-11-17 23:38:54 -08001698 mqs_seg_size = clib_max (mqs_seg_size, smm->wrk_mqs_segment_size);
1699
1700 mqs_seg->ssvm.ssvm_size = mqs_seg_size;
1701 mqs_seg->ssvm.my_pid = getpid ();
1702 mqs_seg->ssvm.name = format (0, "%s%c", "session: wrk-mqs-segment", 0);
Florin Coras6c10ab22020-11-08 16:50:39 -08001703
Florin Coras8afe1b82021-11-17 23:38:54 -08001704 if (ssvm_server_init (&mqs_seg->ssvm, SSVM_SEGMENT_MEMFD))
Florin Corasa5464812017-04-19 13:00:05 -07001705 {
Florin Coras6c10ab22020-11-08 16:50:39 -08001706 clib_warning ("failed to initialize queue segment");
1707 return;
Florin Corasa5464812017-04-19 13:00:05 -07001708 }
Florin Corasb384b542018-01-15 01:08:33 -08001709
Florin Coras8afe1b82021-11-17 23:38:54 -08001710 fifo_segment_init (mqs_seg);
Florin Corasb384b542018-01-15 01:08:33 -08001711
Florin Corasb4624182020-12-11 13:58:12 -08001712 /* Special fifo segment that's filled only with mqs */
Florin Coras8afe1b82021-11-17 23:38:54 -08001713 mqs_seg->h->n_mqs = vec_len (smm->wrk);
Florin Corasb4624182020-12-11 13:58:12 -08001714
Florin Coras5a7ca7b2018-10-30 12:01:48 -07001715 for (i = 0; i < vec_len (smm->wrk); i++)
Florin Coras8afe1b82021-11-17 23:38:54 -08001716 smm->wrk[i].vpp_event_queue = fifo_segment_msg_q_alloc (mqs_seg, i, cfg);
Florin Corasb384b542018-01-15 01:08:33 -08001717}
1718
Florin Coras04943b42020-12-25 11:45:40 -08001719fifo_segment_t *
Florin Coras8afe1b82021-11-17 23:38:54 -08001720session_main_get_wrk_mqs_segment (void)
Florin Corasb384b542018-01-15 01:08:33 -08001721{
Florin Coras8afe1b82021-11-17 23:38:54 -08001722 return &session_main.wrk_mqs_segment;
Florin Corasa5464812017-04-19 13:00:05 -07001723}
1724
Florin Coras31c99552019-03-01 13:00:58 -08001725u64
1726session_segment_handle (session_t * s)
1727{
1728 svm_fifo_t *f;
1729
Aloys Augustincdb71702019-04-08 17:54:39 +02001730 if (!s->rx_fifo)
Florin Coras31c99552019-03-01 13:00:58 -08001731 return SESSION_INVALID_HANDLE;
1732
1733 f = s->rx_fifo;
1734 return segment_manager_make_segment_handle (f->segment_manager,
1735 f->segment_index);
1736}
1737
Florin Coras371ca502018-02-21 12:07:41 -08001738/* *INDENT-OFF* */
1739static session_fifo_rx_fn *session_tx_fns[TRANSPORT_TX_N_FNS] = {
1740 session_tx_fifo_peek_and_snd,
1741 session_tx_fifo_dequeue_and_snd,
Florin Coras7fb0fe12018-04-09 09:24:52 -07001742 session_tx_fifo_dequeue_internal,
1743 session_tx_fifo_dequeue_and_snd
Florin Coras371ca502018-02-21 12:07:41 -08001744};
1745/* *INDENT-ON* */
1746
Florin Coras561af9b2017-12-09 10:19:43 -08001747void
1748session_register_transport (transport_proto_t transport_proto,
1749 const transport_proto_vft_t * vft, u8 is_ip4,
1750 u32 output_node)
Dave Barach2c25a622017-06-26 11:35:07 -04001751{
Florin Coras31c99552019-03-01 13:00:58 -08001752 session_main_t *smm = &session_main;
Florin Coras561af9b2017-12-09 10:19:43 -08001753 session_type_t session_type;
1754 u32 next_index = ~0;
Dave Barach2c25a622017-06-26 11:35:07 -04001755
Florin Coras561af9b2017-12-09 10:19:43 -08001756 session_type = session_type_from_proto_and_ip (transport_proto, is_ip4);
1757
1758 vec_validate (smm->session_type_to_next, session_type);
Florin Coras561af9b2017-12-09 10:19:43 -08001759 vec_validate (smm->session_tx_fns, session_type);
1760
Florin Coras371ca502018-02-21 12:07:41 -08001761 if (output_node != ~0)
Florin Coraseb839cc2021-04-14 11:23:55 -07001762 next_index = vlib_node_add_next (vlib_get_main (),
1763 session_queue_node.index, output_node);
Florin Coras561af9b2017-12-09 10:19:43 -08001764
1765 smm->session_type_to_next[session_type] = next_index;
Nathan Skrzypczake971bc92019-06-19 13:42:37 +02001766 smm->session_tx_fns[session_type] =
1767 session_tx_fns[vft->transport_options.tx_type];
Dave Barach2c25a622017-06-26 11:35:07 -04001768}
1769
Florin Coras5384cca2022-01-20 18:10:26 -08001770void
1771session_register_update_time_fn (session_update_time_fn fn, u8 is_add)
1772{
1773 session_main_t *smm = &session_main;
1774 session_update_time_fn *fi;
1775 u32 fi_pos = ~0;
1776 u8 found = 0;
1777
1778 vec_foreach (fi, smm->update_time_fns)
1779 {
1780 if (*fi == fn)
1781 {
1782 fi_pos = fi - smm->update_time_fns;
1783 found = 1;
1784 break;
1785 }
1786 }
1787
1788 if (is_add)
1789 {
1790 if (found)
1791 {
1792 clib_warning ("update time fn %p already registered", fn);
1793 return;
1794 }
1795 vec_add1 (smm->update_time_fns, fn);
1796 }
1797 else
1798 {
1799 vec_del1 (smm->update_time_fns, fi_pos);
1800 }
1801}
1802
Florin Coras07063b82020-03-13 04:44:51 +00001803transport_proto_t
1804session_add_transport_proto (void)
1805{
1806 session_main_t *smm = &session_main;
1807 session_worker_t *wrk;
1808 u32 thread;
1809
1810 smm->last_transport_proto_type += 1;
1811
1812 for (thread = 0; thread < vec_len (smm->wrk); thread++)
1813 {
1814 wrk = session_main_get_worker (thread);
1815 vec_validate (wrk->session_to_enqueue, smm->last_transport_proto_type);
1816 }
1817
1818 return smm->last_transport_proto_type;
1819}
1820
Florin Coras3cbc04b2017-10-02 00:18:51 -07001821transport_connection_t *
Florin Coras288eaab2019-02-03 15:26:14 -08001822session_get_transport (session_t * s)
Florin Coras3cbc04b2017-10-02 00:18:51 -07001823{
Florin Corasade70e42017-10-14 18:56:41 -07001824 if (s->session_state != SESSION_STATE_LISTENING)
Florin Coras4edc37e2019-02-04 23:01:34 -08001825 return transport_get_connection (session_get_transport_proto (s),
1826 s->connection_index, s->thread_index);
1827 else
1828 return transport_get_listener (session_get_transport_proto (s),
1829 s->connection_index);
Florin Coras3cbc04b2017-10-02 00:18:51 -07001830}
1831
Aloys Augustincdb71702019-04-08 17:54:39 +02001832void
Florin Coras09d18c22019-04-24 11:10:02 -07001833session_get_endpoint (session_t * s, transport_endpoint_t * tep, u8 is_lcl)
Aloys Augustincdb71702019-04-08 17:54:39 +02001834{
1835 if (s->session_state != SESSION_STATE_LISTENING)
1836 return transport_get_endpoint (session_get_transport_proto (s),
Florin Coras09d18c22019-04-24 11:10:02 -07001837 s->connection_index, s->thread_index, tep,
1838 is_lcl);
Aloys Augustincdb71702019-04-08 17:54:39 +02001839 else
1840 return transport_get_listener_endpoint (session_get_transport_proto (s),
Florin Coras09d18c22019-04-24 11:10:02 -07001841 s->connection_index, tep, is_lcl);
Aloys Augustincdb71702019-04-08 17:54:39 +02001842}
1843
Florin Coras04ae8272021-04-12 19:55:37 -07001844int
1845session_transport_attribute (session_t *s, u8 is_get,
1846 transport_endpt_attr_t *attr)
1847{
1848 if (s->session_state < SESSION_STATE_READY)
1849 return -1;
1850
1851 return transport_connection_attribute (session_get_transport_proto (s),
1852 s->connection_index, s->thread_index,
1853 is_get, attr);
1854}
1855
Florin Coras3cbc04b2017-10-02 00:18:51 -07001856transport_connection_t *
Florin Coras288eaab2019-02-03 15:26:14 -08001857listen_session_get_transport (session_t * s)
Florin Coras3cbc04b2017-10-02 00:18:51 -07001858{
Florin Coras4edc37e2019-02-04 23:01:34 -08001859 return transport_get_listener (session_get_transport_proto (s),
1860 s->connection_index);
Florin Coras3cbc04b2017-10-02 00:18:51 -07001861}
1862
Florin Corasfd542f12018-05-16 19:28:24 -07001863void
Florin Coras5484daa2020-03-27 23:55:06 +00001864session_queue_run_on_main_thread (vlib_main_t * vm)
Florin Corasfd542f12018-05-16 19:28:24 -07001865{
1866 ASSERT (vlib_get_thread_index () == 0);
Florin Coras1d2e38b2021-03-17 21:52:49 -07001867 vlib_node_set_interrupt_pending (vm, session_queue_node.index);
Florin Corasfd542f12018-05-16 19:28:24 -07001868}
1869
Florin Corasd82f4712022-04-25 16:15:02 -07001870static void
1871session_stats_collector_fn (vlib_stats_collector_data_t *d)
1872{
1873 u32 i, n_workers, n_wrk_sessions, n_sessions = 0;
1874 session_main_t *smm = &session_main;
1875 session_worker_t *wrk;
1876 counter_t **counters;
1877 counter_t *cb;
1878
1879 n_workers = vec_len (smm->wrk);
1880 vlib_stats_validate (d->entry_index, 0, n_workers - 1);
1881 counters = d->entry->data;
1882 cb = counters[0];
1883
1884 for (i = 0; i < vec_len (smm->wrk); i++)
1885 {
1886 wrk = session_main_get_worker (i);
1887 n_wrk_sessions = pool_elts (wrk->sessions);
1888 cb[i] = n_wrk_sessions;
1889 n_sessions += n_wrk_sessions;
1890 }
1891
1892 vlib_stats_set_gauge (d->private_data, n_sessions);
1893}
1894
1895static void
1896session_stats_collector_init (void)
1897{
Florin Coras975e0df2022-04-26 19:32:11 -07001898 vlib_stats_collector_reg_t reg = {};
Florin Corasd82f4712022-04-25 16:15:02 -07001899
1900 reg.entry_index =
1901 vlib_stats_add_counter_vector ("/sys/session/sessions_per_worker");
1902 reg.private_data = vlib_stats_add_gauge ("/sys/session/sessions_total");
1903 reg.collect_fn = session_stats_collector_fn;
1904 vlib_stats_register_collector_fn (&reg);
1905 vlib_stats_validate (reg.entry_index, 0, vlib_get_n_threads ());
1906}
1907
Dave Barach68b0fb02017-02-28 15:15:56 -05001908static clib_error_t *
Florin Corase04c2992017-03-01 08:17:34 -08001909session_manager_main_enable (vlib_main_t * vm)
Dave Barach68b0fb02017-02-28 15:15:56 -05001910{
Florin Coras31c99552019-03-01 13:00:58 -08001911 session_main_t *smm = &session_main;
Florin Corase04c2992017-03-01 08:17:34 -08001912 vlib_thread_main_t *vtm = vlib_get_thread_main ();
Florin Coras371ca502018-02-21 12:07:41 -08001913 u32 num_threads, preallocated_sessions_per_worker;
Florin Coras31c99552019-03-01 13:00:58 -08001914 session_worker_t *wrk;
Florin Corasd5c604d2019-03-18 09:06:35 -07001915 int i;
Dave Barach68b0fb02017-02-28 15:15:56 -05001916
Florin Corase10da622020-10-25 14:00:29 -07001917 /* We only initialize once and do not de-initialized on disable */
1918 if (smm->is_initialized)
1919 goto done;
1920
Dave Barach68b0fb02017-02-28 15:15:56 -05001921 num_threads = 1 /* main thread */ + vtm->n_threads;
1922
1923 if (num_threads < 1)
1924 return clib_error_return (0, "n_thread_stacks not set");
1925
Florin Coras5f56d732018-10-30 10:21:59 -07001926 /* Allocate cache line aligned worker contexts */
Florin Coras5a7ca7b2018-10-30 12:01:48 -07001927 vec_validate_aligned (smm->wrk, num_threads - 1, CLIB_CACHE_LINE_BYTES);
Florin Coras53d8d4f2022-03-09 13:55:38 -08001928 clib_spinlock_init (&session_main.pool_realloc_lock);
Florin Coras3cbc04b2017-10-02 00:18:51 -07001929
Dave Barachacd2a6a2017-05-16 17:41:34 -04001930 for (i = 0; i < num_threads; i++)
1931 {
Florin Coras5a7ca7b2018-10-30 12:01:48 -07001932 wrk = &smm->wrk[i];
Florin Corasb0ffbee2019-07-21 19:23:46 -07001933 wrk->ctrl_head = clib_llist_make_head (wrk->event_elts, evt_list);
Florin Coras2062ec02019-07-15 13:15:18 -07001934 wrk->new_head = clib_llist_make_head (wrk->event_elts, evt_list);
Florin Coras60183db2019-07-20 15:53:16 -07001935 wrk->old_head = clib_llist_make_head (wrk->event_elts, evt_list);
Florin Corasaf972212021-05-05 09:54:00 -07001936 wrk->pending_connects = clib_llist_make_head (wrk->event_elts, evt_list);
Florin Corase09bd482022-03-21 10:38:01 -07001937 wrk->evts_pending_main =
1938 clib_llist_make_head (wrk->event_elts, evt_list);
Damjan Marion6ffb7c62021-03-26 13:06:13 +01001939 wrk->vm = vlib_get_main_by_index (i);
Dave Barach77d98382020-04-28 18:00:21 -04001940 wrk->last_vlib_time = vlib_time_now (vm);
Florin Corasa8e71c82019-10-22 19:01:39 -07001941 wrk->last_vlib_us_time = wrk->last_vlib_time * CLIB_US_TIME_FREQ;
Florin Corasa48dffe2021-05-16 10:58:53 -07001942 wrk->timerfd = -1;
Florin Coras07063b82020-03-13 04:44:51 +00001943 vec_validate (wrk->session_to_enqueue, smm->last_transport_proto_type);
Florin Corasd67f1122018-05-21 17:47:40 -07001944
Florin Coras7da88292021-03-18 15:04:34 -07001945 if (!smm->no_adaptive && smm->use_private_rx_mqs)
1946 session_wrk_enable_adaptive_mode (wrk);
Dave Barachacd2a6a2017-05-16 17:41:34 -04001947 }
1948
Florin Corasb384b542018-01-15 01:08:33 -08001949 /* Allocate vpp event queues segment and queue */
Florin Coras8afe1b82021-11-17 23:38:54 -08001950 session_vpp_wrk_mqs_alloc (smm);
Florin Corasb384b542018-01-15 01:08:33 -08001951
Florin Coras9a45bd82020-12-28 16:28:07 -08001952 /* Initialize segment manager properties */
1953 segment_manager_main_init ();
Florin Coras6cf30ad2017-04-04 23:08:23 -07001954
Florin Coras66b11312017-07-31 17:18:03 -07001955 /* Preallocate sessions */
Dave Barachb7f1faa2017-08-29 11:43:37 -04001956 if (smm->preallocated_sessions)
Dave Barach68b0fb02017-02-28 15:15:56 -05001957 {
Dave Barachb7f1faa2017-08-29 11:43:37 -04001958 if (num_threads == 1)
Florin Coras66b11312017-07-31 17:18:03 -07001959 {
Florin Coras5a7ca7b2018-10-30 12:01:48 -07001960 pool_init_fixed (smm->wrk[0].sessions, smm->preallocated_sessions);
Florin Coras66b11312017-07-31 17:18:03 -07001961 }
Dave Barachb7f1faa2017-08-29 11:43:37 -04001962 else
Florin Coras66b11312017-07-31 17:18:03 -07001963 {
Dave Barachb7f1faa2017-08-29 11:43:37 -04001964 int j;
1965 preallocated_sessions_per_worker =
1966 (1.1 * (f64) smm->preallocated_sessions /
1967 (f64) (num_threads - 1));
1968
1969 for (j = 1; j < num_threads; j++)
Florin Coras66b11312017-07-31 17:18:03 -07001970 {
Florin Coras5a7ca7b2018-10-30 12:01:48 -07001971 pool_init_fixed (smm->wrk[j].sessions,
Dave Barachb7f1faa2017-08-29 11:43:37 -04001972 preallocated_sessions_per_worker);
Florin Coras66b11312017-07-31 17:18:03 -07001973 }
Florin Coras66b11312017-07-31 17:18:03 -07001974 }
1975 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001976
Florin Coras04e53442017-07-16 17:12:15 -07001977 session_lookup_init ();
Florin Corascea194d2017-10-02 00:18:51 -07001978 app_namespaces_init ();
Florin Coras3cbc04b2017-10-02 00:18:51 -07001979 transport_init ();
Florin Corasd82f4712022-04-25 16:15:02 -07001980 session_stats_collector_init ();
Florin Corase10da622020-10-25 14:00:29 -07001981 smm->is_initialized = 1;
1982
1983done:
Dave Barach68b0fb02017-02-28 15:15:56 -05001984
Florin Corase04c2992017-03-01 08:17:34 -08001985 smm->is_enabled = 1;
1986
Florin Coras561af9b2017-12-09 10:19:43 -08001987 /* Enable transports */
1988 transport_enable_disable (vm, 1);
Florin Coras11166672020-04-13 01:20:25 +00001989 session_debug_init ();
Srikanth Akula73570432020-04-06 19:19:49 -07001990
Dave Barach68b0fb02017-02-28 15:15:56 -05001991 return 0;
1992}
1993
Florin Corase10da622020-10-25 14:00:29 -07001994static void
1995session_manager_main_disable (vlib_main_t * vm)
1996{
1997 transport_enable_disable (vm, 0 /* is_en */ );
1998}
1999
Marvin Liu06542422022-08-16 06:49:09 +00002000/* in this new callback, cookie hint the index */
2001void
2002session_dma_completion_cb (vlib_main_t *vm, struct vlib_dma_batch *batch)
2003{
2004 session_worker_t *wrk;
2005 wrk = session_main_get_worker (vm->thread_index);
2006 session_dma_transfer *dma_transfer;
2007
2008 dma_transfer = &wrk->dma_trans[wrk->trans_head];
2009 vec_add (wrk->pending_tx_buffers, dma_transfer->pending_tx_buffers,
2010 vec_len (dma_transfer->pending_tx_buffers));
2011 vec_add (wrk->pending_tx_nexts, dma_transfer->pending_tx_nexts,
2012 vec_len (dma_transfer->pending_tx_nexts));
2013 vec_reset_length (dma_transfer->pending_tx_buffers);
2014 vec_reset_length (dma_transfer->pending_tx_nexts);
2015 wrk->trans_head++;
2016 if (wrk->trans_head == wrk->trans_size)
2017 wrk->trans_head = 0;
2018 return;
2019}
2020
2021static void
2022session_prepare_dma_args (vlib_dma_config_t *args)
2023{
2024 args->max_transfers = DMA_TRANS_SIZE;
2025 args->max_transfer_size = 65536;
2026 args->features = 0;
2027 args->sw_fallback = 1;
2028 args->barrier_before_last = 1;
2029 args->callback_fn = session_dma_completion_cb;
2030}
2031
2032static void
2033session_node_enable_dma (u8 is_en, int n_vlibs)
2034{
2035 vlib_dma_config_t args;
2036 session_prepare_dma_args (&args);
2037 session_worker_t *wrk;
2038 vlib_main_t *vm;
2039
2040 int config_index = -1;
2041
2042 if (is_en)
2043 {
2044 vm = vlib_get_main_by_index (0);
2045 config_index = vlib_dma_config_add (vm, &args);
2046 }
2047 else
2048 {
2049 vm = vlib_get_main_by_index (0);
2050 wrk = session_main_get_worker (0);
2051 if (wrk->config_index >= 0)
2052 vlib_dma_config_del (vm, wrk->config_index);
2053 }
2054 int i;
2055 for (i = 0; i < n_vlibs; i++)
2056 {
2057 vm = vlib_get_main_by_index (i);
2058 wrk = session_main_get_worker (vm->thread_index);
2059 wrk->config_index = config_index;
2060 if (is_en)
2061 {
2062 if (config_index >= 0)
2063 wrk->dma_enabled = true;
2064 wrk->dma_trans = (session_dma_transfer *) clib_mem_alloc (
2065 sizeof (session_dma_transfer) * DMA_TRANS_SIZE);
2066 bzero (wrk->dma_trans,
2067 sizeof (session_dma_transfer) * DMA_TRANS_SIZE);
2068 }
2069 else
2070 {
2071 if (wrk->dma_trans)
2072 clib_mem_free (wrk->dma_trans);
2073 }
2074 wrk->trans_head = 0;
2075 wrk->trans_tail = 0;
2076 wrk->trans_size = DMA_TRANS_SIZE;
2077 }
2078}
2079
Florin Corasa5464812017-04-19 13:00:05 -07002080void
2081session_node_enable_disable (u8 is_en)
2082{
Florin Coras1d2e38b2021-03-17 21:52:49 -07002083 u8 mstate = is_en ? VLIB_NODE_STATE_INTERRUPT : VLIB_NODE_STATE_DISABLED;
Florin Corasa5464812017-04-19 13:00:05 -07002084 u8 state = is_en ? VLIB_NODE_STATE_POLLING : VLIB_NODE_STATE_DISABLED;
Florin Coras1d2e38b2021-03-17 21:52:49 -07002085 session_main_t *sm = &session_main;
2086 vlib_main_t *vm;
2087 vlib_node_t *n;
2088 int n_vlibs, i;
Florin Corasfd542f12018-05-16 19:28:24 -07002089
Florin Coras1d2e38b2021-03-17 21:52:49 -07002090 n_vlibs = vlib_get_n_threads ();
2091 for (i = 0; i < n_vlibs; i++)
Damjan Marion92ccf9b2021-03-26 11:38:01 +01002092 {
Florin Coras1d2e38b2021-03-17 21:52:49 -07002093 vm = vlib_get_main_by_index (i);
2094 /* main thread with workers and not polling */
2095 if (i == 0 && n_vlibs > 1)
Damjan Marion92ccf9b2021-03-26 11:38:01 +01002096 {
Florin Coras1d2e38b2021-03-17 21:52:49 -07002097 vlib_node_set_state (vm, session_queue_node.index, mstate);
Damjan Marion92ccf9b2021-03-26 11:38:01 +01002098 if (is_en)
2099 {
Florin Corasb32af352021-05-19 07:58:49 -07002100 session_main_get_worker (0)->state = SESSION_WRK_INTERRUPT;
Florin Coras1d2e38b2021-03-17 21:52:49 -07002101 vlib_node_set_state (vm, session_queue_process_node.index,
2102 state);
2103 n = vlib_get_node (vm, session_queue_process_node.index);
2104 vlib_start_process (vm, n->runtime_index);
Damjan Marion92ccf9b2021-03-26 11:38:01 +01002105 }
2106 else
2107 {
Florin Coras1d2e38b2021-03-17 21:52:49 -07002108 vlib_process_signal_event_mt (vm,
Damjan Marion92ccf9b2021-03-26 11:38:01 +01002109 session_queue_process_node.index,
2110 SESSION_Q_PROCESS_STOP, 0);
2111 }
Florin Coras1d2e38b2021-03-17 21:52:49 -07002112 if (!sm->poll_main)
Damjan Marion92ccf9b2021-03-26 11:38:01 +01002113 continue;
2114 }
Florin Coras1d2e38b2021-03-17 21:52:49 -07002115 vlib_node_set_state (vm, session_queue_node.index, state);
Damjan Marion92ccf9b2021-03-26 11:38:01 +01002116 }
Florin Coras41d5f542021-01-15 13:49:33 -08002117
Florin Coras1d2e38b2021-03-17 21:52:49 -07002118 if (sm->use_private_rx_mqs)
Florin Coras41d5f542021-01-15 13:49:33 -08002119 application_enable_rx_mqs_nodes (is_en);
Marvin Liu06542422022-08-16 06:49:09 +00002120
2121 if (sm->dma_enabled)
2122 session_node_enable_dma (is_en, n_vlibs);
Florin Corasa5464812017-04-19 13:00:05 -07002123}
2124
Florin Corase04c2992017-03-01 08:17:34 -08002125clib_error_t *
2126vnet_session_enable_disable (vlib_main_t * vm, u8 is_en)
2127{
Florin Corascea194d2017-10-02 00:18:51 -07002128 clib_error_t *error = 0;
Florin Corase04c2992017-03-01 08:17:34 -08002129 if (is_en)
2130 {
Florin Coras31c99552019-03-01 13:00:58 -08002131 if (session_main.is_enabled)
Florin Corase04c2992017-03-01 08:17:34 -08002132 return 0;
Dave Barach68b0fb02017-02-28 15:15:56 -05002133
Florin Corascea194d2017-10-02 00:18:51 -07002134 error = session_manager_main_enable (vm);
Vladimir Kropyleva2e44512019-07-16 21:32:41 +03002135 session_node_enable_disable (is_en);
Florin Corase04c2992017-03-01 08:17:34 -08002136 }
2137 else
2138 {
Florin Coras31c99552019-03-01 13:00:58 -08002139 session_main.is_enabled = 0;
Florin Corase10da622020-10-25 14:00:29 -07002140 session_manager_main_disable (vm);
Florin Corasa5464812017-04-19 13:00:05 -07002141 session_node_enable_disable (is_en);
Florin Corase04c2992017-03-01 08:17:34 -08002142 }
2143
Florin Corascea194d2017-10-02 00:18:51 -07002144 return error;
Florin Corase04c2992017-03-01 08:17:34 -08002145}
2146
Florin Corase04c2992017-03-01 08:17:34 -08002147clib_error_t *
Florin Corase33c0022020-04-03 17:23:42 +00002148session_main_init (vlib_main_t * vm)
Florin Corase04c2992017-03-01 08:17:34 -08002149{
Florin Coras31c99552019-03-01 13:00:58 -08002150 session_main_t *smm = &session_main;
Florin Corase33c0022020-04-03 17:23:42 +00002151
2152 smm->is_enabled = 0;
2153 smm->session_enable_asap = 0;
Florin Corase92c9462020-11-25 08:44:16 -08002154 smm->poll_main = 0;
Florin Coras41d5f542021-01-15 13:49:33 -08002155 smm->use_private_rx_mqs = 0;
Florin Coras7da88292021-03-18 15:04:34 -07002156 smm->no_adaptive = 0;
Florin Coras0b656212022-01-13 11:59:44 -08002157 smm->last_transport_proto_type = TRANSPORT_PROTO_HTTP;
Florin Corase33c0022020-04-03 17:23:42 +00002158
Nathan Skrzypczak31bd0352019-11-07 17:55:01 +01002159 return 0;
2160}
2161
2162static clib_error_t *
Florin Corase33c0022020-04-03 17:23:42 +00002163session_main_loop_init (vlib_main_t * vm)
Nathan Skrzypczak31bd0352019-11-07 17:55:01 +01002164{
2165 session_main_t *smm = &session_main;
2166 if (smm->session_enable_asap)
2167 {
2168 vlib_worker_thread_barrier_sync (vm);
2169 vnet_session_enable_disable (vm, 1 /* is_en */ );
2170 vlib_worker_thread_barrier_release (vm);
2171 }
Florin Corase04c2992017-03-01 08:17:34 -08002172 return 0;
2173}
2174
Florin Corase33c0022020-04-03 17:23:42 +00002175VLIB_INIT_FUNCTION (session_main_init);
2176VLIB_MAIN_LOOP_ENTER_FUNCTION (session_main_loop_init);
Dave Barach2c25a622017-06-26 11:35:07 -04002177
2178static clib_error_t *
2179session_config_fn (vlib_main_t * vm, unformat_input_t * input)
Dave Barach10d8cc62017-05-30 09:30:07 -04002180{
Florin Coras31c99552019-03-01 13:00:58 -08002181 session_main_t *smm = &session_main;
Dave Barach10d8cc62017-05-30 09:30:07 -04002182 u32 nitems;
Florin Coras66b11312017-07-31 17:18:03 -07002183 uword tmp;
Dave Barach10d8cc62017-05-30 09:30:07 -04002184
2185 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2186 {
Florin Coras8afe1b82021-11-17 23:38:54 -08002187 if (unformat (input, "wrk-mq-length %d", &nitems))
Dave Barach10d8cc62017-05-30 09:30:07 -04002188 {
2189 if (nitems >= 2048)
Florin Coras8afe1b82021-11-17 23:38:54 -08002190 smm->configured_wrk_mq_length = nitems;
Dave Barach10d8cc62017-05-30 09:30:07 -04002191 else
2192 clib_warning ("event queue length %d too small, ignored", nitems);
2193 }
Florin Corascf5c7742022-06-28 14:34:45 -07002194 else if (unformat (input, "wrk-mqs-segment-size %U",
2195 unformat_memory_size, &smm->wrk_mqs_segment_size))
2196 ;
Florin Coras66b11312017-07-31 17:18:03 -07002197 else if (unformat (input, "preallocated-sessions %d",
2198 &smm->preallocated_sessions))
Dave Barach2c25a622017-06-26 11:35:07 -04002199 ;
Florin Coras66b11312017-07-31 17:18:03 -07002200 else if (unformat (input, "v4-session-table-buckets %d",
2201 &smm->configured_v4_session_table_buckets))
2202 ;
2203 else if (unformat (input, "v4-halfopen-table-buckets %d",
2204 &smm->configured_v4_halfopen_table_buckets))
2205 ;
2206 else if (unformat (input, "v6-session-table-buckets %d",
2207 &smm->configured_v6_session_table_buckets))
2208 ;
2209 else if (unformat (input, "v6-halfopen-table-buckets %d",
2210 &smm->configured_v6_halfopen_table_buckets))
2211 ;
2212 else if (unformat (input, "v4-session-table-memory %U",
2213 unformat_memory_size, &tmp))
2214 {
2215 if (tmp >= 0x100000000)
2216 return clib_error_return (0, "memory size %llx (%lld) too large",
2217 tmp, tmp);
2218 smm->configured_v4_session_table_memory = tmp;
2219 }
2220 else if (unformat (input, "v4-halfopen-table-memory %U",
2221 unformat_memory_size, &tmp))
2222 {
2223 if (tmp >= 0x100000000)
2224 return clib_error_return (0, "memory size %llx (%lld) too large",
2225 tmp, tmp);
2226 smm->configured_v4_halfopen_table_memory = tmp;
2227 }
2228 else if (unformat (input, "v6-session-table-memory %U",
2229 unformat_memory_size, &tmp))
2230 {
2231 if (tmp >= 0x100000000)
2232 return clib_error_return (0, "memory size %llx (%lld) too large",
2233 tmp, tmp);
2234 smm->configured_v6_session_table_memory = tmp;
2235 }
2236 else if (unformat (input, "v6-halfopen-table-memory %U",
2237 unformat_memory_size, &tmp))
2238 {
2239 if (tmp >= 0x100000000)
2240 return clib_error_return (0, "memory size %llx (%lld) too large",
2241 tmp, tmp);
2242 smm->configured_v6_halfopen_table_memory = tmp;
2243 }
Florin Coras93e65802017-11-29 00:07:11 -05002244 else if (unformat (input, "local-endpoints-table-memory %U",
2245 unformat_memory_size, &tmp))
2246 {
2247 if (tmp >= 0x100000000)
2248 return clib_error_return (0, "memory size %llx (%lld) too large",
2249 tmp, tmp);
2250 smm->local_endpoints_table_memory = tmp;
2251 }
2252 else if (unformat (input, "local-endpoints-table-buckets %d",
2253 &smm->local_endpoints_table_buckets))
2254 ;
Nathan Skrzypczak1292d192019-09-13 15:44:54 +02002255 else if (unformat (input, "enable"))
Nathan Skrzypczak31bd0352019-11-07 17:55:01 +01002256 smm->session_enable_asap = 1;
Florin Coras61ae0562020-09-02 19:10:28 -07002257 else if (unformat (input, "use-app-socket-api"))
Nathan Skrzypczak7b3a3df2021-07-28 14:09:50 +02002258 (void) appns_sapi_enable_disable (1 /* is_enable */);
Florin Corase92c9462020-11-25 08:44:16 -08002259 else if (unformat (input, "poll-main"))
2260 smm->poll_main = 1;
Florin Coras41d5f542021-01-15 13:49:33 -08002261 else if (unformat (input, "use-private-rx-mqs"))
2262 smm->use_private_rx_mqs = 1;
Florin Coras7da88292021-03-18 15:04:34 -07002263 else if (unformat (input, "no-adaptive"))
2264 smm->no_adaptive = 1;
Marvin Liu06542422022-08-16 06:49:09 +00002265 else if (unformat (input, "use-dma"))
2266 smm->dma_enabled = 1;
Florin Coras8afe1b82021-11-17 23:38:54 -08002267 /*
2268 * Deprecated but maintained for compatibility
2269 */
2270 else if (unformat (input, "evt_qs_memfd_seg"))
2271 ;
Florin Corasef048032021-11-17 23:57:30 -08002272 else if (unformat (input, "segment-baseva 0x%lx", &tmp))
2273 ;
Florin Coras8afe1b82021-11-17 23:38:54 -08002274 else if (unformat (input, "evt_qs_seg_size %U", unformat_memory_size,
Florin Corascf5c7742022-06-28 14:34:45 -07002275 &smm->wrk_mqs_segment_size))
Florin Coras8afe1b82021-11-17 23:38:54 -08002276 ;
2277 else if (unformat (input, "event-queue-length %d", &nitems))
2278 {
2279 if (nitems >= 2048)
2280 smm->configured_wrk_mq_length = nitems;
2281 else
2282 clib_warning ("event queue length %d too small, ignored", nitems);
2283 }
Dave Barach10d8cc62017-05-30 09:30:07 -04002284 else
2285 return clib_error_return (0, "unknown input `%U'",
2286 format_unformat_error, input);
2287 }
2288 return 0;
2289}
2290
2291VLIB_CONFIG_FUNCTION (session_config_fn, "session");
2292
Dave Barach68b0fb02017-02-28 15:15:56 -05002293/*
2294 * fd.io coding-style-patch-verification: ON
2295 *
2296 * Local Variables:
2297 * eval: (c-set-style "gnu")
2298 * End:
2299 */