blob: 9d531240f2746761849c52847674a6fc7921d9d8 [file] [log] [blame]
Dave Barach68b0fb02017-02-28 15:15:56 -05001/*
Florin Coras288eaab2019-02-03 15:26:14 -08002 * Copyright (c) 2017-2019 Cisco and/or its affiliates.
Dave Barach68b0fb02017-02-28 15:15:56 -05003 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15/**
16 * @file
17 * @brief Session and session manager
18 */
19
20#include <vnet/session/session.h>
Florin Coras04e53442017-07-16 17:12:15 -070021#include <vnet/session/session_debug.h>
22#include <vnet/session/application.h>
Dave Barach68b0fb02017-02-28 15:15:56 -050023#include <vnet/dpo/load_balance.h>
24#include <vnet/fib/ip4_fib.h>
Dave Barach68b0fb02017-02-28 15:15:56 -050025
Florin Coras31c99552019-03-01 13:00:58 -080026session_main_t session_main;
Florin Coras3eb50622017-07-13 01:24:57 -040027
Florin Coras3c2fed52018-07-04 04:15:05 -070028static inline int
29session_send_evt_to_thread (void *data, void *args, u32 thread_index,
30 session_evt_type_t evt_type)
Florin Coras3cbc04b2017-10-02 00:18:51 -070031{
Florin Coras52207f12018-07-12 14:48:06 -070032 session_event_t *evt;
Florin Coras3c2fed52018-07-04 04:15:05 -070033 svm_msg_q_msg_t msg;
34 svm_msg_q_t *mq;
Florin Coras3cbc04b2017-10-02 00:18:51 -070035
Florin Coras31c99552019-03-01 13:00:58 -080036 mq = session_main_get_vpp_event_queue (thread_index);
Nathan Skrzypczakb4c67752019-06-20 09:58:37 +020037 if (PREDICT_FALSE (svm_msg_q_lock (mq)))
38 return -1;
Nathan Skrzypczak989bc472019-11-19 16:48:58 +010039 if (PREDICT_FALSE (svm_msg_q_is_full (mq)
40 || svm_msg_q_ring_is_full (mq, SESSION_MQ_IO_EVT_RING)))
Florin Coras3c2fed52018-07-04 04:15:05 -070041 {
42 svm_msg_q_unlock (mq);
43 return -2;
44 }
Florin Coras3c2fed52018-07-04 04:15:05 -070045 switch (evt_type)
46 {
Florin Corasf6c43132019-03-01 12:41:21 -080047 case SESSION_CTRL_EVT_RPC:
Nathan Skrzypczak1afa7af2019-09-13 17:14:57 +020048 msg = svm_msg_q_alloc_msg_w_ring (mq, SESSION_MQ_IO_EVT_RING);
49 evt = (session_event_t *) svm_msg_q_msg_data (mq, &msg);
Florin Coras3c2fed52018-07-04 04:15:05 -070050 evt->rpc_args.fp = data;
51 evt->rpc_args.arg = args;
52 break;
Aloys Augustin0c7f54d2019-07-03 16:59:43 +020053 case SESSION_IO_EVT_RX:
Florin Corasf6c43132019-03-01 12:41:21 -080054 case SESSION_IO_EVT_TX:
Florin Coras844a36d2018-12-20 09:50:50 -080055 case SESSION_IO_EVT_TX_FLUSH:
Florin Corasf6c43132019-03-01 12:41:21 -080056 case SESSION_IO_EVT_BUILTIN_RX:
Nathan Skrzypczak1afa7af2019-09-13 17:14:57 +020057 msg = svm_msg_q_alloc_msg_w_ring (mq, SESSION_MQ_IO_EVT_RING);
58 evt = (session_event_t *) svm_msg_q_msg_data (mq, &msg);
Florin Corasc0737e92019-03-04 14:19:39 -080059 evt->session_index = *(u32 *) data;
Florin Coras3c2fed52018-07-04 04:15:05 -070060 break;
Florin Corasf6c43132019-03-01 12:41:21 -080061 case SESSION_IO_EVT_BUILTIN_TX:
62 case SESSION_CTRL_EVT_CLOSE:
Florin Coras73cad332019-08-28 17:12:32 -070063 case SESSION_CTRL_EVT_RESET:
Nathan Skrzypczak1afa7af2019-09-13 17:14:57 +020064 msg = svm_msg_q_alloc_msg_w_ring (mq, SESSION_MQ_IO_EVT_RING);
65 evt = (session_event_t *) svm_msg_q_msg_data (mq, &msg);
Florin Coras288eaab2019-02-03 15:26:14 -080066 evt->session_handle = session_handle ((session_t *) data);
Florin Coras3c2fed52018-07-04 04:15:05 -070067 break;
68 default:
69 clib_warning ("evt unhandled!");
70 svm_msg_q_unlock (mq);
71 return -1;
72 }
Nathan Skrzypczak1afa7af2019-09-13 17:14:57 +020073 evt->event_type = evt_type;
Florin Coras3c2fed52018-07-04 04:15:05 -070074
Florin Coras52207f12018-07-12 14:48:06 -070075 svm_msg_q_add_and_unlock (mq, &msg);
Florin Coras3c2fed52018-07-04 04:15:05 -070076 return 0;
Florin Coras3cbc04b2017-10-02 00:18:51 -070077}
78
Florin Coras3c2fed52018-07-04 04:15:05 -070079int
80session_send_io_evt_to_thread (svm_fifo_t * f, session_evt_type_t evt_type)
Florin Coras3cbc04b2017-10-02 00:18:51 -070081{
Florin Corasc0737e92019-03-04 14:19:39 -080082 return session_send_evt_to_thread (&f->master_session_index, 0,
83 f->master_thread_index, evt_type);
Florin Coras3c2fed52018-07-04 04:15:05 -070084}
85
86int
Florin Corasef915342018-09-29 10:23:06 -070087session_send_io_evt_to_thread_custom (void *data, u32 thread_index,
Florin Coras3c2fed52018-07-04 04:15:05 -070088 session_evt_type_t evt_type)
89{
Florin Corasef915342018-09-29 10:23:06 -070090 return session_send_evt_to_thread (data, 0, thread_index, evt_type);
Florin Coras3c2fed52018-07-04 04:15:05 -070091}
92
93int
Florin Coras288eaab2019-02-03 15:26:14 -080094session_send_ctrl_evt_to_thread (session_t * s, session_evt_type_t evt_type)
Florin Coras3c2fed52018-07-04 04:15:05 -070095{
Florin Coras458089b2019-08-21 16:20:44 -070096 /* only events supported are disconnect and reset */
97 ASSERT (evt_type == SESSION_CTRL_EVT_CLOSE
98 || evt_type == SESSION_CTRL_EVT_RESET);
99 return session_send_evt_to_thread (s, 0, s->thread_index, evt_type);
Florin Coras3cbc04b2017-10-02 00:18:51 -0700100}
101
102void
Nathan Skrzypczak60f3e652019-03-19 13:57:31 +0100103session_send_rpc_evt_to_thread_force (u32 thread_index, void *fp,
104 void *rpc_args)
105{
106 session_send_evt_to_thread (fp, rpc_args, thread_index,
107 SESSION_CTRL_EVT_RPC);
108}
109
110void
Florin Coras3cbc04b2017-10-02 00:18:51 -0700111session_send_rpc_evt_to_thread (u32 thread_index, void *fp, void *rpc_args)
112{
113 if (thread_index != vlib_get_thread_index ())
Nathan Skrzypczak60f3e652019-03-19 13:57:31 +0100114 session_send_rpc_evt_to_thread_force (thread_index, fp, rpc_args);
Florin Coras3cbc04b2017-10-02 00:18:51 -0700115 else
116 {
117 void (*fnp) (void *) = fp;
118 fnp (rpc_args);
119 }
120}
121
Florin Coras26dd6de2019-07-23 23:54:47 -0700122void
123session_add_self_custom_tx_evt (transport_connection_t * tc, u8 has_prio)
124{
125 session_t *s;
126
127 s = session_get (tc->s_index, tc->thread_index);
128 ASSERT (s->thread_index == vlib_get_thread_index ());
Florin Corasbe237bf2019-09-27 08:16:40 -0700129 ASSERT (s->session_state != SESSION_STATE_TRANSPORT_DELETED);
Florin Coras26dd6de2019-07-23 23:54:47 -0700130 if (!(s->flags & SESSION_F_CUSTOM_TX))
131 {
132 s->flags |= SESSION_F_CUSTOM_TX;
Florin Coras6080e0d2020-03-13 20:39:43 +0000133 if (svm_fifo_set_event (s->tx_fifo)
134 || transport_connection_is_descheduled (tc))
Florin Coras26dd6de2019-07-23 23:54:47 -0700135 {
136 session_worker_t *wrk;
137 session_evt_elt_t *elt;
138 wrk = session_main_get_worker (tc->thread_index);
139 if (has_prio)
140 elt = session_evt_alloc_new (wrk);
141 else
142 elt = session_evt_alloc_old (wrk);
143 elt->evt.session_index = tc->s_index;
144 elt->evt.event_type = SESSION_IO_EVT_TX;
Florin Coras6080e0d2020-03-13 20:39:43 +0000145 tc->flags &= ~TRANSPORT_CONNECTION_F_DESCHED;
Florin Coras26dd6de2019-07-23 23:54:47 -0700146 }
147 }
148}
149
Florin Coras70f879d2020-03-13 17:54:42 +0000150void
151sesssion_reschedule_tx (transport_connection_t * tc)
152{
153 session_worker_t *wrk = session_main_get_worker (tc->thread_index);
154 session_evt_elt_t *elt;
155
156 ASSERT (tc->thread_index == vlib_get_thread_index ());
157
158 elt = session_evt_alloc_new (wrk);
159 elt->evt.session_index = tc->s_index;
160 elt->evt.event_type = SESSION_IO_EVT_TX;
161}
162
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800163static void
Florin Corasdfb3b872019-08-16 17:48:44 -0700164session_program_transport_ctrl_evt (session_t * s, session_evt_type_t evt)
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800165{
166 u32 thread_index = vlib_get_thread_index ();
Florin Coras2062ec02019-07-15 13:15:18 -0700167 session_evt_elt_t *elt;
Florin Coras31c99552019-03-01 13:00:58 -0800168 session_worker_t *wrk;
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800169
170 /* If we are in the handler thread, or being called with the worker barrier
171 * held, just append a new event to pending disconnects vector. */
172 if (vlib_thread_is_main_w_barrier () || thread_index == s->thread_index)
173 {
Florin Coras31c99552019-03-01 13:00:58 -0800174 wrk = session_main_get_worker (s->thread_index);
Florin Corasb0ffbee2019-07-21 19:23:46 -0700175 elt = session_evt_alloc_ctrl (wrk);
Florin Coras2062ec02019-07-15 13:15:18 -0700176 clib_memset (&elt->evt, 0, sizeof (session_event_t));
177 elt->evt.session_handle = session_handle (s);
Florin Corasdfb3b872019-08-16 17:48:44 -0700178 elt->evt.event_type = evt;
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800179 }
180 else
Florin Corasdfb3b872019-08-16 17:48:44 -0700181 session_send_ctrl_evt_to_thread (s, evt);
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800182}
183
Florin Coras288eaab2019-02-03 15:26:14 -0800184session_t *
Florin Coras3cbc04b2017-10-02 00:18:51 -0700185session_alloc (u32 thread_index)
Dave Barach68b0fb02017-02-28 15:15:56 -0500186{
Florin Coras31c99552019-03-01 13:00:58 -0800187 session_worker_t *wrk = &session_main.wrk[thread_index];
Florin Coras288eaab2019-02-03 15:26:14 -0800188 session_t *s;
Florin Coras3cbc04b2017-10-02 00:18:51 -0700189 u8 will_expand = 0;
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700190 pool_get_aligned_will_expand (wrk->sessions, will_expand,
Florin Coras3cbc04b2017-10-02 00:18:51 -0700191 CLIB_CACHE_LINE_BYTES);
192 /* If we have peekers, let them finish */
Florin Coras84a30ef2018-01-24 10:49:23 -0800193 if (PREDICT_FALSE (will_expand && vlib_num_workers ()))
Florin Coras3cbc04b2017-10-02 00:18:51 -0700194 {
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700195 clib_rwlock_writer_lock (&wrk->peekers_rw_locks);
196 pool_get_aligned (wrk->sessions, s, CLIB_CACHE_LINE_BYTES);
197 clib_rwlock_writer_unlock (&wrk->peekers_rw_locks);
Florin Coras3cbc04b2017-10-02 00:18:51 -0700198 }
199 else
200 {
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700201 pool_get_aligned (wrk->sessions, s, CLIB_CACHE_LINE_BYTES);
Florin Coras3cbc04b2017-10-02 00:18:51 -0700202 }
Dave Barachb7b92992018-10-17 10:38:51 -0400203 clib_memset (s, 0, sizeof (*s));
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700204 s->session_index = s - wrk->sessions;
Florin Coras3cbc04b2017-10-02 00:18:51 -0700205 s->thread_index = thread_index;
Nathan Skrzypczak2f0f96b2019-06-13 10:14:28 +0200206 s->app_index = APP_INVALID_INDEX;
Florin Coras3cbc04b2017-10-02 00:18:51 -0700207 return s;
208}
209
Florin Coras371ca502018-02-21 12:07:41 -0800210void
Florin Coras288eaab2019-02-03 15:26:14 -0800211session_free (session_t * s)
Florin Coras3cbc04b2017-10-02 00:18:51 -0700212{
Florin Coras3cbc04b2017-10-02 00:18:51 -0700213 if (CLIB_DEBUG)
Florin Coras6416e622019-04-03 17:52:43 -0700214 {
215 u8 thread_index = s->thread_index;
216 clib_memset (s, 0xFA, sizeof (*s));
217 pool_put (session_main.wrk[thread_index].sessions, s);
218 return;
219 }
Florin Corascca96182019-07-19 07:34:13 -0700220 SESSION_EVT (SESSION_EVT_FREE, s);
Florin Coras6416e622019-04-03 17:52:43 -0700221 pool_put (session_main.wrk[s->thread_index].sessions, s);
Florin Coras3cbc04b2017-10-02 00:18:51 -0700222}
223
Srikanth Akulae140d5d2019-11-18 11:49:58 -0800224u8
225session_is_valid (u32 si, u8 thread_index)
226{
227 session_t *s;
228 transport_connection_t *tc;
229
230 s = pool_elt_at_index (session_main.wrk[thread_index].sessions, si);
231
232 if (!s)
233 return 1;
234
235 if (s->thread_index != thread_index || s->session_index != si)
236 return 0;
237
238 if (s->session_state == SESSION_STATE_TRANSPORT_DELETED
239 || s->session_state <= SESSION_STATE_LISTENING)
240 return 1;
241
242 tc = session_get_transport (s);
243 if (s->connection_index != tc->c_index
244 || s->thread_index != tc->thread_index || tc->s_index != si)
245 return 0;
246
247 return 1;
248}
249
Florin Coras70f26d52019-07-08 11:47:18 -0700250static void
251session_cleanup_notify (session_t * s, session_cleanup_ntf_t ntf)
252{
253 app_worker_t *app_wrk;
254
255 app_wrk = app_worker_get_if_valid (s->app_wrk_index);
256 if (!app_wrk)
257 return;
258 app_worker_cleanup_notify (app_wrk, s, ntf);
259}
260
Florin Coraseb97e5f2018-10-15 21:35:42 -0700261void
Florin Coras288eaab2019-02-03 15:26:14 -0800262session_free_w_fifos (session_t * s)
Florin Coras568ebc72018-09-18 16:12:50 -0700263{
Florin Coras70f26d52019-07-08 11:47:18 -0700264 session_cleanup_notify (s, SESSION_CLEANUP_SESSION);
Florin Coras19223e02019-03-03 14:56:05 -0800265 segment_manager_dealloc_fifos (s->rx_fifo, s->tx_fifo);
Florin Coras568ebc72018-09-18 16:12:50 -0700266 session_free (s);
267}
268
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800269/**
270 * Cleans up session and lookup table.
271 *
272 * Transport connection must still be valid.
273 */
274static void
Florin Coras288eaab2019-02-03 15:26:14 -0800275session_delete (session_t * s)
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800276{
277 int rv;
278
279 /* Delete from the main lookup table. */
280 if ((rv = session_lookup_del_session (s)))
Florin Corasd09236d2019-08-08 17:38:26 -0700281 clib_warning ("session %u hash delete rv %d", s->session_index, rv);
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800282
283 session_free_w_fifos (s);
284}
285
Andreas Schultz1a8c4372020-03-20 09:39:59 +0100286session_t *
Florin Coras3cbc04b2017-10-02 00:18:51 -0700287session_alloc_for_connection (transport_connection_t * tc)
288{
Florin Coras288eaab2019-02-03 15:26:14 -0800289 session_t *s;
Florin Coras3cbc04b2017-10-02 00:18:51 -0700290 u32 thread_index = tc->thread_index;
291
Florin Coras40903ac2018-06-10 14:41:23 -0700292 ASSERT (thread_index == vlib_get_thread_index ()
293 || transport_protocol_is_cl (tc->proto));
Dave Barach2c25a622017-06-26 11:35:07 -0400294
Florin Coras3cbc04b2017-10-02 00:18:51 -0700295 s = session_alloc (thread_index);
296 s->session_type = session_type_from_proto_and_ip (tc->proto, tc->is_ip4);
Florin Corasb0f662f2018-12-27 14:51:46 -0800297 s->session_state = SESSION_STATE_CLOSED;
Dave Barach68b0fb02017-02-28 15:15:56 -0500298
Florin Coras3cbc04b2017-10-02 00:18:51 -0700299 /* Attach transport to session and vice versa */
Dave Barach68b0fb02017-02-28 15:15:56 -0500300 s->connection_index = tc->c_index;
Dave Barach68b0fb02017-02-28 15:15:56 -0500301 tc->s_index = s->session_index;
Florin Coras3cbc04b2017-10-02 00:18:51 -0700302 return s;
303}
304
Florin Coras1f152cd2017-08-18 19:28:03 -0700305/**
306 * Discards bytes from buffer chain
307 *
308 * It discards n_bytes_to_drop starting at first buffer after chain_b
309 */
310always_inline void
311session_enqueue_discard_chain_bytes (vlib_main_t * vm, vlib_buffer_t * b,
312 vlib_buffer_t ** chain_b,
313 u32 n_bytes_to_drop)
314{
315 vlib_buffer_t *next = *chain_b;
316 u32 to_drop = n_bytes_to_drop;
317 ASSERT (b->flags & VLIB_BUFFER_NEXT_PRESENT);
318 while (to_drop && (next->flags & VLIB_BUFFER_NEXT_PRESENT))
319 {
320 next = vlib_get_buffer (vm, next->next_buffer);
321 if (next->current_length > to_drop)
322 {
323 vlib_buffer_advance (next, to_drop);
324 to_drop = 0;
325 }
326 else
327 {
328 to_drop -= next->current_length;
329 next->current_length = 0;
330 }
331 }
332 *chain_b = next;
333
334 if (to_drop == 0)
335 b->total_length_not_including_first_buffer -= n_bytes_to_drop;
336}
337
338/**
339 * Enqueue buffer chain tail
340 */
Florin Corasf6d68ed2017-05-07 19:12:02 -0700341always_inline int
Florin Coras288eaab2019-02-03 15:26:14 -0800342session_enqueue_chain_tail (session_t * s, vlib_buffer_t * b,
Florin Corasf6d68ed2017-05-07 19:12:02 -0700343 u32 offset, u8 is_in_order)
344{
345 vlib_buffer_t *chain_b;
Florin Coras1f152cd2017-08-18 19:28:03 -0700346 u32 chain_bi, len, diff;
Florin Corasf6d68ed2017-05-07 19:12:02 -0700347 vlib_main_t *vm = vlib_get_main ();
Florin Corasb2215d62017-08-01 16:56:58 -0700348 u8 *data;
Florin Coras1f152cd2017-08-18 19:28:03 -0700349 u32 written = 0;
Florin Corasf6d68ed2017-05-07 19:12:02 -0700350 int rv = 0;
351
Florin Coras1f152cd2017-08-18 19:28:03 -0700352 if (is_in_order && offset)
353 {
354 diff = offset - b->current_length;
355 if (diff > b->total_length_not_including_first_buffer)
356 return 0;
357 chain_b = b;
358 session_enqueue_discard_chain_bytes (vm, b, &chain_b, diff);
359 chain_bi = vlib_get_buffer_index (vm, chain_b);
360 }
361 else
362 chain_bi = b->next_buffer;
363
Florin Corasf6d68ed2017-05-07 19:12:02 -0700364 do
365 {
366 chain_b = vlib_get_buffer (vm, chain_bi);
367 data = vlib_buffer_get_current (chain_b);
368 len = chain_b->current_length;
Florin Coras1f152cd2017-08-18 19:28:03 -0700369 if (!len)
370 continue;
Florin Corasf6d68ed2017-05-07 19:12:02 -0700371 if (is_in_order)
372 {
Florin Coras87b15ce2019-04-28 21:16:30 -0700373 rv = svm_fifo_enqueue (s->rx_fifo, len, data);
Florin Coras1f152cd2017-08-18 19:28:03 -0700374 if (rv == len)
375 {
376 written += rv;
377 }
378 else if (rv < len)
Florin Corasf6d68ed2017-05-07 19:12:02 -0700379 {
380 return (rv > 0) ? (written + rv) : written;
381 }
Florin Coras1f152cd2017-08-18 19:28:03 -0700382 else if (rv > len)
383 {
384 written += rv;
385
386 /* written more than what was left in chain */
387 if (written > b->total_length_not_including_first_buffer)
388 return written;
389
390 /* drop the bytes that have already been delivered */
391 session_enqueue_discard_chain_bytes (vm, b, &chain_b, rv - len);
392 }
Florin Corasf6d68ed2017-05-07 19:12:02 -0700393 }
394 else
395 {
Florin Coras288eaab2019-02-03 15:26:14 -0800396 rv = svm_fifo_enqueue_with_offset (s->rx_fifo, offset, len, data);
Florin Corasf6d68ed2017-05-07 19:12:02 -0700397 if (rv)
Florin Coras1f152cd2017-08-18 19:28:03 -0700398 {
399 clib_warning ("failed to enqueue multi-buffer seg");
400 return -1;
401 }
Florin Corasf6d68ed2017-05-07 19:12:02 -0700402 offset += len;
403 }
404 }
405 while ((chain_bi = (chain_b->flags & VLIB_BUFFER_NEXT_PRESENT)
406 ? chain_b->next_buffer : 0));
407
408 if (is_in_order)
409 return written;
410
411 return 0;
412}
413
Ryujiro Shibuyad8f48e22020-01-22 12:11:42 +0000414void
415session_fifo_tuning (session_t * s, svm_fifo_t * f,
416 session_ft_action_t act, u32 len)
417{
418 if (s->flags & SESSION_F_CUSTOM_FIFO_TUNING)
419 {
420 app_worker_t *app_wrk = app_worker_get (s->app_wrk_index);
421 app_worker_session_fifo_tuning (app_wrk, s, f, act, len);
422 if (CLIB_ASSERT_ENABLE)
423 {
424 segment_manager_t *sm;
425 sm = segment_manager_get (f->segment_manager);
426 ASSERT (f->size >= 4096);
427 ASSERT (f->size <= sm->max_fifo_size);
428 }
429 }
430}
431
Dave Barach68b0fb02017-02-28 15:15:56 -0500432/*
433 * Enqueue data for delivery to session peer. Does not notify peer of enqueue
434 * event but on request can queue notification events for later delivery by
435 * calling stream_server_flush_enqueue_events().
436 *
437 * @param tc Transport connection which is to be enqueued data
Florin Corasf6d68ed2017-05-07 19:12:02 -0700438 * @param b Buffer to be enqueued
439 * @param offset Offset at which to start enqueueing if out-of-order
Dave Barach68b0fb02017-02-28 15:15:56 -0500440 * @param queue_event Flag to indicate if peer is to be notified or if event
441 * is to be queued. The former is useful when more data is
442 * enqueued and only one event is to be generated.
Florin Corasf6d68ed2017-05-07 19:12:02 -0700443 * @param is_in_order Flag to indicate if data is in order
Dave Barach68b0fb02017-02-28 15:15:56 -0500444 * @return Number of bytes enqueued or a negative value if enqueueing failed.
445 */
446int
Florin Coras3cbc04b2017-10-02 00:18:51 -0700447session_enqueue_stream_connection (transport_connection_t * tc,
448 vlib_buffer_t * b, u32 offset,
449 u8 queue_event, u8 is_in_order)
Dave Barach68b0fb02017-02-28 15:15:56 -0500450{
Florin Coras288eaab2019-02-03 15:26:14 -0800451 session_t *s;
Florin Coras1f152cd2017-08-18 19:28:03 -0700452 int enqueued = 0, rv, in_order_off;
Dave Barach68b0fb02017-02-28 15:15:56 -0500453
Florin Corascea194d2017-10-02 00:18:51 -0700454 s = session_get (tc->s_index, tc->thread_index);
Dave Barach68b0fb02017-02-28 15:15:56 -0500455
Florin Corasf6d68ed2017-05-07 19:12:02 -0700456 if (is_in_order)
457 {
Florin Coras87b15ce2019-04-28 21:16:30 -0700458 enqueued = svm_fifo_enqueue (s->rx_fifo,
459 b->current_length,
460 vlib_buffer_get_current (b));
Florin Coras1f152cd2017-08-18 19:28:03 -0700461 if (PREDICT_FALSE ((b->flags & VLIB_BUFFER_NEXT_PRESENT)
462 && enqueued >= 0))
Florin Corasf6d68ed2017-05-07 19:12:02 -0700463 {
Florin Coras1f152cd2017-08-18 19:28:03 -0700464 in_order_off = enqueued > b->current_length ? enqueued : 0;
465 rv = session_enqueue_chain_tail (s, b, in_order_off, 1);
466 if (rv > 0)
467 enqueued += rv;
Florin Corasf6d68ed2017-05-07 19:12:02 -0700468 }
469 }
470 else
471 {
Florin Coras288eaab2019-02-03 15:26:14 -0800472 rv = svm_fifo_enqueue_with_offset (s->rx_fifo, offset,
Florin Corasf6d68ed2017-05-07 19:12:02 -0700473 b->current_length,
474 vlib_buffer_get_current (b));
475 if (PREDICT_FALSE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) && !rv))
Florin Coras1f152cd2017-08-18 19:28:03 -0700476 session_enqueue_chain_tail (s, b, offset + b->current_length, 0);
477 /* if something was enqueued, report even this as success for ooo
478 * segment handling */
479 return rv;
Florin Corasf6d68ed2017-05-07 19:12:02 -0700480 }
Dave Barach68b0fb02017-02-28 15:15:56 -0500481
482 if (queue_event)
483 {
484 /* Queue RX event on this fifo. Eventually these will need to be flushed
485 * by calling stream_server_flush_enqueue_events () */
Florin Coras31c99552019-03-01 13:00:58 -0800486 session_worker_t *wrk;
Dave Barach68b0fb02017-02-28 15:15:56 -0500487
Florin Coras31c99552019-03-01 13:00:58 -0800488 wrk = session_main_get_worker (s->thread_index);
Florin Corasd5c604d2019-03-18 09:06:35 -0700489 if (!(s->flags & SESSION_F_RX_EVT))
Dave Barach68b0fb02017-02-28 15:15:56 -0500490 {
Florin Corasd5c604d2019-03-18 09:06:35 -0700491 s->flags |= SESSION_F_RX_EVT;
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700492 vec_add1 (wrk->session_to_enqueue[tc->proto], s->session_index);
Dave Barach68b0fb02017-02-28 15:15:56 -0500493 }
Ryujiro Shibuyad8f48e22020-01-22 12:11:42 +0000494
495 session_fifo_tuning (s, s->rx_fifo, SESSION_FT_ACTION_ENQUEUED, 0);
Dave Barach68b0fb02017-02-28 15:15:56 -0500496 }
497
Chris Lukeb2bcad62017-09-18 08:51:22 -0400498 return enqueued;
Dave Barach68b0fb02017-02-28 15:15:56 -0500499}
500
Florin Coras3cbc04b2017-10-02 00:18:51 -0700501int
Florin Coras288eaab2019-02-03 15:26:14 -0800502session_enqueue_dgram_connection (session_t * s,
Florin Coras7fb0fe12018-04-09 09:24:52 -0700503 session_dgram_hdr_t * hdr,
504 vlib_buffer_t * b, u8 proto, u8 queue_event)
Florin Coras3cbc04b2017-10-02 00:18:51 -0700505{
506 int enqueued = 0, rv, in_order_off;
507
Sirshak Das28aa5392019-02-05 01:33:33 -0600508 ASSERT (svm_fifo_max_enqueue_prod (s->rx_fifo)
Florin Coras7fb0fe12018-04-09 09:24:52 -0700509 >= b->current_length + sizeof (*hdr));
510
Florin Coras87b15ce2019-04-28 21:16:30 -0700511 svm_fifo_enqueue (s->rx_fifo, sizeof (session_dgram_hdr_t), (u8 *) hdr);
512 enqueued = svm_fifo_enqueue (s->rx_fifo, b->current_length,
513 vlib_buffer_get_current (b));
Florin Coras3cbc04b2017-10-02 00:18:51 -0700514 if (PREDICT_FALSE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) && enqueued >= 0))
515 {
516 in_order_off = enqueued > b->current_length ? enqueued : 0;
517 rv = session_enqueue_chain_tail (s, b, in_order_off, 1);
518 if (rv > 0)
519 enqueued += rv;
520 }
521 if (queue_event)
522 {
523 /* Queue RX event on this fifo. Eventually these will need to be flushed
524 * by calling stream_server_flush_enqueue_events () */
Florin Coras31c99552019-03-01 13:00:58 -0800525 session_worker_t *wrk;
Florin Coras3cbc04b2017-10-02 00:18:51 -0700526
Florin Coras31c99552019-03-01 13:00:58 -0800527 wrk = session_main_get_worker (s->thread_index);
Florin Corasd5c604d2019-03-18 09:06:35 -0700528 if (!(s->flags & SESSION_F_RX_EVT))
Florin Coras3cbc04b2017-10-02 00:18:51 -0700529 {
Florin Corasd5c604d2019-03-18 09:06:35 -0700530 s->flags |= SESSION_F_RX_EVT;
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700531 vec_add1 (wrk->session_to_enqueue[proto], s->session_index);
Florin Coras3cbc04b2017-10-02 00:18:51 -0700532 }
Ryujiro Shibuyad8f48e22020-01-22 12:11:42 +0000533
534 session_fifo_tuning (s, s->rx_fifo, SESSION_FT_ACTION_ENQUEUED, 0);
Florin Coras3cbc04b2017-10-02 00:18:51 -0700535 }
536 return enqueued;
537}
538
Florin Coras93992a92017-05-24 18:03:56 -0700539int
Florin Coras31c99552019-03-01 13:00:58 -0800540session_tx_fifo_peek_bytes (transport_connection_t * tc, u8 * buffer,
541 u32 offset, u32 max_bytes)
Dave Barach68b0fb02017-02-28 15:15:56 -0500542{
Florin Coras288eaab2019-02-03 15:26:14 -0800543 session_t *s = session_get (tc->s_index, tc->thread_index);
544 return svm_fifo_peek (s->tx_fifo, offset, max_bytes, buffer);
Dave Barach68b0fb02017-02-28 15:15:56 -0500545}
546
547u32
Florin Coras31c99552019-03-01 13:00:58 -0800548session_tx_fifo_dequeue_drop (transport_connection_t * tc, u32 max_bytes)
Dave Barach68b0fb02017-02-28 15:15:56 -0500549{
Florin Coras288eaab2019-02-03 15:26:14 -0800550 session_t *s = session_get (tc->s_index, tc->thread_index);
Vladimir Kropylevf867cf12019-06-17 21:38:00 +0300551 u32 rv;
552
553 rv = svm_fifo_dequeue_drop (s->tx_fifo, max_bytes);
Ryujiro Shibuyad8f48e22020-01-22 12:11:42 +0000554 session_fifo_tuning (s, s->tx_fifo, SESSION_FT_ACTION_DEQUEUED, rv);
Florin Coras0e573f52019-05-07 16:28:16 -0700555
Florin Coras2d379d82019-06-28 12:45:12 -0700556 if (svm_fifo_needs_deq_ntf (s->tx_fifo, max_bytes))
Florin Coras0e573f52019-05-07 16:28:16 -0700557 session_dequeue_notify (s);
558
Vladimir Kropylevf867cf12019-06-17 21:38:00 +0300559 return rv;
Dave Barach68b0fb02017-02-28 15:15:56 -0500560}
561
Florin Coras72b04282019-01-14 17:23:11 -0800562static inline int
Florin Coras288eaab2019-02-03 15:26:14 -0800563session_notify_subscribers (u32 app_index, session_t * s,
Florin Coras72b04282019-01-14 17:23:11 -0800564 svm_fifo_t * f, session_evt_type_t evt_type)
565{
566 app_worker_t *app_wrk;
567 application_t *app;
568 int i;
569
570 app = application_get (app_index);
571 if (!app)
572 return -1;
573
574 for (i = 0; i < f->n_subscribers; i++)
575 {
576 app_wrk = application_get_worker (app, f->subscribers[i]);
577 if (!app_wrk)
578 continue;
579 if (app_worker_lock_and_send_event (app_wrk, s, evt_type))
580 return -1;
581 }
582
583 return 0;
584}
585
Dave Barach68b0fb02017-02-28 15:15:56 -0500586/**
587 * Notify session peer that new data has been enqueued.
588 *
Florin Coras3c2fed52018-07-04 04:15:05 -0700589 * @param s Stream session for which the event is to be generated.
590 * @param lock Flag to indicate if call should lock message queue.
Dave Barach68b0fb02017-02-28 15:15:56 -0500591 *
Florin Coras3c2fed52018-07-04 04:15:05 -0700592 * @return 0 on success or negative number if failed to send notification.
Dave Barach68b0fb02017-02-28 15:15:56 -0500593 */
Florin Coras3c2fed52018-07-04 04:15:05 -0700594static inline int
Florin Coras653e43f2019-03-04 10:56:23 -0800595session_enqueue_notify_inline (session_t * s)
Dave Barach68b0fb02017-02-28 15:15:56 -0500596{
Florin Coras72b04282019-01-14 17:23:11 -0800597 app_worker_t *app_wrk;
Aloys Augustinb3b267c2019-04-09 11:25:56 +0200598 u32 session_index;
599 u8 n_subscribers;
600
601 session_index = s->session_index;
602 n_subscribers = svm_fifo_n_subscribers (s->rx_fifo);
Dave Barach68b0fb02017-02-28 15:15:56 -0500603
Florin Coras72b04282019-01-14 17:23:11 -0800604 app_wrk = app_worker_get_if_valid (s->app_wrk_index);
605 if (PREDICT_FALSE (!app_wrk))
Dave Barach818eb542017-08-02 13:56:13 -0400606 {
Florin Coras15531972018-08-12 23:50:53 -0700607 SESSION_DBG ("invalid s->app_index = %d", s->app_wrk_index);
Dave Barach818eb542017-08-02 13:56:13 -0400608 return 0;
609 }
Dave Barach68b0fb02017-02-28 15:15:56 -0500610
Florin Corascca96182019-07-19 07:34:13 -0700611 SESSION_EVT (SESSION_EVT_ENQ, s, svm_fifo_max_dequeue_prod (s->rx_fifo));
Dave Barach68b0fb02017-02-28 15:15:56 -0500612
Florin Corasd5c604d2019-03-18 09:06:35 -0700613 s->flags &= ~SESSION_F_RX_EVT;
Florin Coras72b04282019-01-14 17:23:11 -0800614 if (PREDICT_FALSE (app_worker_lock_and_send_event (app_wrk, s,
Florin Corasf6c43132019-03-01 12:41:21 -0800615 SESSION_IO_EVT_RX)))
Florin Coras72b04282019-01-14 17:23:11 -0800616 return -1;
617
Aloys Augustinb3b267c2019-04-09 11:25:56 +0200618 if (PREDICT_FALSE (n_subscribers))
619 {
620 s = session_get (session_index, vlib_get_thread_index ());
621 return session_notify_subscribers (app_wrk->app_index, s,
622 s->rx_fifo, SESSION_IO_EVT_RX);
623 }
Florin Coras72b04282019-01-14 17:23:11 -0800624
625 return 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500626}
627
Florin Coras0d60a0f2018-06-29 02:02:08 -0700628int
Florin Coras653e43f2019-03-04 10:56:23 -0800629session_enqueue_notify (session_t * s)
630{
631 return session_enqueue_notify_inline (s);
632}
633
Aloys Augustinb3392332019-08-06 16:09:01 +0200634static void
635session_enqueue_notify_rpc (void *arg)
636{
Florin Coras5d8a8062019-08-13 08:35:39 -0700637 u32 session_index = pointer_to_uword (arg);
Aloys Augustinb3392332019-08-06 16:09:01 +0200638 session_t *s;
639
Florin Coras5d8a8062019-08-13 08:35:39 -0700640 s = session_get_if_valid (session_index, vlib_get_thread_index ());
Aloys Augustinb3392332019-08-06 16:09:01 +0200641 if (!s)
642 return;
643
644 session_enqueue_notify (s);
645}
646
647/**
648 * Like session_enqueue_notify, but can be called from a thread that does not
649 * own the session.
650 */
651void
652session_enqueue_notify_thread (session_handle_t sh)
653{
654 u32 thread_index = session_thread_from_handle (sh);
Florin Coras5d8a8062019-08-13 08:35:39 -0700655 u32 session_index = session_index_from_handle (sh);
656
657 /*
658 * Pass session index (u32) as opposed to handle (u64) in case pointers
659 * are not 64-bit.
660 */
Aloys Augustinb3392332019-08-06 16:09:01 +0200661 session_send_rpc_evt_to_thread (thread_index,
Florin Coras5d8a8062019-08-13 08:35:39 -0700662 session_enqueue_notify_rpc,
663 uword_to_pointer (session_index, void *));
Aloys Augustinb3392332019-08-06 16:09:01 +0200664}
665
Florin Coras653e43f2019-03-04 10:56:23 -0800666int
Florin Coras288eaab2019-02-03 15:26:14 -0800667session_dequeue_notify (session_t * s)
Florin Coras0d60a0f2018-06-29 02:02:08 -0700668{
Florin Coras72b04282019-01-14 17:23:11 -0800669 app_worker_t *app_wrk;
Florin Coras0d60a0f2018-06-29 02:02:08 -0700670
Vladimir Kropylev5c89fbf2019-08-30 13:04:06 +0300671 svm_fifo_clear_deq_ntf (s->tx_fifo);
672
Florin Coras72b04282019-01-14 17:23:11 -0800673 app_wrk = app_worker_get_if_valid (s->app_wrk_index);
674 if (PREDICT_FALSE (!app_wrk))
Florin Coras208daa12018-07-02 01:30:51 -0700675 return -1;
676
Florin Coras72b04282019-01-14 17:23:11 -0800677 if (PREDICT_FALSE (app_worker_lock_and_send_event (app_wrk, s,
Florin Corasf6c43132019-03-01 12:41:21 -0800678 SESSION_IO_EVT_TX)))
Florin Coras1bcad5c2019-01-09 20:04:38 -0800679 return -1;
680
Florin Coras288eaab2019-02-03 15:26:14 -0800681 if (PREDICT_FALSE (s->tx_fifo->n_subscribers))
Florin Coras72b04282019-01-14 17:23:11 -0800682 return session_notify_subscribers (app_wrk->app_index, s,
Florin Corasf6c43132019-03-01 12:41:21 -0800683 s->tx_fifo, SESSION_IO_EVT_TX);
Florin Coras72b04282019-01-14 17:23:11 -0800684
Florin Coras1bcad5c2019-01-09 20:04:38 -0800685 return 0;
Florin Coras0d60a0f2018-06-29 02:02:08 -0700686}
687
Dave Barach68b0fb02017-02-28 15:15:56 -0500688/**
689 * Flushes queue of sessions that are to be notified of new data
690 * enqueued events.
691 *
692 * @param thread_index Thread index for which the flush is to be performed.
693 * @return 0 on success or a positive number indicating the number of
694 * failures due to API queue being full.
695 */
696int
Florin Coras31c99552019-03-01 13:00:58 -0800697session_main_flush_enqueue_events (u8 transport_proto, u32 thread_index)
Dave Barach68b0fb02017-02-28 15:15:56 -0500698{
Florin Coras31c99552019-03-01 13:00:58 -0800699 session_worker_t *wrk = session_main_get_worker (thread_index);
Florin Coras288eaab2019-02-03 15:26:14 -0800700 session_t *s;
Florin Coras21795132018-09-09 09:40:51 -0700701 int i, errors = 0;
Florin Coras3c2fed52018-07-04 04:15:05 -0700702 u32 *indices;
Dave Barach68b0fb02017-02-28 15:15:56 -0500703
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700704 indices = wrk->session_to_enqueue[transport_proto];
Dave Barach68b0fb02017-02-28 15:15:56 -0500705
Florin Coras3cbc04b2017-10-02 00:18:51 -0700706 for (i = 0; i < vec_len (indices); i++)
Dave Barach68b0fb02017-02-28 15:15:56 -0500707 {
Florin Coras3cbc04b2017-10-02 00:18:51 -0700708 s = session_get_if_valid (indices[i], thread_index);
Florin Coras3c2fed52018-07-04 04:15:05 -0700709 if (PREDICT_FALSE (!s))
710 {
711 errors++;
712 continue;
713 }
Florin Coras653e43f2019-03-04 10:56:23 -0800714
Ryujiro Shibuyad8f48e22020-01-22 12:11:42 +0000715 session_fifo_tuning (s, s->rx_fifo, SESSION_FT_ACTION_ENQUEUED,
716 0 /* TODO/not needed */ );
717
Florin Coras653e43f2019-03-04 10:56:23 -0800718 if (PREDICT_FALSE (session_enqueue_notify_inline (s)))
Florin Coras3cbc04b2017-10-02 00:18:51 -0700719 errors++;
Dave Barach68b0fb02017-02-28 15:15:56 -0500720 }
721
Florin Coras3cbc04b2017-10-02 00:18:51 -0700722 vec_reset_length (indices);
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700723 wrk->session_to_enqueue[transport_proto] = indices;
Dave Barach68b0fb02017-02-28 15:15:56 -0500724
725 return errors;
726}
727
Florin Coras7fb0fe12018-04-09 09:24:52 -0700728int
Florin Coras31c99552019-03-01 13:00:58 -0800729session_main_flush_all_enqueue_events (u8 transport_proto)
Florin Coras7fb0fe12018-04-09 09:24:52 -0700730{
731 vlib_thread_main_t *vtm = vlib_get_thread_main ();
732 int i, errors = 0;
733 for (i = 0; i < 1 + vtm->n_threads; i++)
Florin Coras31c99552019-03-01 13:00:58 -0800734 errors += session_main_flush_enqueue_events (transport_proto, i);
Florin Coras7fb0fe12018-04-09 09:24:52 -0700735 return errors;
736}
737
Nathan Skrzypczaka26349d2019-06-26 13:53:08 +0200738static inline int
Florin Coras00e01d32019-10-21 16:07:46 -0700739session_stream_connect_notify_inline (transport_connection_t * tc,
740 session_error_t err,
Nathan Skrzypczaka26349d2019-06-26 13:53:08 +0200741 session_state_t opened_state)
Dave Barach68b0fb02017-02-28 15:15:56 -0500742{
Florin Coras371ca502018-02-21 12:07:41 -0800743 u32 opaque = 0, new_ti, new_si;
Florin Coras15531972018-08-12 23:50:53 -0700744 app_worker_t *app_wrk;
Florin Corasa27a46e2019-02-18 13:02:28 -0800745 session_t *s = 0;
746 u64 ho_handle;
Dave Barach68b0fb02017-02-28 15:15:56 -0500747
Florin Coras3cbc04b2017-10-02 00:18:51 -0700748 /*
749 * Find connection handle and cleanup half-open table
750 */
Florin Corasa27a46e2019-02-18 13:02:28 -0800751 ho_handle = session_lookup_half_open_handle (tc);
752 if (ho_handle == HALF_OPEN_LOOKUP_INVALID_VALUE)
Dave Barach68b0fb02017-02-28 15:15:56 -0500753 {
Florin Corascea194d2017-10-02 00:18:51 -0700754 SESSION_DBG ("half-open was removed!");
Florin Corasf03a59a2017-06-09 21:07:32 -0700755 return -1;
Dave Barach68b0fb02017-02-28 15:15:56 -0500756 }
Florin Corascea194d2017-10-02 00:18:51 -0700757 session_lookup_del_half_open (tc);
Florin Coras4eeeaaf2017-09-05 14:03:37 -0400758
Florin Corasab0289a2017-08-14 11:25:25 -0700759 /* Get the app's index from the handle we stored when opening connection
760 * and the opaque (api_context for external apps) from transport session
Florin Coras4eeeaaf2017-09-05 14:03:37 -0400761 * index */
Florin Corasa27a46e2019-02-18 13:02:28 -0800762 app_wrk = app_worker_get_if_valid (ho_handle >> 32);
Florin Coras15531972018-08-12 23:50:53 -0700763 if (!app_wrk)
Florin Corasc87c91d2017-08-16 19:55:49 -0700764 return -1;
Florin Corasa27a46e2019-02-18 13:02:28 -0800765
Florin Corasab0289a2017-08-14 11:25:25 -0700766 opaque = tc->s_index;
Dave Barach68b0fb02017-02-28 15:15:56 -0500767
Florin Coras00e01d32019-10-21 16:07:46 -0700768 if (err)
769 return app_worker_connect_notify (app_wrk, s, err, opaque);
Florin Corasa27a46e2019-02-18 13:02:28 -0800770
771 s = session_alloc_for_connection (tc);
772 s->session_state = SESSION_STATE_CONNECTING;
773 s->app_wrk_index = app_wrk->wrk_index;
774 new_si = s->session_index;
775 new_ti = s->thread_index;
776
Florin Coras00e01d32019-10-21 16:07:46 -0700777 if ((err = app_worker_init_connected (app_wrk, s)))
Dave Barach68b0fb02017-02-28 15:15:56 -0500778 {
Florin Corasa27a46e2019-02-18 13:02:28 -0800779 session_free (s);
Florin Coras00e01d32019-10-21 16:07:46 -0700780 app_worker_connect_notify (app_wrk, 0, err, opaque);
Florin Corasa27a46e2019-02-18 13:02:28 -0800781 return -1;
Dave Barach68b0fb02017-02-28 15:15:56 -0500782 }
783
Nathan Skrzypczaka26349d2019-06-26 13:53:08 +0200784 s = session_get (new_si, new_ti);
785 s->session_state = opened_state;
786 session_lookup_add_connection (tc, session_handle (s));
787
Florin Coras00e01d32019-10-21 16:07:46 -0700788 if (app_worker_connect_notify (app_wrk, s, SESSION_E_NONE, opaque))
Florin Coras6534b7a2017-07-18 05:38:03 -0400789 {
Florin Corasa27a46e2019-02-18 13:02:28 -0800790 s = session_get (new_si, new_ti);
791 session_free_w_fifos (s);
792 return -1;
Florin Coras6534b7a2017-07-18 05:38:03 -0400793 }
Dave Barach68b0fb02017-02-28 15:15:56 -0500794
Florin Corasa27a46e2019-02-18 13:02:28 -0800795 return 0;
Dave Barach68b0fb02017-02-28 15:15:56 -0500796}
797
Nathan Skrzypczaka26349d2019-06-26 13:53:08 +0200798int
Florin Coras00e01d32019-10-21 16:07:46 -0700799session_stream_connect_notify (transport_connection_t * tc,
800 session_error_t err)
Nathan Skrzypczaka26349d2019-06-26 13:53:08 +0200801{
Florin Coras00e01d32019-10-21 16:07:46 -0700802 return session_stream_connect_notify_inline (tc, err, SESSION_STATE_READY);
Nathan Skrzypczaka26349d2019-06-26 13:53:08 +0200803}
804
805int
Florin Coras00e01d32019-10-21 16:07:46 -0700806session_ho_stream_connect_notify (transport_connection_t * tc,
807 session_error_t err)
Nathan Skrzypczaka26349d2019-06-26 13:53:08 +0200808{
Florin Coras00e01d32019-10-21 16:07:46 -0700809 return session_stream_connect_notify_inline (tc, err, SESSION_STATE_OPENED);
Nathan Skrzypczaka26349d2019-06-26 13:53:08 +0200810}
811
Florin Coras3cbc04b2017-10-02 00:18:51 -0700812typedef struct _session_switch_pool_args
813{
814 u32 session_index;
815 u32 thread_index;
816 u32 new_thread_index;
817 u32 new_session_index;
818} session_switch_pool_args_t;
819
Florin Coras49568af2019-07-31 16:46:24 -0700820/**
821 * Notify old thread of the session pool switch
822 */
Florin Coras3cbc04b2017-10-02 00:18:51 -0700823static void
824session_switch_pool (void *cb_args)
825{
826 session_switch_pool_args_t *args = (session_switch_pool_args_t *) cb_args;
Florin Coras49568af2019-07-31 16:46:24 -0700827 app_worker_t *app_wrk;
Florin Coras288eaab2019-02-03 15:26:14 -0800828 session_t *s;
Florin Coras49568af2019-07-31 16:46:24 -0700829
Florin Coras3cbc04b2017-10-02 00:18:51 -0700830 ASSERT (args->thread_index == vlib_get_thread_index ());
831 s = session_get (args->session_index, args->thread_index);
Florin Coras288eaab2019-02-03 15:26:14 -0800832 s->tx_fifo->master_session_index = args->new_session_index;
833 s->tx_fifo->master_thread_index = args->new_thread_index;
Florin Coras1ee78302019-02-05 15:51:15 -0800834 transport_cleanup (session_get_transport_proto (s), s->connection_index,
835 s->thread_index);
Florin Coras49568af2019-07-31 16:46:24 -0700836
837 app_wrk = app_worker_get_if_valid (s->app_wrk_index);
838 if (app_wrk)
839 {
840 session_handle_t new_sh;
841 new_sh = session_make_handle (args->new_session_index,
842 args->new_thread_index);
843 app_worker_migrate_notify (app_wrk, s, new_sh);
Aloys Augustinb3392332019-08-06 16:09:01 +0200844
845 /* Trigger app read on the new thread */
846 session_enqueue_notify_thread (new_sh);
Florin Coras49568af2019-07-31 16:46:24 -0700847 }
848
Florin Coras3cbc04b2017-10-02 00:18:51 -0700849 session_free (s);
850 clib_mem_free (cb_args);
851}
852
853/**
854 * Move dgram session to the right thread
855 */
856int
857session_dgram_connect_notify (transport_connection_t * tc,
Florin Coras288eaab2019-02-03 15:26:14 -0800858 u32 old_thread_index, session_t ** new_session)
Florin Coras3cbc04b2017-10-02 00:18:51 -0700859{
Florin Coras288eaab2019-02-03 15:26:14 -0800860 session_t *new_s;
Florin Coras3cbc04b2017-10-02 00:18:51 -0700861 session_switch_pool_args_t *rpc_args;
862
863 /*
864 * Clone half-open session to the right thread.
865 */
866 new_s = session_clone_safe (tc->s_index, old_thread_index);
867 new_s->connection_index = tc->c_index;
Florin Coras288eaab2019-02-03 15:26:14 -0800868 new_s->rx_fifo->master_session_index = new_s->session_index;
869 new_s->rx_fifo->master_thread_index = new_s->thread_index;
Florin Coras3cbc04b2017-10-02 00:18:51 -0700870 new_s->session_state = SESSION_STATE_READY;
Nathan Skrzypczakcaa7acf2019-10-02 10:02:05 +0200871 new_s->flags |= SESSION_F_IS_MIGRATING;
Florin Coras3cbc04b2017-10-02 00:18:51 -0700872 session_lookup_add_connection (tc, session_handle (new_s));
873
874 /*
875 * Ask thread owning the old session to clean it up and make us the tx
876 * fifo owner
877 */
878 rpc_args = clib_mem_alloc (sizeof (*rpc_args));
879 rpc_args->new_session_index = new_s->session_index;
880 rpc_args->new_thread_index = new_s->thread_index;
881 rpc_args->session_index = tc->s_index;
882 rpc_args->thread_index = old_thread_index;
883 session_send_rpc_evt_to_thread (rpc_args->thread_index, session_switch_pool,
884 rpc_args);
885
886 tc->s_index = new_s->session_index;
887 new_s->connection_index = tc->c_index;
888 *new_session = new_s;
889 return 0;
890}
891
Dave Barach68b0fb02017-02-28 15:15:56 -0500892/**
893 * Notification from transport that connection is being closed.
894 *
895 * A disconnect is sent to application but state is not removed. Once
896 * disconnect is acknowledged by application, session disconnect is called.
897 * Ultimately this leads to close being called on transport (passive close).
898 */
899void
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800900session_transport_closing_notify (transport_connection_t * tc)
Dave Barach68b0fb02017-02-28 15:15:56 -0500901{
Florin Coras15531972018-08-12 23:50:53 -0700902 app_worker_t *app_wrk;
Florin Coras288eaab2019-02-03 15:26:14 -0800903 session_t *s;
Dave Barach68b0fb02017-02-28 15:15:56 -0500904
Florin Corascea194d2017-10-02 00:18:51 -0700905 s = session_get (tc->s_index, tc->thread_index);
Florin Coras400ded32018-10-03 01:00:57 -0700906 if (s->session_state >= SESSION_STATE_TRANSPORT_CLOSING)
907 return;
Florin Coras568ebc72018-09-18 16:12:50 -0700908 s->session_state = SESSION_STATE_TRANSPORT_CLOSING;
Florin Corasbf7ce2c2019-03-06 14:44:42 -0800909 app_wrk = app_worker_get (s->app_wrk_index);
910 app_worker_close_notify (app_wrk, s);
Dave Barach68b0fb02017-02-28 15:15:56 -0500911}
912
913/**
Dave Barach68b0fb02017-02-28 15:15:56 -0500914 * Notification from transport that connection is being deleted
915 *
Florin Coras4eeeaaf2017-09-05 14:03:37 -0400916 * This removes the session if it is still valid. It should be called only on
917 * previously fully established sessions. For instance failed connects should
918 * call stream_session_connect_notify and indicate that the connect has
919 * failed.
Dave Barach68b0fb02017-02-28 15:15:56 -0500920 */
921void
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800922session_transport_delete_notify (transport_connection_t * tc)
Dave Barach68b0fb02017-02-28 15:15:56 -0500923{
Florin Coras288eaab2019-02-03 15:26:14 -0800924 session_t *s;
Dave Barach68b0fb02017-02-28 15:15:56 -0500925
Florin Corase04c2992017-03-01 08:17:34 -0800926 /* App might've been removed already */
Florin Coras568ebc72018-09-18 16:12:50 -0700927 if (!(s = session_get_if_valid (tc->s_index, tc->thread_index)))
Florin Corasc87c91d2017-08-16 19:55:49 -0700928 return;
Florin Coras568ebc72018-09-18 16:12:50 -0700929
Florin Coras568ebc72018-09-18 16:12:50 -0700930 switch (s->session_state)
931 {
Florin Coras222e1f412019-02-16 20:47:32 -0800932 case SESSION_STATE_CREATED:
933 /* Session was created but accept notification was not yet sent to the
934 * app. Cleanup everything. */
935 session_lookup_del_session (s);
Zeyu Zhangcbbc4a22019-10-12 14:21:59 +0800936 segment_manager_dealloc_fifos (s->rx_fifo, s->tx_fifo);
937 session_free (s);
Florin Coras222e1f412019-02-16 20:47:32 -0800938 break;
Florin Corasb0f662f2018-12-27 14:51:46 -0800939 case SESSION_STATE_ACCEPTING:
Florin Coras568ebc72018-09-18 16:12:50 -0700940 case SESSION_STATE_TRANSPORT_CLOSING:
Florin Coras692b9492019-07-12 15:01:53 -0700941 case SESSION_STATE_CLOSING:
Florin Coras5f066322019-07-22 19:03:03 -0700942 case SESSION_STATE_TRANSPORT_CLOSED:
Florin Coras568ebc72018-09-18 16:12:50 -0700943 /* If transport finishes or times out before we get a reply
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800944 * from the app, mark transport as closed and wait for reply
945 * before removing the session. Cleanup session table in advance
Florin Corasa9d5bea2018-12-17 08:24:19 -0800946 * because transport will soon be closed and closed sessions
947 * are assumed to have been removed from the lookup table */
948 session_lookup_del_session (s);
Florin Coras5f066322019-07-22 19:03:03 -0700949 s->session_state = SESSION_STATE_TRANSPORT_DELETED;
Florin Coras70f26d52019-07-08 11:47:18 -0700950 session_cleanup_notify (s, SESSION_CLEANUP_TRANSPORT);
951 svm_fifo_dequeue_drop_all (s->tx_fifo);
Florin Coras568ebc72018-09-18 16:12:50 -0700952 break;
Florin Coras692b9492019-07-12 15:01:53 -0700953 case SESSION_STATE_APP_CLOSED:
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800954 /* Cleanup lookup table as transport needs to still be valid.
955 * Program transport close to ensure that all session events
956 * have been cleaned up. Once transport close is called, the
957 * session is just removed because both transport and app have
958 * confirmed the close*/
Florin Coras568ebc72018-09-18 16:12:50 -0700959 session_lookup_del_session (s);
Florin Coras3b5e2222019-10-25 16:23:39 -0700960 s->session_state = SESSION_STATE_TRANSPORT_DELETED;
Florin Coras70f26d52019-07-08 11:47:18 -0700961 session_cleanup_notify (s, SESSION_CLEANUP_TRANSPORT);
962 svm_fifo_dequeue_drop_all (s->tx_fifo);
Florin Corasdfb3b872019-08-16 17:48:44 -0700963 session_program_transport_ctrl_evt (s, SESSION_CTRL_EVT_CLOSE);
Florin Coras568ebc72018-09-18 16:12:50 -0700964 break;
Florin Coras5f066322019-07-22 19:03:03 -0700965 case SESSION_STATE_TRANSPORT_DELETED:
Florin Corasb0f662f2018-12-27 14:51:46 -0800966 break;
Florin Coras568ebc72018-09-18 16:12:50 -0700967 case SESSION_STATE_CLOSED:
Florin Coras70f26d52019-07-08 11:47:18 -0700968 session_cleanup_notify (s, SESSION_CLEANUP_TRANSPORT);
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800969 session_delete (s);
Florin Coras568ebc72018-09-18 16:12:50 -0700970 break;
Florin Corasc01d5782018-10-17 14:53:11 -0700971 default:
Florin Corasb0f662f2018-12-27 14:51:46 -0800972 clib_warning ("session state %u", s->session_state);
Florin Coras70f26d52019-07-08 11:47:18 -0700973 session_cleanup_notify (s, SESSION_CLEANUP_TRANSPORT);
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800974 session_delete (s);
Florin Corasc01d5782018-10-17 14:53:11 -0700975 break;
Florin Coras568ebc72018-09-18 16:12:50 -0700976 }
Dave Barach68b0fb02017-02-28 15:15:56 -0500977}
978
979/**
Florin Coras692b9492019-07-12 15:01:53 -0700980 * Notification from transport that it is closed
Florin Coras54ddf432018-12-21 13:54:09 -0800981 *
Florin Coras692b9492019-07-12 15:01:53 -0700982 * Should be called by transport, prior to calling delete notify, once it
983 * knows that no more data will be exchanged. This could serve as an
984 * early acknowledgment of an active close especially if transport delete
985 * can be delayed a long time, e.g., tcp time-wait.
Florin Coras54ddf432018-12-21 13:54:09 -0800986 */
987void
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800988session_transport_closed_notify (transport_connection_t * tc)
Florin Coras54ddf432018-12-21 13:54:09 -0800989{
Florin Coras692b9492019-07-12 15:01:53 -0700990 app_worker_t *app_wrk;
Florin Coras288eaab2019-02-03 15:26:14 -0800991 session_t *s;
Florin Coras54ddf432018-12-21 13:54:09 -0800992
993 if (!(s = session_get_if_valid (tc->s_index, tc->thread_index)))
994 return;
Florin Corasb0f662f2018-12-27 14:51:46 -0800995
Florin Coras06a2eec2019-05-23 22:28:16 -0700996 /* Transport thinks that app requested close but it actually didn't.
997 * Can happen for tcp if fin and rst are received in close succession. */
998 if (s->session_state == SESSION_STATE_READY)
999 {
1000 session_transport_closing_notify (tc);
1001 svm_fifo_dequeue_drop_all (s->tx_fifo);
Florin Coras692b9492019-07-12 15:01:53 -07001002 s->session_state = SESSION_STATE_TRANSPORT_CLOSED;
Florin Coras06a2eec2019-05-23 22:28:16 -07001003 }
Florin Corasb0f662f2018-12-27 14:51:46 -08001004 /* If app close has not been received or has not yet resulted in
1005 * a transport close, only mark the session transport as closed */
Florin Coras06a2eec2019-05-23 22:28:16 -07001006 else if (s->session_state <= SESSION_STATE_CLOSING)
Florin Corasb0f662f2018-12-27 14:51:46 -08001007 {
Florin Corasb0f662f2018-12-27 14:51:46 -08001008 s->session_state = SESSION_STATE_TRANSPORT_CLOSED;
1009 }
Florin Coras5f066322019-07-22 19:03:03 -07001010 /* If app also closed, switch to closed */
1011 else if (s->session_state == SESSION_STATE_APP_CLOSED)
Florin Corasb0f662f2018-12-27 14:51:46 -08001012 s->session_state = SESSION_STATE_CLOSED;
Florin Coras692b9492019-07-12 15:01:53 -07001013
1014 app_wrk = app_worker_get_if_valid (s->app_wrk_index);
1015 if (app_wrk)
1016 app_worker_transport_closed_notify (app_wrk, s);
Florin Coras54ddf432018-12-21 13:54:09 -08001017}
1018
1019/**
Dave Barach68b0fb02017-02-28 15:15:56 -05001020 * Notify application that connection has been reset.
1021 */
1022void
Florin Coras5a2ec8f2018-12-27 11:53:11 -08001023session_transport_reset_notify (transport_connection_t * tc)
Dave Barach68b0fb02017-02-28 15:15:56 -05001024{
Florin Coras15531972018-08-12 23:50:53 -07001025 app_worker_t *app_wrk;
Florin Coras69b68ef2019-04-02 11:38:51 -07001026 session_t *s;
1027
Florin Corascea194d2017-10-02 00:18:51 -07001028 s = session_get (tc->s_index, tc->thread_index);
Florin Coras288eaab2019-02-03 15:26:14 -08001029 svm_fifo_dequeue_drop_all (s->tx_fifo);
Florin Coras3c7d4f92018-12-14 11:28:43 -08001030 if (s->session_state >= SESSION_STATE_TRANSPORT_CLOSING)
1031 return;
1032 s->session_state = SESSION_STATE_TRANSPORT_CLOSING;
Florin Coras15531972018-08-12 23:50:53 -07001033 app_wrk = app_worker_get (s->app_wrk_index);
Florin Coras69b68ef2019-04-02 11:38:51 -07001034 app_worker_reset_notify (app_wrk, s);
Dave Barach68b0fb02017-02-28 15:15:56 -05001035}
1036
Florin Corasa27a46e2019-02-18 13:02:28 -08001037int
1038session_stream_accept_notify (transport_connection_t * tc)
1039{
1040 app_worker_t *app_wrk;
1041 session_t *s;
1042
1043 s = session_get (tc->s_index, tc->thread_index);
1044 app_wrk = app_worker_get_if_valid (s->app_wrk_index);
1045 if (!app_wrk)
1046 return -1;
1047 s->session_state = SESSION_STATE_ACCEPTING;
Florin Coras4dc10a42020-02-11 05:31:49 +00001048 if (app_worker_accept_notify (app_wrk, s))
1049 {
1050 /* On transport delete, no notifications should be sent. Unless, the
1051 * accept is retried and successful. */
1052 s->session_state = SESSION_STATE_CREATED;
1053 return -1;
1054 }
1055 return 0;
Florin Corasa27a46e2019-02-18 13:02:28 -08001056}
1057
Dave Barach68b0fb02017-02-28 15:15:56 -05001058/**
1059 * Accept a stream session. Optionally ping the server by callback.
1060 */
1061int
Florin Corasc9940fc2019-02-05 20:55:11 -08001062session_stream_accept (transport_connection_t * tc, u32 listener_index,
Nathan Skrzypczak2f0f96b2019-06-13 10:14:28 +02001063 u32 thread_index, u8 notify)
Dave Barach68b0fb02017-02-28 15:15:56 -05001064{
Florin Corasa27a46e2019-02-18 13:02:28 -08001065 session_t *s;
Dave Barach68b0fb02017-02-28 15:15:56 -05001066 int rv;
1067
Florin Corasa27a46e2019-02-18 13:02:28 -08001068 s = session_alloc_for_connection (tc);
Nathan Skrzypczak2f0f96b2019-06-13 10:14:28 +02001069 s->listener_handle = ((u64) thread_index << 32) | (u64) listener_index;
Florin Coras222e1f412019-02-16 20:47:32 -08001070 s->session_state = SESSION_STATE_CREATED;
Dave Barach68b0fb02017-02-28 15:15:56 -05001071
Florin Corasa27a46e2019-02-18 13:02:28 -08001072 if ((rv = app_worker_init_accepted (s)))
1073 return rv;
1074
1075 session_lookup_add_connection (tc, session_handle (s));
1076
Dave Barach68b0fb02017-02-28 15:15:56 -05001077 /* Shoulder-tap the server */
1078 if (notify)
1079 {
Florin Corasa27a46e2019-02-18 13:02:28 -08001080 app_worker_t *app_wrk = app_worker_get (s->app_wrk_index);
1081 return app_worker_accept_notify (app_wrk, s);
Dave Barach68b0fb02017-02-28 15:15:56 -05001082 }
1083
1084 return 0;
1085}
1086
Florin Coras371ca502018-02-21 12:07:41 -08001087int
Florin Corase759bb52020-04-08 01:55:39 +00001088session_dgram_accept (transport_connection_t * tc, u32 listener_index,
1089 u32 thread_index)
1090{
1091 app_worker_t *app_wrk;
1092 session_t *s;
1093 int rv;
1094
1095 s = session_alloc_for_connection (tc);
1096 s->listener_handle = ((u64) thread_index << 32) | (u64) listener_index;
1097
1098 if ((rv = app_worker_init_accepted (s)))
1099 {
1100 session_free (s);
1101 return rv;
1102 }
1103
1104 app_wrk = app_worker_get (s->app_wrk_index);
1105 if ((rv = app_worker_accept_notify (app_wrk, s)))
1106 {
1107 session_free_w_fifos (s);
1108 return rv;
1109 }
1110
1111 s->session_state = SESSION_STATE_READY;
1112 session_lookup_add_connection (tc, session_handle (s));
1113
1114 return 0;
1115}
1116
1117int
Florin Coras15531972018-08-12 23:50:53 -07001118session_open_cl (u32 app_wrk_index, session_endpoint_t * rmt, u32 opaque)
Florin Coras371ca502018-02-21 12:07:41 -08001119{
1120 transport_connection_t *tc;
Florin Coras5665ced2018-10-25 18:03:45 -07001121 transport_endpoint_cfg_t *tep;
Florin Coras15531972018-08-12 23:50:53 -07001122 app_worker_t *app_wrk;
Aloys Augustin20ab31e2019-03-25 11:29:17 +01001123 session_handle_t sh;
Florin Coras288eaab2019-02-03 15:26:14 -08001124 session_t *s;
Florin Coras371ca502018-02-21 12:07:41 -08001125 int rv;
1126
Florin Coras5665ced2018-10-25 18:03:45 -07001127 tep = session_endpoint_to_transport_cfg (rmt);
Florin Coras1ee78302019-02-05 15:51:15 -08001128 rv = transport_connect (rmt->transport_proto, tep);
Florin Coras371ca502018-02-21 12:07:41 -08001129 if (rv < 0)
1130 {
1131 SESSION_DBG ("Transport failed to open connection.");
Florin Coras57660d92020-04-04 22:45:34 +00001132 return rv;
Florin Coras371ca502018-02-21 12:07:41 -08001133 }
1134
Florin Coras1ee78302019-02-05 15:51:15 -08001135 tc = transport_get_half_open (rmt->transport_proto, (u32) rv);
Florin Coras371ca502018-02-21 12:07:41 -08001136
Florin Corasa27a46e2019-02-18 13:02:28 -08001137 /* For dgram type of service, allocate session and fifos now */
Florin Coras15531972018-08-12 23:50:53 -07001138 app_wrk = app_worker_get (app_wrk_index);
Florin Corasa27a46e2019-02-18 13:02:28 -08001139 s = session_alloc_for_connection (tc);
Florin Coras15531972018-08-12 23:50:53 -07001140 s->app_wrk_index = app_wrk->wrk_index;
Florin Coras7fb0fe12018-04-09 09:24:52 -07001141 s->session_state = SESSION_STATE_OPENED;
Florin Corasa27a46e2019-02-18 13:02:28 -08001142 if (app_worker_init_connected (app_wrk, s))
1143 {
1144 session_free (s);
1145 return -1;
1146 }
Florin Coras371ca502018-02-21 12:07:41 -08001147
Aloys Augustin20ab31e2019-03-25 11:29:17 +01001148 sh = session_handle (s);
1149 session_lookup_add_connection (tc, sh);
Florin Coras00e01d32019-10-21 16:07:46 -07001150 return app_worker_connect_notify (app_wrk, s, SESSION_E_NONE, opaque);
Florin Coras371ca502018-02-21 12:07:41 -08001151}
1152
1153int
Florin Coras15531972018-08-12 23:50:53 -07001154session_open_vc (u32 app_wrk_index, session_endpoint_t * rmt, u32 opaque)
Florin Coras371ca502018-02-21 12:07:41 -08001155{
1156 transport_connection_t *tc;
Florin Coras5665ced2018-10-25 18:03:45 -07001157 transport_endpoint_cfg_t *tep;
Florin Coras371ca502018-02-21 12:07:41 -08001158 u64 handle;
1159 int rv;
1160
Florin Coras5665ced2018-10-25 18:03:45 -07001161 tep = session_endpoint_to_transport_cfg (rmt);
Florin Coras1ee78302019-02-05 15:51:15 -08001162 rv = transport_connect (rmt->transport_proto, tep);
Florin Coras371ca502018-02-21 12:07:41 -08001163 if (rv < 0)
1164 {
1165 SESSION_DBG ("Transport failed to open connection.");
Florin Coras57660d92020-04-04 22:45:34 +00001166 return rv;
Florin Coras371ca502018-02-21 12:07:41 -08001167 }
1168
Florin Coras1ee78302019-02-05 15:51:15 -08001169 tc = transport_get_half_open (rmt->transport_proto, (u32) rv);
Florin Coras371ca502018-02-21 12:07:41 -08001170
1171 /* If transport offers a stream service, only allocate session once the
1172 * connection has been established.
1173 * Add connection to half-open table and save app and tc index. The
1174 * latter is needed to help establish the connection while the former
1175 * is needed when the connect notify comes and we have to notify the
1176 * external app
1177 */
Florin Coras15531972018-08-12 23:50:53 -07001178 handle = (((u64) app_wrk_index) << 32) | (u64) tc->c_index;
Florin Coras371ca502018-02-21 12:07:41 -08001179 session_lookup_add_half_open (tc, handle);
1180
1181 /* Store api_context (opaque) for when the reply comes. Not the nicest
1182 * thing but better than allocating a separate half-open pool.
1183 */
1184 tc->s_index = opaque;
Nathan Skrzypczaka26349d2019-06-26 13:53:08 +02001185 if (transport_half_open_has_fifos (rmt->transport_proto))
1186 return session_ho_stream_connect_notify (tc, 0 /* is_fail */ );
Florin Coras371ca502018-02-21 12:07:41 -08001187 return 0;
1188}
1189
1190int
Florin Coras15531972018-08-12 23:50:53 -07001191session_open_app (u32 app_wrk_index, session_endpoint_t * rmt, u32 opaque)
Florin Coras371ca502018-02-21 12:07:41 -08001192{
Florin Coras5665ced2018-10-25 18:03:45 -07001193 session_endpoint_cfg_t *sep = (session_endpoint_cfg_t *) rmt;
1194 transport_endpoint_cfg_t *tep_cfg = session_endpoint_to_transport_cfg (sep);
1195
Florin Coras15531972018-08-12 23:50:53 -07001196 sep->app_wrk_index = app_wrk_index;
Florin Coras8f89dd02018-03-05 16:53:07 -08001197 sep->opaque = opaque;
Florin Coras371ca502018-02-21 12:07:41 -08001198
Florin Coras1ee78302019-02-05 15:51:15 -08001199 return transport_connect (rmt->transport_proto, tep_cfg);
Florin Coras371ca502018-02-21 12:07:41 -08001200}
1201
1202typedef int (*session_open_service_fn) (u32, session_endpoint_t *, u32);
1203
1204/* *INDENT-OFF* */
1205static session_open_service_fn session_open_srv_fns[TRANSPORT_N_SERVICES] = {
1206 session_open_vc,
1207 session_open_cl,
1208 session_open_app,
1209};
1210/* *INDENT-ON* */
1211
Florin Coras6cf30ad2017-04-04 23:08:23 -07001212/**
1213 * Ask transport to open connection to remote transport endpoint.
1214 *
1215 * Stores handle for matching request with reply since the call can be
1216 * asynchronous. For instance, for TCP the 3-way handshake must complete
1217 * before reply comes. Session is only created once connection is established.
1218 *
1219 * @param app_index Index of the application requesting the connect
1220 * @param st Session type requested.
1221 * @param tep Remote transport endpoint
Florin Coras3cbc04b2017-10-02 00:18:51 -07001222 * @param opaque Opaque data (typically, api_context) the application expects
1223 * on open completion.
Florin Coras6cf30ad2017-04-04 23:08:23 -07001224 */
Florin Corase04c2992017-03-01 08:17:34 -08001225int
Florin Coras15531972018-08-12 23:50:53 -07001226session_open (u32 app_wrk_index, session_endpoint_t * rmt, u32 opaque)
Dave Barach68b0fb02017-02-28 15:15:56 -05001227{
Florin Coras1ee78302019-02-05 15:51:15 -08001228 transport_service_type_t tst;
1229 tst = transport_protocol_service_type (rmt->transport_proto);
Florin Coras15531972018-08-12 23:50:53 -07001230 return session_open_srv_fns[tst] (app_wrk_index, rmt, opaque);
Florin Coras6cf30ad2017-04-04 23:08:23 -07001231}
1232
Florin Coras7fb0fe12018-04-09 09:24:52 -07001233/**
Florin Corasab2f6db2018-08-31 14:31:41 -07001234 * Ask transport to listen on session endpoint.
Florin Coras7fb0fe12018-04-09 09:24:52 -07001235 *
1236 * @param s Session for which listen will be called. Note that unlike
1237 * established sessions, listen sessions are not associated to a
1238 * thread.
Florin Corasab2f6db2018-08-31 14:31:41 -07001239 * @param sep Local endpoint to be listened on.
Florin Coras7fb0fe12018-04-09 09:24:52 -07001240 */
Florin Coras371ca502018-02-21 12:07:41 -08001241int
Florin Coras288eaab2019-02-03 15:26:14 -08001242session_listen (session_t * ls, session_endpoint_cfg_t * sep)
Florin Coras371ca502018-02-21 12:07:41 -08001243{
Florin Corasab2f6db2018-08-31 14:31:41 -07001244 transport_endpoint_t *tep;
Florin Corasa8c3b862020-04-07 22:31:06 +00001245 int tc_index;
1246 u32 s_index;
Florin Corasab2f6db2018-08-31 14:31:41 -07001247
1248 /* Transport bind/listen */
1249 tep = session_endpoint_to_transport (sep);
Florin Coras74cac882018-09-07 09:13:15 -07001250 s_index = ls->session_index;
Florin Corasd4295e62019-02-22 13:11:38 -08001251 tc_index = transport_start_listen (session_get_transport_proto (ls),
1252 s_index, tep);
Florin Corasab2f6db2018-08-31 14:31:41 -07001253
Florin Corasa8c3b862020-04-07 22:31:06 +00001254 if (tc_index < 0)
1255 return tc_index;
Florin Corasab2f6db2018-08-31 14:31:41 -07001256
Florin Corasd4295e62019-02-22 13:11:38 -08001257 /* Attach transport to session. Lookup tables are populated by the app
1258 * worker because local tables (for ct sessions) are not backed by a fib */
Florin Coras74cac882018-09-07 09:13:15 -07001259 ls = listen_session_get (s_index);
Florin Corasab2f6db2018-08-31 14:31:41 -07001260 ls->connection_index = tc_index;
1261
Florin Corasab2f6db2018-08-31 14:31:41 -07001262 return 0;
Florin Coras371ca502018-02-21 12:07:41 -08001263}
1264
Florin Coras6cf30ad2017-04-04 23:08:23 -07001265/**
1266 * Ask transport to stop listening on local transport endpoint.
1267 *
1268 * @param s Session to stop listening on. It must be in state LISTENING.
1269 */
1270int
Florin Coras288eaab2019-02-03 15:26:14 -08001271session_stop_listen (session_t * s)
Florin Coras6cf30ad2017-04-04 23:08:23 -07001272{
Florin Coras561af9b2017-12-09 10:19:43 -08001273 transport_proto_t tp = session_get_transport_proto (s);
Florin Coras6cf30ad2017-04-04 23:08:23 -07001274 transport_connection_t *tc;
Florin Coras6cf30ad2017-04-04 23:08:23 -07001275
Florin Coras1ee78302019-02-05 15:51:15 -08001276 if (s->session_state != SESSION_STATE_LISTENING)
Florin Corasa8c3b862020-04-07 22:31:06 +00001277 return SESSION_E_NOLISTEN;
Florin Coras1ee78302019-02-05 15:51:15 -08001278
1279 tc = transport_get_listener (tp, s->connection_index);
Florin Corasa8c3b862020-04-07 22:31:06 +00001280
1281 /* If no transport, assume everything was cleaned up already */
Florin Coras6cf30ad2017-04-04 23:08:23 -07001282 if (!tc)
Florin Corasa8c3b862020-04-07 22:31:06 +00001283 return SESSION_E_NONE;
Florin Coras6cf30ad2017-04-04 23:08:23 -07001284
Nathan Skrzypczak2eed1a12019-07-04 14:26:21 +02001285 if (!(tc->flags & TRANSPORT_CONNECTION_F_NO_LOOKUP))
1286 session_lookup_del_connection (tc);
Florin Corasa8c3b862020-04-07 22:31:06 +00001287
Florin Coras1ee78302019-02-05 15:51:15 -08001288 transport_stop_listen (tp, s->connection_index);
Florin Corase04c2992017-03-01 08:17:34 -08001289 return 0;
Dave Barach68b0fb02017-02-28 15:15:56 -05001290}
1291
1292/**
Florin Coras5a2ec8f2018-12-27 11:53:11 -08001293 * Initialize session closing procedure.
Florin Corasa5464812017-04-19 13:00:05 -07001294 *
Florin Coras2f8d8fa2018-01-26 06:36:04 -08001295 * Request is always sent to session node to ensure that all outstanding
1296 * requests are served before transport is notified.
Dave Barach68b0fb02017-02-28 15:15:56 -05001297 */
1298void
Florin Coras288eaab2019-02-03 15:26:14 -08001299session_close (session_t * s)
Dave Barach68b0fb02017-02-28 15:15:56 -05001300{
Florin Coras25579b42018-06-06 17:55:02 -07001301 if (!s)
Florin Coras2f8d8fa2018-01-26 06:36:04 -08001302 return;
Florin Coras25579b42018-06-06 17:55:02 -07001303
1304 if (s->session_state >= SESSION_STATE_CLOSING)
1305 {
Florin Coras5a2ec8f2018-12-27 11:53:11 -08001306 /* Session will only be removed once both app and transport
1307 * acknowledge the close */
Florin Coras5f066322019-07-22 19:03:03 -07001308 if (s->session_state == SESSION_STATE_TRANSPORT_CLOSED
1309 || s->session_state == SESSION_STATE_TRANSPORT_DELETED)
Florin Corasdfb3b872019-08-16 17:48:44 -07001310 session_program_transport_ctrl_evt (s, SESSION_CTRL_EVT_CLOSE);
Florin Coras25579b42018-06-06 17:55:02 -07001311 return;
1312 }
1313
Florin Corasb2371c22018-05-31 17:14:10 -07001314 s->session_state = SESSION_STATE_CLOSING;
Florin Corasdfb3b872019-08-16 17:48:44 -07001315 session_program_transport_ctrl_evt (s, SESSION_CTRL_EVT_CLOSE);
1316}
1317
1318/**
1319 * Force a close without waiting for data to be flushed
1320 */
1321void
1322session_reset (session_t * s)
1323{
1324 if (s->session_state >= SESSION_STATE_CLOSING)
1325 return;
1326 /* Drop all outstanding tx data */
1327 svm_fifo_dequeue_drop_all (s->tx_fifo);
1328 s->session_state = SESSION_STATE_CLOSING;
1329 session_program_transport_ctrl_evt (s, SESSION_CTRL_EVT_RESET);
Florin Coras2f8d8fa2018-01-26 06:36:04 -08001330}
1331
1332/**
1333 * Notify transport the session can be disconnected. This should eventually
1334 * result in a delete notification that allows us to cleanup session state.
1335 * Called for both active/passive disconnects.
1336 *
1337 * Must be called from the session's thread.
1338 */
1339void
Florin Coras288eaab2019-02-03 15:26:14 -08001340session_transport_close (session_t * s)
Florin Coras2f8d8fa2018-01-26 06:36:04 -08001341{
Florin Coras692b9492019-07-12 15:01:53 -07001342 if (s->session_state >= SESSION_STATE_APP_CLOSED)
Florin Coras568ebc72018-09-18 16:12:50 -07001343 {
Florin Coras5f066322019-07-22 19:03:03 -07001344 if (s->session_state == SESSION_STATE_TRANSPORT_CLOSED)
1345 s->session_state = SESSION_STATE_CLOSED;
1346 /* If transport is already deleted, just free the session */
1347 else if (s->session_state >= SESSION_STATE_TRANSPORT_DELETED)
Florin Coras692b9492019-07-12 15:01:53 -07001348 session_free_w_fifos (s);
Florin Coras568ebc72018-09-18 16:12:50 -07001349 return;
1350 }
Florin Coras78cc4b02018-12-20 18:24:49 -08001351
Florin Coras692b9492019-07-12 15:01:53 -07001352 /* If the tx queue wasn't drained, the transport can continue to try
1353 * sending the outstanding data (in closed state it cannot). It MUST however
1354 * at one point, either after sending everything or after a timeout, call
1355 * delete notify. This will finally lead to the complete cleanup of the
1356 * session.
Florin Coras78cc4b02018-12-20 18:24:49 -08001357 */
Florin Coras692b9492019-07-12 15:01:53 -07001358 s->session_state = SESSION_STATE_APP_CLOSED;
Florin Coras78cc4b02018-12-20 18:24:49 -08001359
Florin Coras1ee78302019-02-05 15:51:15 -08001360 transport_close (session_get_transport_proto (s), s->connection_index,
1361 s->thread_index);
Dave Barach68b0fb02017-02-28 15:15:56 -05001362}
1363
1364/**
Florin Corasdfb3b872019-08-16 17:48:44 -07001365 * Force transport close
1366 */
1367void
1368session_transport_reset (session_t * s)
1369{
1370 if (s->session_state >= SESSION_STATE_APP_CLOSED)
1371 {
1372 if (s->session_state == SESSION_STATE_TRANSPORT_CLOSED)
1373 s->session_state = SESSION_STATE_CLOSED;
1374 else if (s->session_state >= SESSION_STATE_TRANSPORT_DELETED)
1375 session_free_w_fifos (s);
1376 return;
1377 }
1378
1379 s->session_state = SESSION_STATE_APP_CLOSED;
1380 transport_reset (session_get_transport_proto (s), s->connection_index,
1381 s->thread_index);
1382}
1383
1384/**
Dave Barach68b0fb02017-02-28 15:15:56 -05001385 * Cleanup transport and session state.
Florin Corasd79b41e2017-03-04 05:37:52 -08001386 *
Florin Coras25579b42018-06-06 17:55:02 -07001387 * Notify transport of the cleanup and free the session. This should
1388 * be called only if transport reported some error and is already
1389 * closed.
Dave Barach68b0fb02017-02-28 15:15:56 -05001390 */
1391void
Florin Coras288eaab2019-02-03 15:26:14 -08001392session_transport_cleanup (session_t * s)
Dave Barach68b0fb02017-02-28 15:15:56 -05001393{
Florin Coras25579b42018-06-06 17:55:02 -07001394 /* Delete from main lookup table before we axe the the transport */
1395 session_lookup_del_session (s);
Florin Coras5afea122019-10-28 08:46:37 -07001396 if (s->session_state != SESSION_STATE_TRANSPORT_DELETED)
1397 transport_cleanup (session_get_transport_proto (s), s->connection_index,
1398 s->thread_index);
Florin Coras25579b42018-06-06 17:55:02 -07001399 /* Since we called cleanup, no delete notification will come. So, make
1400 * sure the session is properly freed. */
Florin Corasd3955fb2019-11-29 09:31:47 -08001401 segment_manager_dealloc_fifos (s->rx_fifo, s->tx_fifo);
1402 session_free (s);
Dave Barach68b0fb02017-02-28 15:15:56 -05001403}
1404
Florin Corasa5464812017-04-19 13:00:05 -07001405/**
Florin Corasb384b542018-01-15 01:08:33 -08001406 * Allocate event queues in the shared-memory segment
1407 *
1408 * That can either be a newly created memfd segment, that will need to be
1409 * mapped by all stack users, or the binary api's svm region. The latter is
1410 * assumed to be already mapped. NOTE that this assumption DOES NOT hold if
1411 * api clients bootstrap shm api over sockets (i.e. use memfd segments) and
1412 * vpp uses api svm region for event queues.
Florin Corasa5464812017-04-19 13:00:05 -07001413 */
1414void
Florin Coras31c99552019-03-01 13:00:58 -08001415session_vpp_event_queues_allocate (session_main_t * smm)
Florin Corasa5464812017-04-19 13:00:05 -07001416{
Florin Coras52207f12018-07-12 14:48:06 -07001417 u32 evt_q_length = 2048, evt_size = sizeof (session_event_t);
Florin Corasb384b542018-01-15 01:08:33 -08001418 ssvm_private_t *eqs = &smm->evt_qs_segment;
Florin Corasb4c14912019-02-09 13:51:25 -08001419 uword eqs_size = 64 << 20;
Florin Corasb384b542018-01-15 01:08:33 -08001420 pid_t vpp_pid = getpid ();
Florin Corasa5464812017-04-19 13:00:05 -07001421 void *oldheap;
Florin Corasb384b542018-01-15 01:08:33 -08001422 int i;
Florin Corasa5464812017-04-19 13:00:05 -07001423
Florin Corasb384b542018-01-15 01:08:33 -08001424 if (smm->configured_event_queue_length)
1425 evt_q_length = smm->configured_event_queue_length;
1426
1427 if (smm->evt_qs_use_memfd_seg)
Florin Corasa5464812017-04-19 13:00:05 -07001428 {
Florin Corasb384b542018-01-15 01:08:33 -08001429 if (smm->evt_qs_segment_size)
1430 eqs_size = smm->evt_qs_segment_size;
Florin Corasa5464812017-04-19 13:00:05 -07001431
Florin Corasb384b542018-01-15 01:08:33 -08001432 eqs->ssvm_size = eqs_size;
1433 eqs->i_am_master = 1;
1434 eqs->my_pid = vpp_pid;
1435 eqs->name = format (0, "%s%c", "evt-qs-segment", 0);
1436 eqs->requested_va = smm->session_baseva;
Dave Barach10d8cc62017-05-30 09:30:07 -04001437
Florin Coras95e0ce02018-07-05 23:44:23 -07001438 if (ssvm_master_init (eqs, SSVM_SEGMENT_MEMFD))
1439 {
1440 clib_warning ("failed to initialize queue segment");
1441 return;
1442 }
Florin Corasa5464812017-04-19 13:00:05 -07001443 }
Florin Corasb384b542018-01-15 01:08:33 -08001444
1445 if (smm->evt_qs_use_memfd_seg)
1446 oldheap = ssvm_push_heap (eqs->sh);
1447 else
Nathan Skrzypczak0aa40132019-11-25 16:29:38 +01001448 oldheap = vl_msg_push_heap ();
Florin Corasb384b542018-01-15 01:08:33 -08001449
Florin Coras5a7ca7b2018-10-30 12:01:48 -07001450 for (i = 0; i < vec_len (smm->wrk); i++)
Florin Corasb384b542018-01-15 01:08:33 -08001451 {
Florin Coras3c2fed52018-07-04 04:15:05 -07001452 svm_msg_q_cfg_t _cfg, *cfg = &_cfg;
Florin Coras3c2fed52018-07-04 04:15:05 -07001453 svm_msg_q_ring_cfg_t rc[SESSION_MQ_N_RINGS] = {
1454 {evt_q_length, evt_size, 0}
1455 ,
Florin Corasb4c14912019-02-09 13:51:25 -08001456 {evt_q_length >> 1, 256, 0}
Florin Coras3c2fed52018-07-04 04:15:05 -07001457 };
1458 cfg->consumer_pid = 0;
1459 cfg->n_rings = 2;
1460 cfg->q_nitems = evt_q_length;
1461 cfg->ring_cfgs = rc;
Florin Coras5a7ca7b2018-10-30 12:01:48 -07001462 smm->wrk[i].vpp_event_queue = svm_msg_q_alloc (cfg);
Florin Coras99368312018-08-02 10:45:44 -07001463 if (smm->evt_qs_use_memfd_seg)
1464 {
Florin Coras5a7ca7b2018-10-30 12:01:48 -07001465 if (svm_msg_q_alloc_consumer_eventfd (smm->wrk[i].vpp_event_queue))
Florin Coras99368312018-08-02 10:45:44 -07001466 clib_warning ("eventfd returned");
1467 }
Florin Corasb384b542018-01-15 01:08:33 -08001468 }
1469
1470 if (smm->evt_qs_use_memfd_seg)
1471 ssvm_pop_heap (oldheap);
1472 else
Nathan Skrzypczak0aa40132019-11-25 16:29:38 +01001473 vl_msg_pop_heap (oldheap);
Florin Corasb384b542018-01-15 01:08:33 -08001474}
1475
1476ssvm_private_t *
Florin Coras31c99552019-03-01 13:00:58 -08001477session_main_get_evt_q_segment (void)
Florin Corasb384b542018-01-15 01:08:33 -08001478{
Florin Coras31c99552019-03-01 13:00:58 -08001479 session_main_t *smm = &session_main;
Florin Corasb384b542018-01-15 01:08:33 -08001480 if (smm->evt_qs_use_memfd_seg)
1481 return &smm->evt_qs_segment;
1482 return 0;
Florin Corasa5464812017-04-19 13:00:05 -07001483}
1484
Florin Coras31c99552019-03-01 13:00:58 -08001485u64
1486session_segment_handle (session_t * s)
1487{
1488 svm_fifo_t *f;
1489
Aloys Augustincdb71702019-04-08 17:54:39 +02001490 if (!s->rx_fifo)
Florin Coras31c99552019-03-01 13:00:58 -08001491 return SESSION_INVALID_HANDLE;
1492
1493 f = s->rx_fifo;
1494 return segment_manager_make_segment_handle (f->segment_manager,
1495 f->segment_index);
1496}
1497
Florin Coras371ca502018-02-21 12:07:41 -08001498/* *INDENT-OFF* */
1499static session_fifo_rx_fn *session_tx_fns[TRANSPORT_TX_N_FNS] = {
1500 session_tx_fifo_peek_and_snd,
1501 session_tx_fifo_dequeue_and_snd,
Florin Coras7fb0fe12018-04-09 09:24:52 -07001502 session_tx_fifo_dequeue_internal,
1503 session_tx_fifo_dequeue_and_snd
Florin Coras371ca502018-02-21 12:07:41 -08001504};
1505/* *INDENT-ON* */
1506
Florin Coras561af9b2017-12-09 10:19:43 -08001507void
1508session_register_transport (transport_proto_t transport_proto,
1509 const transport_proto_vft_t * vft, u8 is_ip4,
1510 u32 output_node)
Dave Barach2c25a622017-06-26 11:35:07 -04001511{
Florin Coras31c99552019-03-01 13:00:58 -08001512 session_main_t *smm = &session_main;
Florin Coras561af9b2017-12-09 10:19:43 -08001513 session_type_t session_type;
1514 u32 next_index = ~0;
Dave Barach2c25a622017-06-26 11:35:07 -04001515
Florin Coras561af9b2017-12-09 10:19:43 -08001516 session_type = session_type_from_proto_and_ip (transport_proto, is_ip4);
1517
1518 vec_validate (smm->session_type_to_next, session_type);
Florin Coras561af9b2017-12-09 10:19:43 -08001519 vec_validate (smm->session_tx_fns, session_type);
1520
1521 /* *INDENT-OFF* */
Florin Coras371ca502018-02-21 12:07:41 -08001522 if (output_node != ~0)
1523 {
1524 foreach_vlib_main (({
1525 next_index = vlib_node_add_next (this_vlib_main,
1526 session_queue_node.index,
1527 output_node);
1528 }));
1529 }
Florin Coras561af9b2017-12-09 10:19:43 -08001530 /* *INDENT-ON* */
1531
1532 smm->session_type_to_next[session_type] = next_index;
Nathan Skrzypczake971bc92019-06-19 13:42:37 +02001533 smm->session_tx_fns[session_type] =
1534 session_tx_fns[vft->transport_options.tx_type];
Dave Barach2c25a622017-06-26 11:35:07 -04001535}
1536
Florin Coras07063b82020-03-13 04:44:51 +00001537transport_proto_t
1538session_add_transport_proto (void)
1539{
1540 session_main_t *smm = &session_main;
1541 session_worker_t *wrk;
1542 u32 thread;
1543
1544 smm->last_transport_proto_type += 1;
1545
1546 for (thread = 0; thread < vec_len (smm->wrk); thread++)
1547 {
1548 wrk = session_main_get_worker (thread);
1549 vec_validate (wrk->session_to_enqueue, smm->last_transport_proto_type);
1550 }
1551
1552 return smm->last_transport_proto_type;
1553}
1554
Florin Coras3cbc04b2017-10-02 00:18:51 -07001555transport_connection_t *
Florin Coras288eaab2019-02-03 15:26:14 -08001556session_get_transport (session_t * s)
Florin Coras3cbc04b2017-10-02 00:18:51 -07001557{
Florin Corasade70e42017-10-14 18:56:41 -07001558 if (s->session_state != SESSION_STATE_LISTENING)
Florin Coras4edc37e2019-02-04 23:01:34 -08001559 return transport_get_connection (session_get_transport_proto (s),
1560 s->connection_index, s->thread_index);
1561 else
1562 return transport_get_listener (session_get_transport_proto (s),
1563 s->connection_index);
Florin Coras3cbc04b2017-10-02 00:18:51 -07001564}
1565
Aloys Augustincdb71702019-04-08 17:54:39 +02001566void
Florin Coras09d18c22019-04-24 11:10:02 -07001567session_get_endpoint (session_t * s, transport_endpoint_t * tep, u8 is_lcl)
Aloys Augustincdb71702019-04-08 17:54:39 +02001568{
1569 if (s->session_state != SESSION_STATE_LISTENING)
1570 return transport_get_endpoint (session_get_transport_proto (s),
Florin Coras09d18c22019-04-24 11:10:02 -07001571 s->connection_index, s->thread_index, tep,
1572 is_lcl);
Aloys Augustincdb71702019-04-08 17:54:39 +02001573 else
1574 return transport_get_listener_endpoint (session_get_transport_proto (s),
Florin Coras09d18c22019-04-24 11:10:02 -07001575 s->connection_index, tep, is_lcl);
Aloys Augustincdb71702019-04-08 17:54:39 +02001576}
1577
Florin Coras3cbc04b2017-10-02 00:18:51 -07001578transport_connection_t *
Florin Coras288eaab2019-02-03 15:26:14 -08001579listen_session_get_transport (session_t * s)
Florin Coras3cbc04b2017-10-02 00:18:51 -07001580{
Florin Coras4edc37e2019-02-04 23:01:34 -08001581 return transport_get_listener (session_get_transport_proto (s),
1582 s->connection_index);
Florin Coras3cbc04b2017-10-02 00:18:51 -07001583}
1584
Florin Corasfd542f12018-05-16 19:28:24 -07001585void
Florin Coras5484daa2020-03-27 23:55:06 +00001586session_queue_run_on_main_thread (vlib_main_t * vm)
Florin Corasfd542f12018-05-16 19:28:24 -07001587{
1588 ASSERT (vlib_get_thread_index () == 0);
1589 vlib_process_signal_event_mt (vm, session_queue_process_node.index,
Florin Coras5484daa2020-03-27 23:55:06 +00001590 SESSION_Q_PROCESS_RUN_ON_MAIN, 0);
Florin Corasfd542f12018-05-16 19:28:24 -07001591}
1592
Dave Barach68b0fb02017-02-28 15:15:56 -05001593static clib_error_t *
Florin Corase04c2992017-03-01 08:17:34 -08001594session_manager_main_enable (vlib_main_t * vm)
Dave Barach68b0fb02017-02-28 15:15:56 -05001595{
Florin Corasa332c462018-01-31 06:52:17 -08001596 segment_manager_main_init_args_t _sm_args = { 0 }, *sm_args = &_sm_args;
Florin Coras31c99552019-03-01 13:00:58 -08001597 session_main_t *smm = &session_main;
Florin Corase04c2992017-03-01 08:17:34 -08001598 vlib_thread_main_t *vtm = vlib_get_thread_main ();
Florin Coras371ca502018-02-21 12:07:41 -08001599 u32 num_threads, preallocated_sessions_per_worker;
Florin Coras31c99552019-03-01 13:00:58 -08001600 session_worker_t *wrk;
Florin Corasd5c604d2019-03-18 09:06:35 -07001601 int i;
Dave Barach68b0fb02017-02-28 15:15:56 -05001602
Dave Barach68b0fb02017-02-28 15:15:56 -05001603 num_threads = 1 /* main thread */ + vtm->n_threads;
1604
1605 if (num_threads < 1)
1606 return clib_error_return (0, "n_thread_stacks not set");
1607
Florin Coras5f56d732018-10-30 10:21:59 -07001608 /* Allocate cache line aligned worker contexts */
Florin Coras5a7ca7b2018-10-30 12:01:48 -07001609 vec_validate_aligned (smm->wrk, num_threads - 1, CLIB_CACHE_LINE_BYTES);
Florin Coras3cbc04b2017-10-02 00:18:51 -07001610
Dave Barachacd2a6a2017-05-16 17:41:34 -04001611 for (i = 0; i < num_threads; i++)
1612 {
Florin Coras5a7ca7b2018-10-30 12:01:48 -07001613 wrk = &smm->wrk[i];
Florin Corasb0ffbee2019-07-21 19:23:46 -07001614 wrk->ctrl_head = clib_llist_make_head (wrk->event_elts, evt_list);
Florin Coras2062ec02019-07-15 13:15:18 -07001615 wrk->new_head = clib_llist_make_head (wrk->event_elts, evt_list);
Florin Coras60183db2019-07-20 15:53:16 -07001616 wrk->old_head = clib_llist_make_head (wrk->event_elts, evt_list);
Florin Coras60183db2019-07-20 15:53:16 -07001617 wrk->vm = vlib_mains[i];
Florin Coras5a7ca7b2018-10-30 12:01:48 -07001618 wrk->last_vlib_time = vlib_time_now (vlib_mains[i]);
Florin Corasa8e71c82019-10-22 19:01:39 -07001619 wrk->last_vlib_us_time = wrk->last_vlib_time * CLIB_US_TIME_FREQ;
Florin Coras07063b82020-03-13 04:44:51 +00001620 vec_validate (wrk->session_to_enqueue, smm->last_transport_proto_type);
Florin Corasd67f1122018-05-21 17:47:40 -07001621
Florin Coras29ca16f2017-11-02 18:14:17 -04001622 if (num_threads > 1)
Florin Coras5a7ca7b2018-10-30 12:01:48 -07001623 clib_rwlock_init (&smm->wrk[i].peekers_rw_locks);
Dave Barachacd2a6a2017-05-16 17:41:34 -04001624 }
1625
Florin Corasb384b542018-01-15 01:08:33 -08001626 /* Allocate vpp event queues segment and queue */
1627 session_vpp_event_queues_allocate (smm);
1628
1629 /* Initialize fifo segment main baseva and timeout */
Florin Corasa332c462018-01-31 06:52:17 -08001630 sm_args->baseva = smm->session_baseva + smm->evt_qs_segment_size;
1631 sm_args->size = smm->session_va_space_size;
1632 segment_manager_main_init (sm_args);
Florin Coras6cf30ad2017-04-04 23:08:23 -07001633
Florin Coras66b11312017-07-31 17:18:03 -07001634 /* Preallocate sessions */
Dave Barachb7f1faa2017-08-29 11:43:37 -04001635 if (smm->preallocated_sessions)
Dave Barach68b0fb02017-02-28 15:15:56 -05001636 {
Dave Barachb7f1faa2017-08-29 11:43:37 -04001637 if (num_threads == 1)
Florin Coras66b11312017-07-31 17:18:03 -07001638 {
Florin Coras5a7ca7b2018-10-30 12:01:48 -07001639 pool_init_fixed (smm->wrk[0].sessions, smm->preallocated_sessions);
Florin Coras66b11312017-07-31 17:18:03 -07001640 }
Dave Barachb7f1faa2017-08-29 11:43:37 -04001641 else
Florin Coras66b11312017-07-31 17:18:03 -07001642 {
Dave Barachb7f1faa2017-08-29 11:43:37 -04001643 int j;
1644 preallocated_sessions_per_worker =
1645 (1.1 * (f64) smm->preallocated_sessions /
1646 (f64) (num_threads - 1));
1647
1648 for (j = 1; j < num_threads; j++)
Florin Coras66b11312017-07-31 17:18:03 -07001649 {
Florin Coras5a7ca7b2018-10-30 12:01:48 -07001650 pool_init_fixed (smm->wrk[j].sessions,
Dave Barachb7f1faa2017-08-29 11:43:37 -04001651 preallocated_sessions_per_worker);
Florin Coras66b11312017-07-31 17:18:03 -07001652 }
Florin Coras66b11312017-07-31 17:18:03 -07001653 }
1654 }
Dave Barach68b0fb02017-02-28 15:15:56 -05001655
Florin Coras04e53442017-07-16 17:12:15 -07001656 session_lookup_init ();
Florin Corascea194d2017-10-02 00:18:51 -07001657 app_namespaces_init ();
Florin Coras3cbc04b2017-10-02 00:18:51 -07001658 transport_init ();
Dave Barach68b0fb02017-02-28 15:15:56 -05001659
Florin Corase04c2992017-03-01 08:17:34 -08001660 smm->is_enabled = 1;
1661
Florin Coras561af9b2017-12-09 10:19:43 -08001662 /* Enable transports */
1663 transport_enable_disable (vm, 1);
Dave Barach68b0fb02017-02-28 15:15:56 -05001664 return 0;
1665}
1666
Florin Corasa5464812017-04-19 13:00:05 -07001667void
1668session_node_enable_disable (u8 is_en)
1669{
1670 u8 state = is_en ? VLIB_NODE_STATE_POLLING : VLIB_NODE_STATE_DISABLED;
Florin Corasfd542f12018-05-16 19:28:24 -07001671 vlib_thread_main_t *vtm = vlib_get_thread_main ();
1672 u8 have_workers = vtm->n_threads != 0;
1673
Florin Corasa5464812017-04-19 13:00:05 -07001674 /* *INDENT-OFF* */
1675 foreach_vlib_main (({
Florin Corasfd542f12018-05-16 19:28:24 -07001676 if (have_workers && ii == 0)
1677 {
1678 vlib_node_set_state (this_vlib_main, session_queue_process_node.index,
1679 state);
1680 if (is_en)
1681 {
1682 vlib_node_t *n = vlib_get_node (this_vlib_main,
1683 session_queue_process_node.index);
1684 vlib_start_process (this_vlib_main, n->runtime_index);
1685 }
1686 else
1687 {
1688 vlib_process_signal_event_mt (this_vlib_main,
1689 session_queue_process_node.index,
1690 SESSION_Q_PROCESS_STOP, 0);
1691 }
1692
1693 continue;
1694 }
Florin Corasa5464812017-04-19 13:00:05 -07001695 vlib_node_set_state (this_vlib_main, session_queue_node.index,
1696 state);
1697 }));
1698 /* *INDENT-ON* */
1699}
1700
Florin Corase04c2992017-03-01 08:17:34 -08001701clib_error_t *
1702vnet_session_enable_disable (vlib_main_t * vm, u8 is_en)
1703{
Florin Corascea194d2017-10-02 00:18:51 -07001704 clib_error_t *error = 0;
Florin Corase04c2992017-03-01 08:17:34 -08001705 if (is_en)
1706 {
Florin Coras31c99552019-03-01 13:00:58 -08001707 if (session_main.is_enabled)
Florin Corase04c2992017-03-01 08:17:34 -08001708 return 0;
Dave Barach68b0fb02017-02-28 15:15:56 -05001709
Florin Corascea194d2017-10-02 00:18:51 -07001710 error = session_manager_main_enable (vm);
Vladimir Kropyleva2e44512019-07-16 21:32:41 +03001711 session_node_enable_disable (is_en);
Florin Corase04c2992017-03-01 08:17:34 -08001712 }
1713 else
1714 {
Florin Coras31c99552019-03-01 13:00:58 -08001715 session_main.is_enabled = 0;
Florin Corasa5464812017-04-19 13:00:05 -07001716 session_node_enable_disable (is_en);
Florin Corase04c2992017-03-01 08:17:34 -08001717 }
1718
Florin Corascea194d2017-10-02 00:18:51 -07001719 return error;
Florin Corase04c2992017-03-01 08:17:34 -08001720}
1721
Florin Corase04c2992017-03-01 08:17:34 -08001722clib_error_t *
Florin Corase33c0022020-04-03 17:23:42 +00001723session_main_init (vlib_main_t * vm)
Florin Corase04c2992017-03-01 08:17:34 -08001724{
Florin Coras31c99552019-03-01 13:00:58 -08001725 session_main_t *smm = &session_main;
Florin Corase33c0022020-04-03 17:23:42 +00001726
1727 smm->is_enabled = 0;
1728 smm->session_enable_asap = 0;
David Johnsond9818dd2018-12-14 14:53:41 -05001729 smm->session_baseva = HIGH_SEGMENT_BASEVA;
Florin Corase33c0022020-04-03 17:23:42 +00001730
David Johnsond9818dd2018-12-14 14:53:41 -05001731#if (HIGH_SEGMENT_BASEVA > (4ULL << 30))
1732 smm->session_va_space_size = 128ULL << 30;
Florin Corasb384b542018-01-15 01:08:33 -08001733 smm->evt_qs_segment_size = 64 << 20;
David Johnsond9818dd2018-12-14 14:53:41 -05001734#else
1735 smm->session_va_space_size = 128 << 20;
1736 smm->evt_qs_segment_size = 1 << 20;
1737#endif
Florin Corase33c0022020-04-03 17:23:42 +00001738
1739 smm->last_transport_proto_type = TRANSPORT_PROTO_QUIC;
1740
Nathan Skrzypczak31bd0352019-11-07 17:55:01 +01001741 return 0;
1742}
1743
1744static clib_error_t *
Florin Corase33c0022020-04-03 17:23:42 +00001745session_main_loop_init (vlib_main_t * vm)
Nathan Skrzypczak31bd0352019-11-07 17:55:01 +01001746{
1747 session_main_t *smm = &session_main;
1748 if (smm->session_enable_asap)
1749 {
1750 vlib_worker_thread_barrier_sync (vm);
1751 vnet_session_enable_disable (vm, 1 /* is_en */ );
1752 vlib_worker_thread_barrier_release (vm);
1753 }
Florin Corase04c2992017-03-01 08:17:34 -08001754 return 0;
1755}
1756
Florin Corase33c0022020-04-03 17:23:42 +00001757VLIB_INIT_FUNCTION (session_main_init);
1758VLIB_MAIN_LOOP_ENTER_FUNCTION (session_main_loop_init);
Dave Barach2c25a622017-06-26 11:35:07 -04001759
1760static clib_error_t *
1761session_config_fn (vlib_main_t * vm, unformat_input_t * input)
Dave Barach10d8cc62017-05-30 09:30:07 -04001762{
Florin Coras31c99552019-03-01 13:00:58 -08001763 session_main_t *smm = &session_main;
Dave Barach10d8cc62017-05-30 09:30:07 -04001764 u32 nitems;
Florin Coras66b11312017-07-31 17:18:03 -07001765 uword tmp;
Dave Barach10d8cc62017-05-30 09:30:07 -04001766
1767 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1768 {
1769 if (unformat (input, "event-queue-length %d", &nitems))
1770 {
1771 if (nitems >= 2048)
1772 smm->configured_event_queue_length = nitems;
1773 else
1774 clib_warning ("event queue length %d too small, ignored", nitems);
1775 }
Florin Coras66b11312017-07-31 17:18:03 -07001776 else if (unformat (input, "preallocated-sessions %d",
1777 &smm->preallocated_sessions))
Dave Barach2c25a622017-06-26 11:35:07 -04001778 ;
Florin Coras66b11312017-07-31 17:18:03 -07001779 else if (unformat (input, "v4-session-table-buckets %d",
1780 &smm->configured_v4_session_table_buckets))
1781 ;
1782 else if (unformat (input, "v4-halfopen-table-buckets %d",
1783 &smm->configured_v4_halfopen_table_buckets))
1784 ;
1785 else if (unformat (input, "v6-session-table-buckets %d",
1786 &smm->configured_v6_session_table_buckets))
1787 ;
1788 else if (unformat (input, "v6-halfopen-table-buckets %d",
1789 &smm->configured_v6_halfopen_table_buckets))
1790 ;
1791 else if (unformat (input, "v4-session-table-memory %U",
1792 unformat_memory_size, &tmp))
1793 {
1794 if (tmp >= 0x100000000)
1795 return clib_error_return (0, "memory size %llx (%lld) too large",
1796 tmp, tmp);
1797 smm->configured_v4_session_table_memory = tmp;
1798 }
1799 else if (unformat (input, "v4-halfopen-table-memory %U",
1800 unformat_memory_size, &tmp))
1801 {
1802 if (tmp >= 0x100000000)
1803 return clib_error_return (0, "memory size %llx (%lld) too large",
1804 tmp, tmp);
1805 smm->configured_v4_halfopen_table_memory = tmp;
1806 }
1807 else if (unformat (input, "v6-session-table-memory %U",
1808 unformat_memory_size, &tmp))
1809 {
1810 if (tmp >= 0x100000000)
1811 return clib_error_return (0, "memory size %llx (%lld) too large",
1812 tmp, tmp);
1813 smm->configured_v6_session_table_memory = tmp;
1814 }
1815 else if (unformat (input, "v6-halfopen-table-memory %U",
1816 unformat_memory_size, &tmp))
1817 {
1818 if (tmp >= 0x100000000)
1819 return clib_error_return (0, "memory size %llx (%lld) too large",
1820 tmp, tmp);
1821 smm->configured_v6_halfopen_table_memory = tmp;
1822 }
Florin Coras93e65802017-11-29 00:07:11 -05001823 else if (unformat (input, "local-endpoints-table-memory %U",
1824 unformat_memory_size, &tmp))
1825 {
1826 if (tmp >= 0x100000000)
1827 return clib_error_return (0, "memory size %llx (%lld) too large",
1828 tmp, tmp);
1829 smm->local_endpoints_table_memory = tmp;
1830 }
1831 else if (unformat (input, "local-endpoints-table-buckets %d",
1832 &smm->local_endpoints_table_buckets))
1833 ;
Florin Corasb384b542018-01-15 01:08:33 -08001834 else if (unformat (input, "evt_qs_memfd_seg"))
1835 smm->evt_qs_use_memfd_seg = 1;
Florin Corasb4c14912019-02-09 13:51:25 -08001836 else if (unformat (input, "evt_qs_seg_size %U", unformat_memory_size,
1837 &smm->evt_qs_segment_size))
1838 ;
Nathan Skrzypczak1292d192019-09-13 15:44:54 +02001839 else if (unformat (input, "enable"))
Nathan Skrzypczak31bd0352019-11-07 17:55:01 +01001840 smm->session_enable_asap = 1;
Dave Barach10d8cc62017-05-30 09:30:07 -04001841 else
1842 return clib_error_return (0, "unknown input `%U'",
1843 format_unformat_error, input);
1844 }
1845 return 0;
1846}
1847
1848VLIB_CONFIG_FUNCTION (session_config_fn, "session");
1849
Dave Barach68b0fb02017-02-28 15:15:56 -05001850/*
1851 * fd.io coding-style-patch-verification: ON
1852 *
1853 * Local Variables:
1854 * eval: (c-set-style "gnu")
1855 * End:
1856 */