blob: 605fd584e2dcdabcfd06024ab9a20c279eb4cea5 [file] [log] [blame]
Dave Barach68b0fb02017-02-28 15:15:56 -05001/*
Florin Coras288eaab2019-02-03 15:26:14 -08002 * Copyright (c) 2017-2019 Cisco and/or its affiliates.
Dave Barach68b0fb02017-02-28 15:15:56 -05003 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
Florin Coras6792ec02017-03-13 03:49:51 -070016#include <math.h>
Dave Barach68b0fb02017-02-28 15:15:56 -050017#include <vlib/vlib.h>
18#include <vnet/vnet.h>
Dave Barach68b0fb02017-02-28 15:15:56 -050019#include <vppinfra/elog.h>
Florin Coras561af9b2017-12-09 10:19:43 -080020#include <vnet/session/transport.h>
Florin Coras8e43d042018-05-04 15:46:57 -070021#include <vnet/session/session.h>
Florin Coras6792ec02017-03-13 03:49:51 -070022#include <vnet/session/application.h>
Florin Coras52207f12018-07-12 14:48:06 -070023#include <vnet/session/application_interface.h>
Florin Corasba7d8f52019-02-22 13:11:38 -080024#include <vnet/session/application_local.h>
Florin Corase69f4952017-03-07 10:06:24 -080025#include <vnet/session/session_debug.h>
Florin Corase86a8ed2018-01-05 03:20:25 -080026#include <svm/queue.h>
Dave Barach68b0fb02017-02-28 15:15:56 -050027
Florin Coras2b81e3c2019-02-27 07:55:46 -080028static void session_mq_accepted_reply_handler (void *data);
29
30static void
31accepted_notify_cb (void *data, u32 data_len)
32{
33 session_mq_accepted_reply_handler (data);
34}
35
Florin Coras52207f12018-07-12 14:48:06 -070036static void
37session_mq_accepted_reply_handler (void *data)
38{
39 session_accepted_reply_msg_t *mp = (session_accepted_reply_msg_t *) data;
40 vnet_disconnect_args_t _a = { 0 }, *a = &_a;
Florin Coras288eaab2019-02-03 15:26:14 -080041 session_state_t old_state;
Florin Coras21795132018-09-09 09:40:51 -070042 app_worker_t *app_wrk;
Florin Coras288eaab2019-02-03 15:26:14 -080043 session_t *s;
Florin Coras52207f12018-07-12 14:48:06 -070044
45 /* Server isn't interested, kill the session */
46 if (mp->retval)
47 {
48 a->app_index = mp->context;
49 a->handle = mp->handle;
50 vnet_disconnect_session (a);
51 return;
52 }
53
Florin Coras2b81e3c2019-02-27 07:55:46 -080054 /* Mail this back from the main thread. We're not polling in main
55 * thread so we're using other workers for notifications. */
56 if (vlib_num_workers () && vlib_get_thread_index () != 0
57 && session_thread_from_handle (mp->handle) == 0)
Florin Coras52207f12018-07-12 14:48:06 -070058 {
Florin Coras2b81e3c2019-02-27 07:55:46 -080059 vl_api_rpc_call_main_thread (accepted_notify_cb, data,
60 sizeof (session_accepted_reply_msg_t));
61 return;
62 }
63
64 s = session_get_from_handle_if_valid (mp->handle);
65 if (!s)
66 return;
67
68 app_wrk = app_worker_get (s->app_wrk_index);
69 if (app_wrk->app_index != mp->context)
70 {
71 clib_warning ("app doesn't own session");
72 return;
73 }
74
75 if (!session_has_transport (s))
76 {
Florin Coras653e43f2019-03-04 10:56:23 -080077 s->session_state = SESSION_STATE_READY;
Florin Coras2b81e3c2019-02-27 07:55:46 -080078 if (ct_session_connect_notify (s))
Florin Coras52207f12018-07-12 14:48:06 -070079 return;
Florin Coras52207f12018-07-12 14:48:06 -070080 }
81 else
82 {
Florin Corasb0f662f2018-12-27 14:51:46 -080083 old_state = s->session_state;
Florin Coras52207f12018-07-12 14:48:06 -070084 s->session_state = SESSION_STATE_READY;
Sirshak Das28aa5392019-02-05 01:33:33 -060085
86 if (!svm_fifo_is_empty_prod (s->rx_fifo))
Florin Corasf6c43132019-03-01 12:41:21 -080087 app_worker_lock_and_send_event (app_wrk, s, SESSION_IO_EVT_RX);
Florin Corasb0f662f2018-12-27 14:51:46 -080088
89 /* Closed while waiting for app to reply. Resend disconnect */
90 if (old_state >= SESSION_STATE_TRANSPORT_CLOSING)
91 {
Florin Coras69b68ef2019-04-02 11:38:51 -070092 app_worker_close_notify (app_wrk, s);
Florin Corasb0f662f2018-12-27 14:51:46 -080093 s->session_state = old_state;
94 return;
95 }
Florin Coras52207f12018-07-12 14:48:06 -070096 }
97}
98
99static void
100session_mq_reset_reply_handler (void *data)
101{
Florin Coras4af830c2018-12-04 09:21:36 -0800102 vnet_disconnect_args_t _a = { 0 }, *a = &_a;
Florin Coras52207f12018-07-12 14:48:06 -0700103 session_reset_reply_msg_t *mp;
Florin Coras15531972018-08-12 23:50:53 -0700104 app_worker_t *app_wrk;
Florin Coras288eaab2019-02-03 15:26:14 -0800105 session_t *s;
Florin Coras15531972018-08-12 23:50:53 -0700106 application_t *app;
Florin Coras52207f12018-07-12 14:48:06 -0700107 u32 index, thread_index;
108
109 mp = (session_reset_reply_msg_t *) data;
Florin Coras3c7d4f92018-12-14 11:28:43 -0800110 app = application_lookup (mp->context);
Florin Coras52207f12018-07-12 14:48:06 -0700111 if (!app)
112 return;
113
114 session_parse_handle (mp->handle, &index, &thread_index);
115 s = session_get_if_valid (index, thread_index);
Florin Coras3c7d4f92018-12-14 11:28:43 -0800116
117 /* Session was already closed or already cleaned up */
118 if (!s || s->session_state != SESSION_STATE_TRANSPORT_CLOSING)
119 return;
120
Florin Coras15531972018-08-12 23:50:53 -0700121 app_wrk = app_worker_get (s->app_wrk_index);
122 if (!app_wrk || app_wrk->app_index != app->app_index)
123 {
124 clib_warning ("App % does not own handle 0x%lx!", app->app_index,
125 mp->handle);
126 return;
127 }
Florin Coras52207f12018-07-12 14:48:06 -0700128
129 /* Client objected to resetting the session, log and continue */
130 if (mp->retval)
131 {
132 clib_warning ("client retval %d", mp->retval);
133 return;
134 }
135
136 /* This comes as a response to a reset, transport only waiting for
137 * confirmation to remove connection state, no need to disconnect */
Florin Coras4af830c2018-12-04 09:21:36 -0800138 a->handle = mp->handle;
139 a->app_index = app->app_index;
140 vnet_disconnect_session (a);
Florin Coras52207f12018-07-12 14:48:06 -0700141}
142
143static void
144session_mq_disconnected_handler (void *data)
145{
146 session_disconnected_reply_msg_t *rmp;
147 vnet_disconnect_args_t _a, *a = &_a;
148 svm_msg_q_msg_t _msg, *msg = &_msg;
149 session_disconnected_msg_t *mp;
Florin Coras15531972018-08-12 23:50:53 -0700150 app_worker_t *app_wrk;
Florin Coras52207f12018-07-12 14:48:06 -0700151 session_event_t *evt;
Florin Coras288eaab2019-02-03 15:26:14 -0800152 session_t *s;
Florin Coras52207f12018-07-12 14:48:06 -0700153 application_t *app;
154 int rv = 0;
155
156 mp = (session_disconnected_msg_t *) data;
Florin Corasc3638fe2018-08-24 13:58:49 -0700157 if (!(s = session_get_from_handle_if_valid (mp->handle)))
158 {
159 clib_warning ("could not disconnect handle %llu", mp->handle);
160 return;
161 }
Florin Coras15531972018-08-12 23:50:53 -0700162 app_wrk = app_worker_get (s->app_wrk_index);
163 app = application_lookup (mp->client_index);
Florin Corasc3638fe2018-08-24 13:58:49 -0700164 if (!(app_wrk && app && app->app_index == app_wrk->app_index))
Florin Coras52207f12018-07-12 14:48:06 -0700165 {
Florin Corasc3638fe2018-08-24 13:58:49 -0700166 clib_warning ("could not disconnect session: %llu app: %u",
Florin Coras15531972018-08-12 23:50:53 -0700167 mp->handle, mp->client_index);
Florin Coras3d0fadc2018-07-17 05:35:47 -0700168 return;
Florin Coras52207f12018-07-12 14:48:06 -0700169 }
Florin Coras3d0fadc2018-07-17 05:35:47 -0700170
171 a->handle = mp->handle;
Florin Coras15531972018-08-12 23:50:53 -0700172 a->app_index = app_wrk->wrk_index;
Florin Coras3d0fadc2018-07-17 05:35:47 -0700173 rv = vnet_disconnect_session (a);
Florin Coras52207f12018-07-12 14:48:06 -0700174
Florin Coras15531972018-08-12 23:50:53 -0700175 svm_msg_q_lock_and_alloc_msg_w_ring (app_wrk->event_queue,
Florin Coras52207f12018-07-12 14:48:06 -0700176 SESSION_MQ_CTRL_EVT_RING,
177 SVM_Q_WAIT, msg);
Florin Coras15531972018-08-12 23:50:53 -0700178 svm_msg_q_unlock (app_wrk->event_queue);
179 evt = svm_msg_q_msg_data (app_wrk->event_queue, msg);
Dave Barachb7b92992018-10-17 10:38:51 -0400180 clib_memset (evt, 0, sizeof (*evt));
Florin Coras30e79c22019-01-02 19:31:22 -0800181 evt->event_type = SESSION_CTRL_EVT_DISCONNECTED_REPLY;
Florin Coras52207f12018-07-12 14:48:06 -0700182 rmp = (session_disconnected_reply_msg_t *) evt->data;
183 rmp->handle = mp->handle;
184 rmp->context = mp->context;
185 rmp->retval = rv;
Florin Coras15531972018-08-12 23:50:53 -0700186 svm_msg_q_add (app_wrk->event_queue, msg, SVM_Q_WAIT);
Florin Coras52207f12018-07-12 14:48:06 -0700187}
188
189static void
190session_mq_disconnected_reply_handler (void *data)
191{
192 session_disconnected_reply_msg_t *mp;
193 vnet_disconnect_args_t _a, *a = &_a;
194 application_t *app;
195
196 mp = (session_disconnected_reply_msg_t *) data;
197
198 /* Client objected to disconnecting the session, log and continue */
199 if (mp->retval)
200 {
201 clib_warning ("client retval %d", mp->retval);
202 return;
203 }
204
205 /* Disconnect has been confirmed. Confirm close to transport */
206 app = application_lookup (mp->context);
207 if (app)
208 {
209 a->handle = mp->handle;
Florin Coras15531972018-08-12 23:50:53 -0700210 a->app_index = app->app_index;
Florin Coras52207f12018-07-12 14:48:06 -0700211 vnet_disconnect_session (a);
212 }
213}
214
Florin Coras30e79c22019-01-02 19:31:22 -0800215static void
216session_mq_worker_update_handler (void *data)
217{
218 session_worker_update_msg_t *mp = (session_worker_update_msg_t *) data;
219 session_worker_update_reply_msg_t *rmp;
220 svm_msg_q_msg_t _msg, *msg = &_msg;
221 app_worker_t *app_wrk;
222 u32 owner_app_wrk_map;
223 session_event_t *evt;
Florin Coras288eaab2019-02-03 15:26:14 -0800224 session_t *s;
Florin Coras30e79c22019-01-02 19:31:22 -0800225 application_t *app;
226
227 app = application_lookup (mp->client_index);
228 if (!app)
229 return;
230 if (!(s = session_get_from_handle_if_valid (mp->handle)))
231 {
232 clib_warning ("invalid handle %llu", mp->handle);
233 return;
234 }
235 app_wrk = app_worker_get (s->app_wrk_index);
236 if (app_wrk->app_index != app->app_index)
237 {
238 clib_warning ("app %u does not own session %llu", app->app_index,
239 mp->handle);
240 return;
241 }
242 owner_app_wrk_map = app_wrk->wrk_map_index;
243 app_wrk = application_get_worker (app, mp->wrk_index);
244
245 /* This needs to come from the new owner */
246 if (mp->req_wrk_index == owner_app_wrk_map)
247 {
248 session_req_worker_update_msg_t *wump;
249
250 svm_msg_q_lock_and_alloc_msg_w_ring (app_wrk->event_queue,
251 SESSION_MQ_CTRL_EVT_RING,
252 SVM_Q_WAIT, msg);
253 svm_msg_q_unlock (app_wrk->event_queue);
254 evt = svm_msg_q_msg_data (app_wrk->event_queue, msg);
255 clib_memset (evt, 0, sizeof (*evt));
256 evt->event_type = SESSION_CTRL_EVT_REQ_WORKER_UPDATE;
257 wump = (session_req_worker_update_msg_t *) evt->data;
258 wump->session_handle = mp->handle;
259 svm_msg_q_add (app_wrk->event_queue, msg, SVM_Q_WAIT);
260 return;
261 }
262
263 app_worker_own_session (app_wrk, s);
264
265 /*
266 * Send reply
267 */
268 svm_msg_q_lock_and_alloc_msg_w_ring (app_wrk->event_queue,
269 SESSION_MQ_CTRL_EVT_RING,
270 SVM_Q_WAIT, msg);
271 svm_msg_q_unlock (app_wrk->event_queue);
272 evt = svm_msg_q_msg_data (app_wrk->event_queue, msg);
273 clib_memset (evt, 0, sizeof (*evt));
274 evt->event_type = SESSION_CTRL_EVT_WORKER_UPDATE_REPLY;
275 rmp = (session_worker_update_reply_msg_t *) evt->data;
276 rmp->handle = mp->handle;
Florin Coras288eaab2019-02-03 15:26:14 -0800277 rmp->rx_fifo = pointer_to_uword (s->rx_fifo);
278 rmp->tx_fifo = pointer_to_uword (s->tx_fifo);
Florin Coras30e79c22019-01-02 19:31:22 -0800279 rmp->segment_handle = session_segment_handle (s);
280 svm_msg_q_add (app_wrk->event_queue, msg, SVM_Q_WAIT);
281
282 /*
283 * Retransmit messages that may have been lost
284 */
Florin Coras288eaab2019-02-03 15:26:14 -0800285 if (s->tx_fifo && !svm_fifo_is_empty (s->tx_fifo))
Florin Corasf6c43132019-03-01 12:41:21 -0800286 session_send_io_evt_to_thread (s->tx_fifo, SESSION_IO_EVT_TX);
Florin Coras30e79c22019-01-02 19:31:22 -0800287
Florin Coras288eaab2019-02-03 15:26:14 -0800288 if (s->rx_fifo && !svm_fifo_is_empty (s->rx_fifo))
Florin Corasf6c43132019-03-01 12:41:21 -0800289 app_worker_lock_and_send_event (app_wrk, s, SESSION_IO_EVT_RX);
Florin Coras30e79c22019-01-02 19:31:22 -0800290
291 if (s->session_state >= SESSION_STATE_TRANSPORT_CLOSING)
Florin Coras69b68ef2019-04-02 11:38:51 -0700292 app_worker_close_notify (app_wrk, s);
Florin Coras30e79c22019-01-02 19:31:22 -0800293}
294
Dave Barach68b0fb02017-02-28 15:15:56 -0500295vlib_node_registration_t session_queue_node;
296
297typedef struct
298{
299 u32 session_index;
300 u32 server_thread_index;
301} session_queue_trace_t;
302
303/* packet trace format function */
304static u8 *
305format_session_queue_trace (u8 * s, va_list * args)
306{
307 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
308 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
309 session_queue_trace_t *t = va_arg (*args, session_queue_trace_t *);
310
311 s = format (s, "SESSION_QUEUE: session index %d, server thread index %d",
312 t->session_index, t->server_thread_index);
313 return s;
314}
315
Florin Coras6792ec02017-03-13 03:49:51 -0700316#define foreach_session_queue_error \
317_(TX, "Packets transmitted") \
Florin Coras93992a92017-05-24 18:03:56 -0700318_(TIMER, "Timer events") \
319_(NO_BUFFER, "Out of buffers")
Dave Barach68b0fb02017-02-28 15:15:56 -0500320
321typedef enum
322{
323#define _(sym,str) SESSION_QUEUE_ERROR_##sym,
324 foreach_session_queue_error
325#undef _
326 SESSION_QUEUE_N_ERROR,
327} session_queue_error_t;
328
329static char *session_queue_error_strings[] = {
330#define _(sym,string) string,
331 foreach_session_queue_error
332#undef _
333};
334
Florin Coras0d60a0f2018-06-29 02:02:08 -0700335enum
336{
337 SESSION_TX_NO_BUFFERS = -2,
338 SESSION_TX_NO_DATA,
339 SESSION_TX_OK
340};
341
Florin Coras66cf5e02018-06-08 09:17:39 -0700342static void
343session_tx_trace_frame (vlib_main_t * vm, vlib_node_runtime_t * node,
344 u32 next_index, u32 * to_next, u16 n_segs,
Florin Coras288eaab2019-02-03 15:26:14 -0800345 session_t * s, u32 n_trace)
Florin Corasf6d68ed2017-05-07 19:12:02 -0700346{
Florin Coras8e43d042018-05-04 15:46:57 -0700347 session_queue_trace_t *t;
Florin Coras66cf5e02018-06-08 09:17:39 -0700348 vlib_buffer_t *b;
349 int i;
350
351 for (i = 0; i < clib_min (n_trace, n_segs); i++)
352 {
353 b = vlib_get_buffer (vm, to_next[i - n_segs]);
354 vlib_trace_buffer (vm, node, next_index, b, 1 /* follow_chain */ );
Florin Coras66cf5e02018-06-08 09:17:39 -0700355 t = vlib_add_trace (vm, node, b, sizeof (*t));
356 t->session_index = s->session_index;
357 t->server_thread_index = s->thread_index;
358 }
Florin Coras4df38712018-06-20 12:44:16 -0700359 vlib_set_trace_count (vm, node, n_trace - i);
Florin Coras8e43d042018-05-04 15:46:57 -0700360}
Florin Corasf6d68ed2017-05-07 19:12:02 -0700361
Florin Coras8e43d042018-05-04 15:46:57 -0700362always_inline void
363session_tx_fifo_chain_tail (vlib_main_t * vm, session_tx_context_t * ctx,
364 vlib_buffer_t * b, u16 * n_bufs, u8 peek_data)
365{
Florin Coras8e43d042018-05-04 15:46:57 -0700366 vlib_buffer_t *chain_b, *prev_b;
367 u32 chain_bi0, to_deq, left_from_seg;
Florin Coras31c99552019-03-01 13:00:58 -0800368 session_worker_t *wrk;
Florin Coras8e43d042018-05-04 15:46:57 -0700369 u16 len_to_deq, n_bytes_read;
370 u8 *data, j;
Florin Corasb2215d62017-08-01 16:56:58 -0700371
Florin Coras31c99552019-03-01 13:00:58 -0800372 wrk = session_main_get_worker (ctx->s->thread_index);
Florin Coras8e43d042018-05-04 15:46:57 -0700373 b->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
374 b->total_length_not_including_first_buffer = 0;
375
376 chain_b = b;
377 left_from_seg = clib_min (ctx->snd_mss - b->current_length,
378 ctx->left_to_snd);
Florin Corasb2215d62017-08-01 16:56:58 -0700379 to_deq = left_from_seg;
Florin Coras8e43d042018-05-04 15:46:57 -0700380 for (j = 1; j < ctx->n_bufs_per_seg; j++)
Florin Corasf6d68ed2017-05-07 19:12:02 -0700381 {
Florin Coras8e43d042018-05-04 15:46:57 -0700382 prev_b = chain_b;
383 len_to_deq = clib_min (to_deq, ctx->deq_per_buf);
Florin Corasf6d68ed2017-05-07 19:12:02 -0700384
385 *n_bufs -= 1;
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700386 chain_bi0 = wrk->tx_buffers[*n_bufs];
Florin Coras8e43d042018-05-04 15:46:57 -0700387 chain_b = vlib_get_buffer (vm, chain_bi0);
388 chain_b->current_data = 0;
389 data = vlib_buffer_get_current (chain_b);
Florin Corasf6d68ed2017-05-07 19:12:02 -0700390 if (peek_data)
391 {
Florin Coras288eaab2019-02-03 15:26:14 -0800392 n_bytes_read = svm_fifo_peek (ctx->s->tx_fifo,
Florin Coras8e43d042018-05-04 15:46:57 -0700393 ctx->tx_offset, len_to_deq, data);
394 ctx->tx_offset += n_bytes_read;
Florin Corasf6d68ed2017-05-07 19:12:02 -0700395 }
396 else
397 {
Florin Coras8e43d042018-05-04 15:46:57 -0700398 if (ctx->transport_vft->tx_type == TRANSPORT_TX_DGRAM)
Florin Coras7fb0fe12018-04-09 09:24:52 -0700399 {
Florin Coras288eaab2019-02-03 15:26:14 -0800400 svm_fifo_t *f = ctx->s->tx_fifo;
Florin Coras8e43d042018-05-04 15:46:57 -0700401 session_dgram_hdr_t *hdr = &ctx->hdr;
Florin Coras7fb0fe12018-04-09 09:24:52 -0700402 u16 deq_now;
Florin Coras7fb0fe12018-04-09 09:24:52 -0700403 deq_now = clib_min (hdr->data_length - hdr->data_offset,
Florin Coras8e43d042018-05-04 15:46:57 -0700404 len_to_deq);
405 n_bytes_read = svm_fifo_peek (f, hdr->data_offset, deq_now,
406 data);
Florin Coras7fb0fe12018-04-09 09:24:52 -0700407 ASSERT (n_bytes_read > 0);
408
409 hdr->data_offset += n_bytes_read;
410 if (hdr->data_offset == hdr->data_length)
shubing guoaea5f392018-08-30 13:29:49 +0800411 {
412 u32 offset = hdr->data_length + SESSION_CONN_HDR_LEN;
413 svm_fifo_dequeue_drop (f, offset);
414 }
Florin Coras7fb0fe12018-04-09 09:24:52 -0700415 }
416 else
Florin Coras288eaab2019-02-03 15:26:14 -0800417 n_bytes_read = svm_fifo_dequeue_nowait (ctx->s->tx_fifo,
Florin Coras8e43d042018-05-04 15:46:57 -0700418 len_to_deq, data);
Florin Corasf6d68ed2017-05-07 19:12:02 -0700419 }
Florin Coras8e43d042018-05-04 15:46:57 -0700420 ASSERT (n_bytes_read == len_to_deq);
421 chain_b->current_length = n_bytes_read;
422 b->total_length_not_including_first_buffer += chain_b->current_length;
Florin Corasf6d68ed2017-05-07 19:12:02 -0700423
424 /* update previous buffer */
Florin Coras8e43d042018-05-04 15:46:57 -0700425 prev_b->next_buffer = chain_bi0;
426 prev_b->flags |= VLIB_BUFFER_NEXT_PRESENT;
Florin Corasf6d68ed2017-05-07 19:12:02 -0700427
428 /* update current buffer */
Florin Coras8e43d042018-05-04 15:46:57 -0700429 chain_b->next_buffer = 0;
Florin Corasf6d68ed2017-05-07 19:12:02 -0700430
Florin Corasb2215d62017-08-01 16:56:58 -0700431 to_deq -= n_bytes_read;
432 if (to_deq == 0)
Florin Corasf6d68ed2017-05-07 19:12:02 -0700433 break;
434 }
Florin Coras1f152cd2017-08-18 19:28:03 -0700435 ASSERT (to_deq == 0
Florin Coras8e43d042018-05-04 15:46:57 -0700436 && b->total_length_not_including_first_buffer == left_from_seg);
437 ctx->left_to_snd -= left_from_seg;
Florin Corasf6d68ed2017-05-07 19:12:02 -0700438}
439
Florin Coras8e43d042018-05-04 15:46:57 -0700440always_inline void
441session_tx_fill_buffer (vlib_main_t * vm, session_tx_context_t * ctx,
442 vlib_buffer_t * b, u16 * n_bufs, u8 peek_data)
443{
444 u32 len_to_deq;
445 u8 *data0;
446 int n_bytes_read;
447
448 /*
449 * Start with the first buffer in chain
450 */
451 b->error = 0;
452 b->flags = VNET_BUFFER_F_LOCALLY_ORIGINATED;
453 b->current_data = 0;
Florin Coras8e43d042018-05-04 15:46:57 -0700454
Florin Coras1ee78302019-02-05 15:51:15 -0800455 data0 = vlib_buffer_make_headroom (b, TRANSPORT_MAX_HDRS_LEN);
Florin Coras8e43d042018-05-04 15:46:57 -0700456 len_to_deq = clib_min (ctx->left_to_snd, ctx->deq_per_first_buf);
457
Florin Coras7fb0fe12018-04-09 09:24:52 -0700458 if (peek_data)
459 {
Florin Coras288eaab2019-02-03 15:26:14 -0800460 n_bytes_read = svm_fifo_peek (ctx->s->tx_fifo, ctx->tx_offset,
Florin Coras8e43d042018-05-04 15:46:57 -0700461 len_to_deq, data0);
462 ASSERT (n_bytes_read > 0);
463 /* Keep track of progress locally, transport is also supposed to
464 * increment it independently when pushing the header */
465 ctx->tx_offset += n_bytes_read;
Florin Coras7fb0fe12018-04-09 09:24:52 -0700466 }
467 else
468 {
Florin Coras8e43d042018-05-04 15:46:57 -0700469 if (ctx->transport_vft->tx_type == TRANSPORT_TX_DGRAM)
470 {
471 session_dgram_hdr_t *hdr = &ctx->hdr;
Florin Coras288eaab2019-02-03 15:26:14 -0800472 svm_fifo_t *f = ctx->s->tx_fifo;
Florin Coras8e43d042018-05-04 15:46:57 -0700473 u16 deq_now;
474 u32 offset;
475
476 ASSERT (hdr->data_length > hdr->data_offset);
477 deq_now = clib_min (hdr->data_length - hdr->data_offset,
478 len_to_deq);
479 offset = hdr->data_offset + SESSION_CONN_HDR_LEN;
480 n_bytes_read = svm_fifo_peek (f, offset, deq_now, data0);
481 ASSERT (n_bytes_read > 0);
482
483 if (ctx->s->session_state == SESSION_STATE_LISTENING)
484 {
485 ip_copy (&ctx->tc->rmt_ip, &hdr->rmt_ip, ctx->tc->is_ip4);
486 ctx->tc->rmt_port = hdr->rmt_port;
487 }
488 hdr->data_offset += n_bytes_read;
489 if (hdr->data_offset == hdr->data_length)
490 {
491 offset = hdr->data_length + SESSION_CONN_HDR_LEN;
492 svm_fifo_dequeue_drop (f, offset);
493 }
494 }
Florin Coras7fb0fe12018-04-09 09:24:52 -0700495 else
496 {
Florin Coras288eaab2019-02-03 15:26:14 -0800497 n_bytes_read = svm_fifo_dequeue_nowait (ctx->s->tx_fifo,
Florin Coras8e43d042018-05-04 15:46:57 -0700498 len_to_deq, data0);
499 ASSERT (n_bytes_read > 0);
Florin Coras7fb0fe12018-04-09 09:24:52 -0700500 }
501 }
Florin Coras8e43d042018-05-04 15:46:57 -0700502 b->current_length = n_bytes_read;
503 ctx->left_to_snd -= n_bytes_read;
Dave Barach68b0fb02017-02-28 15:15:56 -0500504
Florin Coras8e43d042018-05-04 15:46:57 -0700505 /*
506 * Fill in the remaining buffers in the chain, if any
507 */
508 if (PREDICT_FALSE (ctx->n_bufs_per_seg > 1 && ctx->left_to_snd))
509 session_tx_fifo_chain_tail (vm, ctx, b, n_bufs, peek_data);
Dave Barach68b0fb02017-02-28 15:15:56 -0500510
Florin Coras8e43d042018-05-04 15:46:57 -0700511 /* *INDENT-OFF* */
Florin Coras8b20bf52018-06-14 14:55:50 -0700512 SESSION_EVT_DBG(SESSION_EVT_DEQ, ctx->s, ({
Florin Corasf6c43132019-03-01 12:41:21 -0800513 ed->data[0] = SESSION_IO_EVT_TX;
Florin Coras8b20bf52018-06-14 14:55:50 -0700514 ed->data[1] = ctx->max_dequeue;
Florin Coras8e43d042018-05-04 15:46:57 -0700515 ed->data[2] = len_to_deq;
Florin Coras8b20bf52018-06-14 14:55:50 -0700516 ed->data[3] = ctx->left_to_snd;
Florin Coras8e43d042018-05-04 15:46:57 -0700517 }));
518 /* *INDENT-ON* */
519}
520
521always_inline u8
Florin Coras288eaab2019-02-03 15:26:14 -0800522session_tx_not_ready (session_t * s, u8 peek_data)
Florin Coras8e43d042018-05-04 15:46:57 -0700523{
524 if (peek_data)
Florin Corase04c2992017-03-01 08:17:34 -0800525 {
Florin Coras8e43d042018-05-04 15:46:57 -0700526 /* Can retransmit for closed sessions but can't send new data if
527 * session is not ready or closed */
528 if (s->session_state < SESSION_STATE_READY)
529 return 1;
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800530 if (s->session_state >= SESSION_STATE_TRANSPORT_CLOSED)
Florin Corasca1c8f32018-05-23 21:01:30 -0700531 return 2;
Florin Corase04c2992017-03-01 08:17:34 -0800532 }
Florin Coras8e43d042018-05-04 15:46:57 -0700533 return 0;
534}
Dave Barach68b0fb02017-02-28 15:15:56 -0500535
Florin Coras8e43d042018-05-04 15:46:57 -0700536always_inline transport_connection_t *
537session_tx_get_transport (session_tx_context_t * ctx, u8 peek_data)
538{
539 if (peek_data)
540 {
541 return ctx->transport_vft->get_connection (ctx->s->connection_index,
542 ctx->s->thread_index);
543 }
544 else
545 {
546 if (ctx->s->session_state == SESSION_STATE_LISTENING)
547 return ctx->transport_vft->get_listener (ctx->s->connection_index);
548 else
549 {
550 return ctx->transport_vft->get_connection (ctx->s->connection_index,
551 ctx->s->thread_index);
552 }
553 }
554}
Florin Coras6a9b68b2017-11-21 04:20:42 -0800555
Florin Coras8e43d042018-05-04 15:46:57 -0700556always_inline void
557session_tx_set_dequeue_params (vlib_main_t * vm, session_tx_context_t * ctx,
Florin Corasf08f26d2018-05-10 13:20:47 -0700558 u32 max_segs, u8 peek_data)
Florin Coras8e43d042018-05-04 15:46:57 -0700559{
560 u32 n_bytes_per_buf, n_bytes_per_seg;
Sirshak Das28aa5392019-02-05 01:33:33 -0600561 ctx->max_dequeue = svm_fifo_max_dequeue_cons (ctx->s->tx_fifo);
Dave Barach68b0fb02017-02-28 15:15:56 -0500562 if (peek_data)
563 {
Florin Coras9d063042017-09-14 03:08:00 -0400564 /* Offset in rx fifo from where to peek data */
Florin Coras8e43d042018-05-04 15:46:57 -0700565 ctx->tx_offset = ctx->transport_vft->tx_fifo_offset (ctx->tc);
566 if (PREDICT_FALSE (ctx->tx_offset >= ctx->max_dequeue))
567 {
568 ctx->max_len_to_snd = 0;
569 return;
570 }
571 ctx->max_dequeue -= ctx->tx_offset;
Dave Barach68b0fb02017-02-28 15:15:56 -0500572 }
Florin Coras7fb0fe12018-04-09 09:24:52 -0700573 else
574 {
Florin Coras8e43d042018-05-04 15:46:57 -0700575 if (ctx->transport_vft->tx_type == TRANSPORT_TX_DGRAM)
Florin Coras7fb0fe12018-04-09 09:24:52 -0700576 {
Florin Coras8e43d042018-05-04 15:46:57 -0700577 if (ctx->max_dequeue <= sizeof (ctx->hdr))
578 {
579 ctx->max_len_to_snd = 0;
580 return;
581 }
Florin Coras288eaab2019-02-03 15:26:14 -0800582 svm_fifo_peek (ctx->s->tx_fifo, 0, sizeof (ctx->hdr),
Florin Coras8e43d042018-05-04 15:46:57 -0700583 (u8 *) & ctx->hdr);
584 ASSERT (ctx->hdr.data_length > ctx->hdr.data_offset);
585 ctx->max_dequeue = ctx->hdr.data_length - ctx->hdr.data_offset;
Florin Coras7fb0fe12018-04-09 09:24:52 -0700586 }
587 }
Florin Coras8e43d042018-05-04 15:46:57 -0700588 ASSERT (ctx->max_dequeue > 0);
Florin Coras6792ec02017-03-13 03:49:51 -0700589
590 /* Ensure we're not writing more than transport window allows */
Florin Coras8e43d042018-05-04 15:46:57 -0700591 if (ctx->max_dequeue < ctx->snd_space)
Florin Coras3e350af2017-03-30 02:54:28 -0700592 {
593 /* Constrained by tx queue. Try to send only fully formed segments */
Florin Coras8e43d042018-05-04 15:46:57 -0700594 ctx->max_len_to_snd =
595 (ctx->max_dequeue > ctx->snd_mss) ?
596 ctx->max_dequeue - ctx->max_dequeue % ctx->snd_mss : ctx->max_dequeue;
Florin Coras3e350af2017-03-30 02:54:28 -0700597 /* TODO Nagle ? */
598 }
599 else
600 {
Florin Coras1f152cd2017-08-18 19:28:03 -0700601 /* Expectation is that snd_space0 is already a multiple of snd_mss */
Florin Coras8e43d042018-05-04 15:46:57 -0700602 ctx->max_len_to_snd = ctx->snd_space;
Florin Coras3e350af2017-03-30 02:54:28 -0700603 }
Dave Barach68b0fb02017-02-28 15:15:56 -0500604
Florin Corasf08f26d2018-05-10 13:20:47 -0700605 /* Check if we're tx constrained by the node */
606 ctx->n_segs_per_evt = ceil ((f64) ctx->max_len_to_snd / ctx->snd_mss);
607 if (ctx->n_segs_per_evt > max_segs)
608 {
609 ctx->n_segs_per_evt = max_segs;
610 ctx->max_len_to_snd = max_segs * ctx->snd_mss;
611 }
612
Damjan Marion8934a042019-02-09 23:29:26 +0100613 n_bytes_per_buf = vlib_buffer_get_default_data_size (vm);
Florin Coras1ee78302019-02-05 15:51:15 -0800614 ASSERT (n_bytes_per_buf > TRANSPORT_MAX_HDRS_LEN);
615 n_bytes_per_seg = TRANSPORT_MAX_HDRS_LEN + ctx->snd_mss;
Florin Corasf08f26d2018-05-10 13:20:47 -0700616 ctx->n_bufs_per_seg = ceil ((f64) n_bytes_per_seg / n_bytes_per_buf);
Florin Coras8e43d042018-05-04 15:46:57 -0700617 ctx->deq_per_buf = clib_min (ctx->snd_mss, n_bytes_per_buf);
618 ctx->deq_per_first_buf = clib_min (ctx->snd_mss,
Florin Coras1ee78302019-02-05 15:51:15 -0800619 n_bytes_per_buf -
620 TRANSPORT_MAX_HDRS_LEN);
Florin Coras8e43d042018-05-04 15:46:57 -0700621}
Florin Corasf6d68ed2017-05-07 19:12:02 -0700622
Florin Coras8e43d042018-05-04 15:46:57 -0700623always_inline int
624session_tx_fifo_read_and_snd_i (vlib_main_t * vm, vlib_node_runtime_t * node,
Florin Coras31c99552019-03-01 13:00:58 -0800625 session_worker_t * wrk,
Florin Coras1bcad5c2019-01-09 20:04:38 -0800626 session_event_t * e, int *n_tx_packets,
Florin Coras8e43d042018-05-04 15:46:57 -0700627 u8 peek_data)
628{
Florin Coras2068fd42019-01-31 14:35:50 -0800629 u32 next_index, next0, next1, *to_next, n_left_to_next, n_left, pbi;
Florin Corasf08f26d2018-05-10 13:20:47 -0700630 u32 n_trace = vlib_get_trace_count (vm, node), n_bufs_needed = 0;
Florin Coras31c99552019-03-01 13:00:58 -0800631 session_main_t *smm = &session_main;
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700632 session_tx_context_t *ctx = &wrk->ctx;
Florin Coras8e43d042018-05-04 15:46:57 -0700633 transport_proto_t tp;
634 vlib_buffer_t *pb;
Florin Corasca1c8f32018-05-23 21:01:30 -0700635 u16 n_bufs, rv;
Florin Coras8e43d042018-05-04 15:46:57 -0700636
Florin Coras1bcad5c2019-01-09 20:04:38 -0800637 if (PREDICT_FALSE ((rv = session_tx_not_ready (ctx->s, peek_data))))
Florin Coras8e43d042018-05-04 15:46:57 -0700638 {
Florin Corasca1c8f32018-05-23 21:01:30 -0700639 if (rv < 2)
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700640 vec_add1 (wrk->pending_event_vector, *e);
Florin Coras0d60a0f2018-06-29 02:02:08 -0700641 return SESSION_TX_NO_DATA;
Florin Coras8e43d042018-05-04 15:46:57 -0700642 }
643
Florin Coras1bcad5c2019-01-09 20:04:38 -0800644 next_index = smm->session_type_to_next[ctx->s->session_type];
Florin Coras66cf5e02018-06-08 09:17:39 -0700645 next0 = next1 = next_index;
Florin Coras8e43d042018-05-04 15:46:57 -0700646
Florin Coras1bcad5c2019-01-09 20:04:38 -0800647 tp = session_get_transport_proto (ctx->s);
Florin Coras8e43d042018-05-04 15:46:57 -0700648 ctx->transport_vft = transport_protocol_get_vft (tp);
649 ctx->tc = session_tx_get_transport (ctx, peek_data);
650 ctx->snd_mss = ctx->transport_vft->send_mss (ctx->tc);
Florin Coras42ceddb2018-12-12 10:56:01 -0800651
652 if (PREDICT_FALSE (e->event_type == SESSION_IO_EVT_TX_FLUSH))
653 {
654 if (ctx->transport_vft->flush_data)
655 ctx->transport_vft->flush_data (ctx->tc);
656 }
657
658 ctx->snd_space = transport_connection_snd_space (ctx->tc,
David Johnsond9818dd2018-12-14 14:53:41 -0500659 vm->clib_time.
660 last_cpu_time,
Florin Coras42ceddb2018-12-12 10:56:01 -0800661 ctx->snd_mss);
Florin Coras8e43d042018-05-04 15:46:57 -0700662 if (ctx->snd_space == 0 || ctx->snd_mss == 0)
663 {
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700664 vec_add1 (wrk->pending_event_vector, *e);
Florin Coras0d60a0f2018-06-29 02:02:08 -0700665 return SESSION_TX_NO_DATA;
Florin Coras8e43d042018-05-04 15:46:57 -0700666 }
667
668 /* Allow enqueuing of a new event */
Florin Coras288eaab2019-02-03 15:26:14 -0800669 svm_fifo_unset_event (ctx->s->tx_fifo);
Florin Coras8e43d042018-05-04 15:46:57 -0700670
671 /* Check how much we can pull. */
Florin Corasf08f26d2018-05-10 13:20:47 -0700672 session_tx_set_dequeue_params (vm, ctx, VLIB_FRAME_SIZE - *n_tx_packets,
673 peek_data);
674
Florin Coras8e43d042018-05-04 15:46:57 -0700675 if (PREDICT_FALSE (!ctx->max_len_to_snd))
Florin Coras0d60a0f2018-06-29 02:02:08 -0700676 return SESSION_TX_NO_DATA;
Dave Barach68b0fb02017-02-28 15:15:56 -0500677
Florin Corasf08f26d2018-05-10 13:20:47 -0700678 n_bufs_needed = ctx->n_segs_per_evt * ctx->n_bufs_per_seg;
Florin Coras2068fd42019-01-31 14:35:50 -0800679 vec_validate_aligned (wrk->tx_buffers, n_bufs_needed - 1,
680 CLIB_CACHE_LINE_BYTES);
681 n_bufs = vlib_buffer_alloc (vm, wrk->tx_buffers, n_bufs_needed);
682 if (PREDICT_FALSE (n_bufs < n_bufs_needed))
Dave Barach68b0fb02017-02-28 15:15:56 -0500683 {
Florin Coras2068fd42019-01-31 14:35:50 -0800684 if (n_bufs)
685 vlib_buffer_free (vm, wrk->tx_buffers, n_bufs);
686 vec_add1 (wrk->pending_event_vector, *e);
687 return SESSION_TX_NO_BUFFERS;
Florin Coras8e43d042018-05-04 15:46:57 -0700688 }
Dave Barach68b0fb02017-02-28 15:15:56 -0500689
Florin Coras8e43d042018-05-04 15:46:57 -0700690 /*
691 * Write until we fill up a frame
692 */
693 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
Florin Corasf08f26d2018-05-10 13:20:47 -0700694 if (PREDICT_FALSE (ctx->n_segs_per_evt > n_left_to_next))
Florin Coras8e43d042018-05-04 15:46:57 -0700695 {
Florin Corasf08f26d2018-05-10 13:20:47 -0700696 ctx->n_segs_per_evt = n_left_to_next;
697 ctx->max_len_to_snd = ctx->snd_mss * n_left_to_next;
698 }
699 ctx->left_to_snd = ctx->max_len_to_snd;
700 n_left = ctx->n_segs_per_evt;
Florin Coras66cf5e02018-06-08 09:17:39 -0700701
702 while (n_left >= 4)
703 {
704 vlib_buffer_t *b0, *b1;
705 u32 bi0, bi1;
706
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700707 pbi = wrk->tx_buffers[n_bufs - 3];
Florin Coras66cf5e02018-06-08 09:17:39 -0700708 pb = vlib_get_buffer (vm, pbi);
709 vlib_prefetch_buffer_header (pb, STORE);
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700710 pbi = wrk->tx_buffers[n_bufs - 4];
Florin Coras66cf5e02018-06-08 09:17:39 -0700711 pb = vlib_get_buffer (vm, pbi);
712 vlib_prefetch_buffer_header (pb, STORE);
713
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700714 to_next[0] = bi0 = wrk->tx_buffers[--n_bufs];
715 to_next[1] = bi1 = wrk->tx_buffers[--n_bufs];
Florin Coras66cf5e02018-06-08 09:17:39 -0700716
717 b0 = vlib_get_buffer (vm, bi0);
718 b1 = vlib_get_buffer (vm, bi1);
719
720 session_tx_fill_buffer (vm, ctx, b0, &n_bufs, peek_data);
721 session_tx_fill_buffer (vm, ctx, b1, &n_bufs, peek_data);
722
723 ctx->transport_vft->push_header (ctx->tc, b0);
724 ctx->transport_vft->push_header (ctx->tc, b1);
725
726 to_next += 2;
727 n_left_to_next -= 2;
728 n_left -= 2;
729
730 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
731 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b1);
732
733 vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next,
734 n_left_to_next, bi0, bi1, next0,
735 next1);
736 }
Florin Corasf08f26d2018-05-10 13:20:47 -0700737 while (n_left)
738 {
Florin Coras66cf5e02018-06-08 09:17:39 -0700739 vlib_buffer_t *b0;
740 u32 bi0;
Florin Corasf6d68ed2017-05-07 19:12:02 -0700741
Florin Coras91ca4622018-06-30 11:27:59 -0700742 if (n_left > 1)
743 {
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700744 pbi = wrk->tx_buffers[n_bufs - 2];
Florin Coras91ca4622018-06-30 11:27:59 -0700745 pb = vlib_get_buffer (vm, pbi);
746 vlib_prefetch_buffer_header (pb, STORE);
747 }
748
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700749 to_next[0] = bi0 = wrk->tx_buffers[--n_bufs];
Florin Coras66cf5e02018-06-08 09:17:39 -0700750 b0 = vlib_get_buffer (vm, bi0);
751 session_tx_fill_buffer (vm, ctx, b0, &n_bufs, peek_data);
Florin Coras81a13db2018-03-16 08:48:31 -0700752
Florin Coras66cf5e02018-06-08 09:17:39 -0700753 /* Ask transport to push header after current_length and
754 * total_length_not_including_first_buffer are updated */
755 ctx->transport_vft->push_header (ctx->tc, b0);
Florin Corasf6359c82017-06-19 12:26:09 -0400756
Florin Coras66cf5e02018-06-08 09:17:39 -0700757 to_next += 1;
758 n_left_to_next -= 1;
759 n_left -= 1;
Dave Barach68b0fb02017-02-28 15:15:56 -0500760
Florin Coras66cf5e02018-06-08 09:17:39 -0700761 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
Florin Coras7fb0fe12018-04-09 09:24:52 -0700762
Florin Coras66cf5e02018-06-08 09:17:39 -0700763 vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
764 n_left_to_next, bi0, next0);
Dave Barach68b0fb02017-02-28 15:15:56 -0500765 }
Florin Coras66cf5e02018-06-08 09:17:39 -0700766
767 if (PREDICT_FALSE (n_trace > 0))
768 session_tx_trace_frame (vm, node, next_index, to_next,
Florin Coras1bcad5c2019-01-09 20:04:38 -0800769 ctx->n_segs_per_evt, ctx->s, n_trace);
Florin Coras2068fd42019-01-31 14:35:50 -0800770 if (PREDICT_FALSE (n_bufs))
771 {
772 clib_warning ("not all buffers consumed");
773 vlib_buffer_free (vm, wrk->tx_buffers, n_bufs);
774 }
Florin Corasf08f26d2018-05-10 13:20:47 -0700775 *n_tx_packets += ctx->n_segs_per_evt;
Florin Corasd67f1122018-05-21 17:47:40 -0700776 transport_connection_update_tx_stats (ctx->tc, ctx->max_len_to_snd);
Florin Coras8e43d042018-05-04 15:46:57 -0700777 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
Dave Barach68b0fb02017-02-28 15:15:56 -0500778
Florin Coras6792ec02017-03-13 03:49:51 -0700779 /* If we couldn't dequeue all bytes mark as partially read */
Florin Corasf08f26d2018-05-10 13:20:47 -0700780 ASSERT (ctx->left_to_snd == 0);
Florin Coras8e43d042018-05-04 15:46:57 -0700781 if (ctx->max_len_to_snd < ctx->max_dequeue)
Florin Coras288eaab2019-02-03 15:26:14 -0800782 if (svm_fifo_set_event (ctx->s->tx_fifo))
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700783 vec_add1 (wrk->pending_event_vector, *e);
Florin Coras7fb0fe12018-04-09 09:24:52 -0700784
Florin Coras8e43d042018-05-04 15:46:57 -0700785 if (!peek_data && ctx->transport_vft->tx_type == TRANSPORT_TX_DGRAM)
Dave Barach68b0fb02017-02-28 15:15:56 -0500786 {
Florin Coras7fb0fe12018-04-09 09:24:52 -0700787 /* Fix dgram pre header */
Florin Coras8e43d042018-05-04 15:46:57 -0700788 if (ctx->max_len_to_snd < ctx->max_dequeue)
Florin Coras288eaab2019-02-03 15:26:14 -0800789 svm_fifo_overwrite_head (ctx->s->tx_fifo, (u8 *) & ctx->hdr,
Florin Coras7fb0fe12018-04-09 09:24:52 -0700790 sizeof (session_dgram_pre_hdr_t));
791 /* More data needs to be read */
Sirshak Das28aa5392019-02-05 01:33:33 -0600792 else if (svm_fifo_max_dequeue_cons (ctx->s->tx_fifo) > 0)
Florin Coras288eaab2019-02-03 15:26:14 -0800793 if (svm_fifo_set_event (ctx->s->tx_fifo))
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700794 vec_add1 (wrk->pending_event_vector, *e);
Dave Barach68b0fb02017-02-28 15:15:56 -0500795 }
Florin Coras0d60a0f2018-06-29 02:02:08 -0700796 return SESSION_TX_OK;
Dave Barach68b0fb02017-02-28 15:15:56 -0500797}
798
799int
Florin Corasd79b41e2017-03-04 05:37:52 -0800800session_tx_fifo_peek_and_snd (vlib_main_t * vm, vlib_node_runtime_t * node,
Florin Coras31c99552019-03-01 13:00:58 -0800801 session_worker_t * wrk,
Florin Coras1bcad5c2019-01-09 20:04:38 -0800802 session_event_t * e, int *n_tx_pkts)
Dave Barach68b0fb02017-02-28 15:15:56 -0500803{
Florin Coras1bcad5c2019-01-09 20:04:38 -0800804 return session_tx_fifo_read_and_snd_i (vm, node, wrk, e, n_tx_pkts, 1);
Dave Barach68b0fb02017-02-28 15:15:56 -0500805}
806
807int
Florin Corasd79b41e2017-03-04 05:37:52 -0800808session_tx_fifo_dequeue_and_snd (vlib_main_t * vm, vlib_node_runtime_t * node,
Florin Coras31c99552019-03-01 13:00:58 -0800809 session_worker_t * wrk,
Florin Coras1bcad5c2019-01-09 20:04:38 -0800810 session_event_t * e, int *n_tx_pkts)
Dave Barach68b0fb02017-02-28 15:15:56 -0500811{
Florin Coras1bcad5c2019-01-09 20:04:38 -0800812 return session_tx_fifo_read_and_snd_i (vm, node, wrk, e, n_tx_pkts, 0);
Dave Barach68b0fb02017-02-28 15:15:56 -0500813}
814
Florin Coras371ca502018-02-21 12:07:41 -0800815int
816session_tx_fifo_dequeue_internal (vlib_main_t * vm,
817 vlib_node_runtime_t * node,
Florin Coras31c99552019-03-01 13:00:58 -0800818 session_worker_t * wrk,
Florin Coras1bcad5c2019-01-09 20:04:38 -0800819 session_event_t * e, int *n_tx_pkts)
Florin Coras371ca502018-02-21 12:07:41 -0800820{
Florin Coras288eaab2019-02-03 15:26:14 -0800821 session_t *s = wrk->ctx.s;
Florin Coras1bcad5c2019-01-09 20:04:38 -0800822
Florin Corasc0737e92019-03-04 14:19:39 -0800823 if (PREDICT_FALSE (s->session_state >= SESSION_STATE_TRANSPORT_CLOSED))
Florin Corasef915342018-09-29 10:23:06 -0700824 return 0;
Florin Coras288eaab2019-02-03 15:26:14 -0800825 svm_fifo_unset_event (s->tx_fifo);
Florin Corasf940f8a2019-03-06 10:44:38 -0800826 return transport_custom_tx (session_get_transport_proto (s), s);
Florin Coras371ca502018-02-21 12:07:41 -0800827}
828
Florin Coras288eaab2019-02-03 15:26:14 -0800829always_inline session_t *
Florin Coras52207f12018-07-12 14:48:06 -0700830session_event_get_session (session_event_t * e, u8 thread_index)
Florin Corasa5464812017-04-19 13:00:05 -0700831{
Florin Corasc0737e92019-03-04 14:19:39 -0800832 return session_get_if_valid (e->session_index, thread_index);
Florin Corasa5464812017-04-19 13:00:05 -0700833}
834
Florin Corasd67f1122018-05-21 17:47:40 -0700835static void
Florin Coras31c99552019-03-01 13:00:58 -0800836session_update_dispatch_period (session_worker_t * wrk, f64 now,
Florin Corasd67f1122018-05-21 17:47:40 -0700837 u32 thread_index)
838{
Florin Corasefefc6b2018-11-07 12:49:19 -0800839 if (wrk->last_tx_packets)
Florin Corasc44a5582018-11-01 16:30:54 -0700840 {
841 f64 sample = now - wrk->last_vlib_time;
Florin Corasc44a5582018-11-01 16:30:54 -0700842 wrk->dispatch_period = (wrk->dispatch_period + sample) * 0.5;
843 }
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700844 wrk->last_vlib_time = now;
Florin Corasd67f1122018-05-21 17:47:40 -0700845}
846
Florin Coras0d60a0f2018-06-29 02:02:08 -0700847static uword
848session_queue_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
849 vlib_frame_t * frame)
850{
Florin Coras31c99552019-03-01 13:00:58 -0800851 session_main_t *smm = vnet_get_session_main ();
Florin Coras3c2fed52018-07-04 04:15:05 -0700852 u32 thread_index = vm->thread_index, n_to_dequeue, n_events;
Florin Coras31c99552019-03-01 13:00:58 -0800853 session_worker_t *wrk = &smm->wrk[thread_index];
Florin Coras5f56d732018-10-30 10:21:59 -0700854 session_event_t *e, *fifo_events;
Florin Coras3c2fed52018-07-04 04:15:05 -0700855 svm_msg_q_msg_t _msg, *msg = &_msg;
Florin Coras0d60a0f2018-06-29 02:02:08 -0700856 f64 now = vlib_time_now (vm);
Florin Coras3c2fed52018-07-04 04:15:05 -0700857 int n_tx_packets = 0, i, rv;
Florin Coras15531972018-08-12 23:50:53 -0700858 app_worker_t *app_wrk;
Florin Coras3c2fed52018-07-04 04:15:05 -0700859 svm_msg_q_t *mq;
Florin Coras0d60a0f2018-06-29 02:02:08 -0700860 void (*fp) (void *);
861
862 SESSION_EVT_DBG (SESSION_EVT_POLL_GAP_TRACK, smm, thread_index);
863
864 /*
865 * Update transport time
866 */
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700867 session_update_dispatch_period (wrk, now, thread_index);
Florin Coras0d60a0f2018-06-29 02:02:08 -0700868 transport_update_time (now, thread_index);
869
Florin Coras0d60a0f2018-06-29 02:02:08 -0700870 SESSION_EVT_DBG (SESSION_EVT_DEQ_NODE, 0);
871
Florin Coras5f56d732018-10-30 10:21:59 -0700872 /* Make sure postponed events are handled first */
873 fifo_events = wrk->free_event_vector;
874 vec_append (fifo_events, wrk->postponed_event_vector);
Florin Coras36ee9f12018-11-02 12:52:10 -0700875 _vec_len (wrk->postponed_event_vector) = 0;
Florin Coras0d60a0f2018-06-29 02:02:08 -0700876
Florin Coras5f56d732018-10-30 10:21:59 -0700877 /* Try to dequeue what is available. Don't wait for lock.
878 * XXX: we may need priorities here */
879 mq = wrk->vpp_event_queue;
880 n_to_dequeue = svm_msg_q_size (mq);
881 if (n_to_dequeue && svm_msg_q_try_lock (mq) == 0)
882 {
883 for (i = 0; i < n_to_dequeue; i++)
884 {
885 vec_add2 (fifo_events, e, 1);
886 svm_msg_q_sub_w_lock (mq, msg);
Florin Coras3c7d4f92018-12-14 11:28:43 -0800887 /* Works because reply messages are smaller than a session evt.
888 * If we ever need to support bigger messages this needs to be
889 * fixed */
Dave Barach178cf492018-11-13 16:34:13 -0500890 clib_memcpy_fast (e, svm_msg_q_msg_data (mq, msg), sizeof (*e));
Florin Coras5f56d732018-10-30 10:21:59 -0700891 svm_msg_q_free_msg (mq, msg);
892 }
893 svm_msg_q_unlock (mq);
894 }
895
896 vec_append (fifo_events, wrk->pending_event_vector);
897 vec_append (fifo_events, wrk->pending_disconnects);
898
Florin Coras5f56d732018-10-30 10:21:59 -0700899 _vec_len (wrk->pending_event_vector) = 0;
Florin Coras36ee9f12018-11-02 12:52:10 -0700900 _vec_len (wrk->pending_disconnects) = 0;
Florin Coras5f56d732018-10-30 10:21:59 -0700901
Florin Coras0d60a0f2018-06-29 02:02:08 -0700902 n_events = vec_len (fifo_events);
Florin Coras5f56d732018-10-30 10:21:59 -0700903 if (PREDICT_FALSE (!n_events))
904 return 0;
905
Florin Coras0d60a0f2018-06-29 02:02:08 -0700906 for (i = 0; i < n_events; i++)
907 {
Florin Coras288eaab2019-02-03 15:26:14 -0800908 session_t *s; /* $$$ prefetch 1 ahead maybe */
Florin Coras52207f12018-07-12 14:48:06 -0700909 session_event_t *e;
Florin Coras1bcad5c2019-01-09 20:04:38 -0800910 u8 need_tx_ntf;
Florin Coras0d60a0f2018-06-29 02:02:08 -0700911
912 e = &fifo_events[i];
913 switch (e->event_type)
914 {
Florin Coras42ceddb2018-12-12 10:56:01 -0800915 case SESSION_IO_EVT_TX_FLUSH:
Florin Corasf6c43132019-03-01 12:41:21 -0800916 case SESSION_IO_EVT_TX:
Florin Coras0d60a0f2018-06-29 02:02:08 -0700917 /* Don't try to send more that one frame per dispatch cycle */
918 if (n_tx_packets == VLIB_FRAME_SIZE)
919 {
Florin Coras5f56d732018-10-30 10:21:59 -0700920 vec_add1 (wrk->postponed_event_vector, *e);
Florin Coras0d60a0f2018-06-29 02:02:08 -0700921 break;
922 }
923
924 s = session_event_get_session (e, thread_index);
925 if (PREDICT_FALSE (!s))
926 {
Florin Coras326b81e2018-10-04 19:03:05 -0700927 clib_warning ("session was freed!");
Florin Coras0d60a0f2018-06-29 02:02:08 -0700928 continue;
929 }
Florin Coras14ed6df2019-03-06 21:13:42 -0800930 CLIB_PREFETCH (s->tx_fifo, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
Florin Coras1bcad5c2019-01-09 20:04:38 -0800931 wrk->ctx.s = s;
Florin Coras0d60a0f2018-06-29 02:02:08 -0700932 /* Spray packets in per session type frames, since they go to
933 * different nodes */
Florin Coras1bcad5c2019-01-09 20:04:38 -0800934 rv = (smm->session_tx_fns[s->session_type]) (vm, node, wrk, e,
Florin Coras0d60a0f2018-06-29 02:02:08 -0700935 &n_tx_packets);
936 if (PREDICT_TRUE (rv == SESSION_TX_OK))
937 {
Florin Coras288eaab2019-02-03 15:26:14 -0800938 need_tx_ntf = svm_fifo_needs_tx_ntf (s->tx_fifo,
Florin Coras1bcad5c2019-01-09 20:04:38 -0800939 wrk->ctx.max_len_to_snd);
940 if (PREDICT_FALSE (need_tx_ntf))
941 session_dequeue_notify (s);
Florin Coras0d60a0f2018-06-29 02:02:08 -0700942 }
943 else if (PREDICT_FALSE (rv == SESSION_TX_NO_BUFFERS))
944 {
945 vlib_node_increment_counter (vm, node->node_index,
946 SESSION_QUEUE_ERROR_NO_BUFFER, 1);
947 continue;
948 }
949 break;
Florin Coras317b8e02019-04-17 09:57:46 -0700950 case SESSION_IO_EVT_RX:
951 s = session_event_get_session (e, thread_index);
952 if (!s)
953 break;
954 transport_app_rx_evt (session_get_transport_proto (s),
955 s->connection_index, s->thread_index);
956 break;
Florin Corasf6c43132019-03-01 12:41:21 -0800957 case SESSION_CTRL_EVT_CLOSE:
Florin Coras3c7d4f92018-12-14 11:28:43 -0800958 s = session_get_from_handle_if_valid (e->session_handle);
959 if (PREDICT_FALSE (!s))
960 break;
961
Florin Corase96bf632018-12-18 22:44:27 -0800962 /* Make sure session disconnects run after the pending list is
963 * drained, i.e., postpone if the first time. If not the first
964 * and the tx queue is still not empty, try to wait for some
965 * dispatch cycles */
966 if (!e->postponed
Sirshak Das28aa5392019-02-05 01:33:33 -0600967 || (e->postponed < 200
968 && svm_fifo_max_dequeue_cons (s->tx_fifo)))
Florin Coras0d60a0f2018-06-29 02:02:08 -0700969 {
Florin Corase96bf632018-12-18 22:44:27 -0800970 e->postponed += 1;
Florin Coras5a7ca7b2018-10-30 12:01:48 -0700971 vec_add1 (wrk->pending_disconnects, *e);
Florin Coras0d60a0f2018-06-29 02:02:08 -0700972 continue;
973 }
974
Florin Coras5a2ec8f2018-12-27 11:53:11 -0800975 session_transport_close (s);
Florin Coras0d60a0f2018-06-29 02:02:08 -0700976 break;
Florin Corasf6c43132019-03-01 12:41:21 -0800977 case SESSION_IO_EVT_BUILTIN_RX:
Florin Coras0d60a0f2018-06-29 02:02:08 -0700978 s = session_event_get_session (e, thread_index);
Florin Corasef915342018-09-29 10:23:06 -0700979 if (PREDICT_FALSE (!s || s->session_state >= SESSION_STATE_CLOSING))
Florin Coras0d60a0f2018-06-29 02:02:08 -0700980 continue;
Florin Coras288eaab2019-02-03 15:26:14 -0800981 svm_fifo_unset_event (s->rx_fifo);
Florin Coras15531972018-08-12 23:50:53 -0700982 app_wrk = app_worker_get (s->app_wrk_index);
Florin Corasbf7ce2c2019-03-06 14:44:42 -0800983 app_worker_builtin_rx (app_wrk, s);
Florin Coras0d60a0f2018-06-29 02:02:08 -0700984 break;
Florin Corasf6c43132019-03-01 12:41:21 -0800985 case SESSION_IO_EVT_BUILTIN_TX:
Florin Corasef915342018-09-29 10:23:06 -0700986 s = session_get_from_handle_if_valid (e->session_handle);
Florin Coras1bcad5c2019-01-09 20:04:38 -0800987 wrk->ctx.s = s;
Florin Corasef915342018-09-29 10:23:06 -0700988 if (PREDICT_TRUE (s != 0))
Florin Coras1bcad5c2019-01-09 20:04:38 -0800989 session_tx_fifo_dequeue_internal (vm, node, wrk, e,
990 &n_tx_packets);
Florin Corasef915342018-09-29 10:23:06 -0700991 break;
Florin Corasf6c43132019-03-01 12:41:21 -0800992 case SESSION_CTRL_EVT_RPC:
Florin Coras0d60a0f2018-06-29 02:02:08 -0700993 fp = e->rpc_args.fp;
994 (*fp) (e->rpc_args.arg);
995 break;
Florin Coras52207f12018-07-12 14:48:06 -0700996 case SESSION_CTRL_EVT_DISCONNECTED:
997 session_mq_disconnected_handler (e->data);
998 break;
999 case SESSION_CTRL_EVT_ACCEPTED_REPLY:
1000 session_mq_accepted_reply_handler (e->data);
1001 break;
1002 case SESSION_CTRL_EVT_CONNECTED_REPLY:
1003 break;
1004 case SESSION_CTRL_EVT_DISCONNECTED_REPLY:
1005 session_mq_disconnected_reply_handler (e->data);
1006 break;
1007 case SESSION_CTRL_EVT_RESET_REPLY:
1008 session_mq_reset_reply_handler (e->data);
1009 break;
Florin Coras30e79c22019-01-02 19:31:22 -08001010 case SESSION_CTRL_EVT_WORKER_UPDATE:
1011 session_mq_worker_update_handler (e->data);
1012 break;
Florin Coras0d60a0f2018-06-29 02:02:08 -07001013 default:
1014 clib_warning ("unhandled event type %d", e->event_type);
1015 }
1016 }
1017
1018 _vec_len (fifo_events) = 0;
Florin Coras5a7ca7b2018-10-30 12:01:48 -07001019 wrk->free_event_vector = fifo_events;
Florin Corasc44a5582018-11-01 16:30:54 -07001020 wrk->last_tx_packets = n_tx_packets;
Florin Coras0d60a0f2018-06-29 02:02:08 -07001021
1022 vlib_node_increment_counter (vm, session_queue_node.index,
1023 SESSION_QUEUE_ERROR_TX, n_tx_packets);
1024
1025 SESSION_EVT_DBG (SESSION_EVT_DISPATCH_END, smm, thread_index);
1026
1027 return n_tx_packets;
1028}
1029
1030/* *INDENT-OFF* */
1031VLIB_REGISTER_NODE (session_queue_node) =
1032{
1033 .function = session_queue_node_fn,
1034 .name = "session-queue",
1035 .format_trace = format_session_queue_trace,
1036 .type = VLIB_NODE_TYPE_INPUT,
1037 .n_errors = ARRAY_LEN (session_queue_error_strings),
1038 .error_strings = session_queue_error_strings,
1039 .state = VLIB_NODE_STATE_DISABLED,
1040};
1041/* *INDENT-ON* */
1042
Dave Barachacd2a6a2017-05-16 17:41:34 -04001043void
1044dump_thread_0_event_queue (void)
1045{
Florin Coras31c99552019-03-01 13:00:58 -08001046 session_main_t *smm = vnet_get_session_main ();
Dave Barachacd2a6a2017-05-16 17:41:34 -04001047 vlib_main_t *vm = &vlib_global_main;
1048 u32 my_thread_index = vm->thread_index;
Florin Coras52207f12018-07-12 14:48:06 -07001049 session_event_t _e, *e = &_e;
Florin Coras3c2fed52018-07-04 04:15:05 -07001050 svm_msg_q_ring_t *ring;
Florin Coras288eaab2019-02-03 15:26:14 -08001051 session_t *s0;
Florin Coras3c2fed52018-07-04 04:15:05 -07001052 svm_msg_q_msg_t *msg;
1053 svm_msg_q_t *mq;
Dave Barachacd2a6a2017-05-16 17:41:34 -04001054 int i, index;
Dave Barachacd2a6a2017-05-16 17:41:34 -04001055
Florin Coras5a7ca7b2018-10-30 12:01:48 -07001056 mq = smm->wrk[my_thread_index].vpp_event_queue;
Florin Coras3c2fed52018-07-04 04:15:05 -07001057 index = mq->q->head;
Dave Barachacd2a6a2017-05-16 17:41:34 -04001058
Florin Coras3c2fed52018-07-04 04:15:05 -07001059 for (i = 0; i < mq->q->cursize; i++)
Dave Barachacd2a6a2017-05-16 17:41:34 -04001060 {
Florin Coras3c2fed52018-07-04 04:15:05 -07001061 msg = (svm_msg_q_msg_t *) (&mq->q->data[0] + mq->q->elsize * index);
1062 ring = svm_msg_q_ring (mq, msg->ring_index);
Dave Barach178cf492018-11-13 16:34:13 -05001063 clib_memcpy_fast (e, svm_msg_q_msg_data (mq, msg), ring->elsize);
Dave Barachacd2a6a2017-05-16 17:41:34 -04001064
1065 switch (e->event_type)
1066 {
Florin Corasf6c43132019-03-01 12:41:21 -08001067 case SESSION_IO_EVT_TX:
Dave Barachacd2a6a2017-05-16 17:41:34 -04001068 s0 = session_event_get_session (e, my_thread_index);
1069 fformat (stdout, "[%04d] TX session %d\n", i, s0->session_index);
1070 break;
1071
Florin Corasf6c43132019-03-01 12:41:21 -08001072 case SESSION_CTRL_EVT_CLOSE:
Florin Corascea194d2017-10-02 00:18:51 -07001073 s0 = session_get_from_handle (e->session_handle);
Dave Barachacd2a6a2017-05-16 17:41:34 -04001074 fformat (stdout, "[%04d] disconnect session %d\n", i,
1075 s0->session_index);
1076 break;
1077
Florin Corasf6c43132019-03-01 12:41:21 -08001078 case SESSION_IO_EVT_BUILTIN_RX:
Dave Barachacd2a6a2017-05-16 17:41:34 -04001079 s0 = session_event_get_session (e, my_thread_index);
1080 fformat (stdout, "[%04d] builtin_rx %d\n", i, s0->session_index);
1081 break;
1082
Florin Corasf6c43132019-03-01 12:41:21 -08001083 case SESSION_CTRL_EVT_RPC:
Dave Barachacd2a6a2017-05-16 17:41:34 -04001084 fformat (stdout, "[%04d] RPC call %llx with %llx\n",
David Johnsond9818dd2018-12-14 14:53:41 -05001085 i, (u64) (uword) (e->rpc_args.fp),
1086 (u64) (uword) (e->rpc_args.arg));
Dave Barachacd2a6a2017-05-16 17:41:34 -04001087 break;
1088
1089 default:
1090 fformat (stdout, "[%04d] unhandled event type %d\n",
1091 i, e->event_type);
1092 break;
1093 }
1094
1095 index++;
1096
Florin Coras3c2fed52018-07-04 04:15:05 -07001097 if (index == mq->q->maxsize)
Dave Barachacd2a6a2017-05-16 17:41:34 -04001098 index = 0;
1099 }
1100}
1101
Florin Coras6534b7a2017-07-18 05:38:03 -04001102static u8
Florin Coras52207f12018-07-12 14:48:06 -07001103session_node_cmp_event (session_event_t * e, svm_fifo_t * f)
Florin Coras6534b7a2017-07-18 05:38:03 -04001104{
Florin Coras288eaab2019-02-03 15:26:14 -08001105 session_t *s;
Florin Coras6534b7a2017-07-18 05:38:03 -04001106 switch (e->event_type)
1107 {
Florin Corasf6c43132019-03-01 12:41:21 -08001108 case SESSION_IO_EVT_RX:
1109 case SESSION_IO_EVT_TX:
1110 case SESSION_IO_EVT_BUILTIN_RX:
Florin Corasc0737e92019-03-04 14:19:39 -08001111 if (e->session_index == f->master_session_index)
Florin Coras6534b7a2017-07-18 05:38:03 -04001112 return 1;
1113 break;
Florin Corasf6c43132019-03-01 12:41:21 -08001114 case SESSION_CTRL_EVT_CLOSE:
Florin Coras6534b7a2017-07-18 05:38:03 -04001115 break;
Florin Corasf6c43132019-03-01 12:41:21 -08001116 case SESSION_CTRL_EVT_RPC:
Florin Corascea194d2017-10-02 00:18:51 -07001117 s = session_get_from_handle (e->session_handle);
Florin Coras6534b7a2017-07-18 05:38:03 -04001118 if (!s)
1119 {
1120 clib_warning ("session has event but doesn't exist!");
1121 break;
1122 }
Florin Coras288eaab2019-02-03 15:26:14 -08001123 if (s->rx_fifo == f || s->tx_fifo == f)
Florin Coras6534b7a2017-07-18 05:38:03 -04001124 return 1;
1125 break;
1126 default:
1127 break;
1128 }
1129 return 0;
1130}
1131
1132u8
Florin Coras52207f12018-07-12 14:48:06 -07001133session_node_lookup_fifo_event (svm_fifo_t * f, session_event_t * e)
Florin Coras6534b7a2017-07-18 05:38:03 -04001134{
Florin Coras52207f12018-07-12 14:48:06 -07001135 session_event_t *pending_event_vector, *evt;
Florin Coras31c99552019-03-01 13:00:58 -08001136 session_worker_t *wrk;
Florin Coras6534b7a2017-07-18 05:38:03 -04001137 int i, index, found = 0;
Florin Coras3c2fed52018-07-04 04:15:05 -07001138 svm_msg_q_msg_t *msg;
1139 svm_msg_q_ring_t *ring;
Florin Coras5a7ca7b2018-10-30 12:01:48 -07001140 svm_msg_q_t *mq;
Florin Coras6534b7a2017-07-18 05:38:03 -04001141 u8 thread_index;
1142
1143 ASSERT (e);
1144 thread_index = f->master_thread_index;
Florin Coras31c99552019-03-01 13:00:58 -08001145 wrk = session_main_get_worker (thread_index);
Florin Coras5a7ca7b2018-10-30 12:01:48 -07001146
Florin Coras6534b7a2017-07-18 05:38:03 -04001147 /*
1148 * Search evt queue
1149 */
Florin Coras5a7ca7b2018-10-30 12:01:48 -07001150 mq = wrk->vpp_event_queue;
Florin Coras3c2fed52018-07-04 04:15:05 -07001151 index = mq->q->head;
1152 for (i = 0; i < mq->q->cursize; i++)
Florin Coras6534b7a2017-07-18 05:38:03 -04001153 {
Florin Coras3c2fed52018-07-04 04:15:05 -07001154 msg = (svm_msg_q_msg_t *) (&mq->q->data[0] + mq->q->elsize * index);
1155 ring = svm_msg_q_ring (mq, msg->ring_index);
Dave Barach178cf492018-11-13 16:34:13 -05001156 clib_memcpy_fast (e, svm_msg_q_msg_data (mq, msg), ring->elsize);
Florin Coras6534b7a2017-07-18 05:38:03 -04001157 found = session_node_cmp_event (e, f);
1158 if (found)
Florin Coras371ca502018-02-21 12:07:41 -08001159 return 1;
Florin Coras3c2fed52018-07-04 04:15:05 -07001160 if (++index == mq->q->maxsize)
Florin Coras6534b7a2017-07-18 05:38:03 -04001161 index = 0;
1162 }
1163 /*
1164 * Search pending events vector
1165 */
Florin Coras5a7ca7b2018-10-30 12:01:48 -07001166 pending_event_vector = wrk->pending_event_vector;
Florin Coras6534b7a2017-07-18 05:38:03 -04001167 vec_foreach (evt, pending_event_vector)
1168 {
1169 found = session_node_cmp_event (evt, f);
1170 if (found)
1171 {
Dave Barach178cf492018-11-13 16:34:13 -05001172 clib_memcpy_fast (e, evt, sizeof (*evt));
Florin Coras6534b7a2017-07-18 05:38:03 -04001173 break;
1174 }
1175 }
1176 return found;
1177}
1178
Dave Barach2a863912017-11-28 10:11:42 -05001179static clib_error_t *
1180session_queue_exit (vlib_main_t * vm)
1181{
1182 if (vec_len (vlib_mains) < 2)
1183 return 0;
1184
1185 /*
1186 * Shut off (especially) worker-thread session nodes.
1187 * Otherwise, vpp can crash as the main thread unmaps the
1188 * API segment.
1189 */
1190 vlib_worker_thread_barrier_sync (vm);
1191 session_node_enable_disable (0 /* is_enable */ );
1192 vlib_worker_thread_barrier_release (vm);
1193 return 0;
1194}
1195
1196VLIB_MAIN_LOOP_EXIT_FUNCTION (session_queue_exit);
1197
Florin Corasfd542f12018-05-16 19:28:24 -07001198static uword
1199session_queue_process (vlib_main_t * vm, vlib_node_runtime_t * rt,
1200 vlib_frame_t * f)
1201{
1202 f64 now, timeout = 1.0;
1203 uword *event_data = 0;
1204 uword event_type;
1205
1206 while (1)
1207 {
1208 vlib_process_wait_for_event_or_clock (vm, timeout);
1209 now = vlib_time_now (vm);
1210 event_type = vlib_process_get_events (vm, (uword **) & event_data);
1211
1212 switch (event_type)
1213 {
1214 case SESSION_Q_PROCESS_FLUSH_FRAMES:
1215 /* Flush the frames by updating all transports times */
1216 transport_update_time (now, 0);
1217 break;
1218 case SESSION_Q_PROCESS_STOP:
1219 timeout = 100000.0;
1220 break;
1221 case ~0:
1222 /* Timed out. Update time for all transports to trigger all
1223 * outstanding retransmits. */
1224 transport_update_time (now, 0);
1225 break;
1226 }
1227 vec_reset_length (event_data);
1228 }
1229 return 0;
1230}
1231
1232/* *INDENT-OFF* */
1233VLIB_REGISTER_NODE (session_queue_process_node) =
1234{
1235 .function = session_queue_process,
1236 .type = VLIB_NODE_TYPE_PROCESS,
1237 .name = "session-queue-process",
1238 .state = VLIB_NODE_STATE_DISABLED,
1239};
1240/* *INDENT-ON* */
1241
Florin Coras653e43f2019-03-04 10:56:23 -08001242static_always_inline uword
1243session_queue_pre_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
1244 vlib_frame_t * frame)
1245{
1246 session_main_t *sm = &session_main;
1247 if (!sm->wrk[0].vpp_event_queue)
1248 return 0;
1249 return session_queue_node_fn (vm, node, frame);
1250}
1251
1252/* *INDENT-OFF* */
1253VLIB_REGISTER_NODE (session_queue_pre_input_node) =
1254{
1255 .function = session_queue_pre_input_inline,
1256 .type = VLIB_NODE_TYPE_PRE_INPUT,
1257 .name = "session-queue-main",
1258 .state = VLIB_NODE_STATE_DISABLED,
1259};
1260/* *INDENT-ON* */
1261
Dave Barach68b0fb02017-02-28 15:15:56 -05001262/*
1263 * fd.io coding-style-patch-verification: ON
1264 *
1265 * Local Variables:
1266 * eval: (c-set-style "gnu")
1267 * End:
1268 */