Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 1 | /* |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 2 | * Copyright (c) 2017-2019 Cisco and/or its affiliates. |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 3 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | * you may not use this file except in compliance with the License. |
| 5 | * You may obtain a copy of the License at: |
| 6 | * |
| 7 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | * |
| 9 | * Unless required by applicable law or agreed to in writing, software |
| 10 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | * See the License for the specific language governing permissions and |
| 13 | * limitations under the License. |
| 14 | */ |
| 15 | |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 16 | #include <math.h> |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 17 | #include <vlib/vlib.h> |
| 18 | #include <vnet/vnet.h> |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 19 | #include <vppinfra/elog.h> |
Florin Coras | 561af9b | 2017-12-09 10:19:43 -0800 | [diff] [blame] | 20 | #include <vnet/session/transport.h> |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 21 | #include <vnet/session/session.h> |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 22 | #include <vnet/session/application.h> |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 23 | #include <vnet/session/application_interface.h> |
Florin Coras | ba7d8f5 | 2019-02-22 13:11:38 -0800 | [diff] [blame] | 24 | #include <vnet/session/application_local.h> |
Florin Coras | e69f495 | 2017-03-07 10:06:24 -0800 | [diff] [blame] | 25 | #include <vnet/session/session_debug.h> |
Florin Coras | e86a8ed | 2018-01-05 03:20:25 -0800 | [diff] [blame] | 26 | #include <svm/queue.h> |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 27 | |
Florin Coras | 2b81e3c | 2019-02-27 07:55:46 -0800 | [diff] [blame] | 28 | static void session_mq_accepted_reply_handler (void *data); |
| 29 | |
| 30 | static void |
| 31 | accepted_notify_cb (void *data, u32 data_len) |
| 32 | { |
| 33 | session_mq_accepted_reply_handler (data); |
| 34 | } |
| 35 | |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 36 | static void |
| 37 | session_mq_accepted_reply_handler (void *data) |
| 38 | { |
| 39 | session_accepted_reply_msg_t *mp = (session_accepted_reply_msg_t *) data; |
| 40 | vnet_disconnect_args_t _a = { 0 }, *a = &_a; |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 41 | session_state_t old_state; |
Florin Coras | 2179513 | 2018-09-09 09:40:51 -0700 | [diff] [blame] | 42 | app_worker_t *app_wrk; |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 43 | session_t *s; |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 44 | |
| 45 | /* Server isn't interested, kill the session */ |
| 46 | if (mp->retval) |
| 47 | { |
| 48 | a->app_index = mp->context; |
| 49 | a->handle = mp->handle; |
| 50 | vnet_disconnect_session (a); |
| 51 | return; |
| 52 | } |
| 53 | |
Florin Coras | 2b81e3c | 2019-02-27 07:55:46 -0800 | [diff] [blame] | 54 | /* Mail this back from the main thread. We're not polling in main |
| 55 | * thread so we're using other workers for notifications. */ |
| 56 | if (vlib_num_workers () && vlib_get_thread_index () != 0 |
| 57 | && session_thread_from_handle (mp->handle) == 0) |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 58 | { |
Florin Coras | 2b81e3c | 2019-02-27 07:55:46 -0800 | [diff] [blame] | 59 | vl_api_rpc_call_main_thread (accepted_notify_cb, data, |
| 60 | sizeof (session_accepted_reply_msg_t)); |
| 61 | return; |
| 62 | } |
| 63 | |
| 64 | s = session_get_from_handle_if_valid (mp->handle); |
| 65 | if (!s) |
| 66 | return; |
| 67 | |
| 68 | app_wrk = app_worker_get (s->app_wrk_index); |
| 69 | if (app_wrk->app_index != mp->context) |
| 70 | { |
| 71 | clib_warning ("app doesn't own session"); |
| 72 | return; |
| 73 | } |
| 74 | |
| 75 | if (!session_has_transport (s)) |
| 76 | { |
Florin Coras | 653e43f | 2019-03-04 10:56:23 -0800 | [diff] [blame] | 77 | s->session_state = SESSION_STATE_READY; |
Florin Coras | 2b81e3c | 2019-02-27 07:55:46 -0800 | [diff] [blame] | 78 | if (ct_session_connect_notify (s)) |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 79 | return; |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 80 | } |
| 81 | else |
| 82 | { |
Florin Coras | b0f662f | 2018-12-27 14:51:46 -0800 | [diff] [blame] | 83 | old_state = s->session_state; |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 84 | s->session_state = SESSION_STATE_READY; |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 85 | if (!svm_fifo_is_empty (s->rx_fifo)) |
Florin Coras | f6c4313 | 2019-03-01 12:41:21 -0800 | [diff] [blame] | 86 | app_worker_lock_and_send_event (app_wrk, s, SESSION_IO_EVT_RX); |
Florin Coras | b0f662f | 2018-12-27 14:51:46 -0800 | [diff] [blame] | 87 | |
| 88 | /* Closed while waiting for app to reply. Resend disconnect */ |
| 89 | if (old_state >= SESSION_STATE_TRANSPORT_CLOSING) |
| 90 | { |
Florin Coras | 69b68ef | 2019-04-02 11:38:51 -0700 | [diff] [blame] | 91 | app_worker_close_notify (app_wrk, s); |
Florin Coras | b0f662f | 2018-12-27 14:51:46 -0800 | [diff] [blame] | 92 | s->session_state = old_state; |
| 93 | return; |
| 94 | } |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 95 | } |
| 96 | } |
| 97 | |
| 98 | static void |
| 99 | session_mq_reset_reply_handler (void *data) |
| 100 | { |
Florin Coras | 4af830c | 2018-12-04 09:21:36 -0800 | [diff] [blame] | 101 | vnet_disconnect_args_t _a = { 0 }, *a = &_a; |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 102 | session_reset_reply_msg_t *mp; |
Florin Coras | 1553197 | 2018-08-12 23:50:53 -0700 | [diff] [blame] | 103 | app_worker_t *app_wrk; |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 104 | session_t *s; |
Florin Coras | 1553197 | 2018-08-12 23:50:53 -0700 | [diff] [blame] | 105 | application_t *app; |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 106 | u32 index, thread_index; |
| 107 | |
| 108 | mp = (session_reset_reply_msg_t *) data; |
Florin Coras | 3c7d4f9 | 2018-12-14 11:28:43 -0800 | [diff] [blame] | 109 | app = application_lookup (mp->context); |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 110 | if (!app) |
| 111 | return; |
| 112 | |
| 113 | session_parse_handle (mp->handle, &index, &thread_index); |
| 114 | s = session_get_if_valid (index, thread_index); |
Florin Coras | 3c7d4f9 | 2018-12-14 11:28:43 -0800 | [diff] [blame] | 115 | |
| 116 | /* Session was already closed or already cleaned up */ |
| 117 | if (!s || s->session_state != SESSION_STATE_TRANSPORT_CLOSING) |
| 118 | return; |
| 119 | |
Florin Coras | 1553197 | 2018-08-12 23:50:53 -0700 | [diff] [blame] | 120 | app_wrk = app_worker_get (s->app_wrk_index); |
| 121 | if (!app_wrk || app_wrk->app_index != app->app_index) |
| 122 | { |
| 123 | clib_warning ("App % does not own handle 0x%lx!", app->app_index, |
| 124 | mp->handle); |
| 125 | return; |
| 126 | } |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 127 | |
| 128 | /* Client objected to resetting the session, log and continue */ |
| 129 | if (mp->retval) |
| 130 | { |
| 131 | clib_warning ("client retval %d", mp->retval); |
| 132 | return; |
| 133 | } |
| 134 | |
| 135 | /* This comes as a response to a reset, transport only waiting for |
| 136 | * confirmation to remove connection state, no need to disconnect */ |
Florin Coras | 4af830c | 2018-12-04 09:21:36 -0800 | [diff] [blame] | 137 | a->handle = mp->handle; |
| 138 | a->app_index = app->app_index; |
| 139 | vnet_disconnect_session (a); |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 140 | } |
| 141 | |
| 142 | static void |
| 143 | session_mq_disconnected_handler (void *data) |
| 144 | { |
| 145 | session_disconnected_reply_msg_t *rmp; |
| 146 | vnet_disconnect_args_t _a, *a = &_a; |
| 147 | svm_msg_q_msg_t _msg, *msg = &_msg; |
| 148 | session_disconnected_msg_t *mp; |
Florin Coras | 1553197 | 2018-08-12 23:50:53 -0700 | [diff] [blame] | 149 | app_worker_t *app_wrk; |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 150 | session_event_t *evt; |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 151 | session_t *s; |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 152 | application_t *app; |
| 153 | int rv = 0; |
| 154 | |
| 155 | mp = (session_disconnected_msg_t *) data; |
Florin Coras | c3638fe | 2018-08-24 13:58:49 -0700 | [diff] [blame] | 156 | if (!(s = session_get_from_handle_if_valid (mp->handle))) |
| 157 | { |
| 158 | clib_warning ("could not disconnect handle %llu", mp->handle); |
| 159 | return; |
| 160 | } |
Florin Coras | 1553197 | 2018-08-12 23:50:53 -0700 | [diff] [blame] | 161 | app_wrk = app_worker_get (s->app_wrk_index); |
| 162 | app = application_lookup (mp->client_index); |
Florin Coras | c3638fe | 2018-08-24 13:58:49 -0700 | [diff] [blame] | 163 | if (!(app_wrk && app && app->app_index == app_wrk->app_index)) |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 164 | { |
Florin Coras | c3638fe | 2018-08-24 13:58:49 -0700 | [diff] [blame] | 165 | clib_warning ("could not disconnect session: %llu app: %u", |
Florin Coras | 1553197 | 2018-08-12 23:50:53 -0700 | [diff] [blame] | 166 | mp->handle, mp->client_index); |
Florin Coras | 3d0fadc | 2018-07-17 05:35:47 -0700 | [diff] [blame] | 167 | return; |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 168 | } |
Florin Coras | 3d0fadc | 2018-07-17 05:35:47 -0700 | [diff] [blame] | 169 | |
| 170 | a->handle = mp->handle; |
Florin Coras | 1553197 | 2018-08-12 23:50:53 -0700 | [diff] [blame] | 171 | a->app_index = app_wrk->wrk_index; |
Florin Coras | 3d0fadc | 2018-07-17 05:35:47 -0700 | [diff] [blame] | 172 | rv = vnet_disconnect_session (a); |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 173 | |
Florin Coras | 1553197 | 2018-08-12 23:50:53 -0700 | [diff] [blame] | 174 | svm_msg_q_lock_and_alloc_msg_w_ring (app_wrk->event_queue, |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 175 | SESSION_MQ_CTRL_EVT_RING, |
| 176 | SVM_Q_WAIT, msg); |
Florin Coras | 1553197 | 2018-08-12 23:50:53 -0700 | [diff] [blame] | 177 | svm_msg_q_unlock (app_wrk->event_queue); |
| 178 | evt = svm_msg_q_msg_data (app_wrk->event_queue, msg); |
Dave Barach | b7b9299 | 2018-10-17 10:38:51 -0400 | [diff] [blame] | 179 | clib_memset (evt, 0, sizeof (*evt)); |
Florin Coras | 30e79c2 | 2019-01-02 19:31:22 -0800 | [diff] [blame] | 180 | evt->event_type = SESSION_CTRL_EVT_DISCONNECTED_REPLY; |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 181 | rmp = (session_disconnected_reply_msg_t *) evt->data; |
| 182 | rmp->handle = mp->handle; |
| 183 | rmp->context = mp->context; |
| 184 | rmp->retval = rv; |
Florin Coras | 1553197 | 2018-08-12 23:50:53 -0700 | [diff] [blame] | 185 | svm_msg_q_add (app_wrk->event_queue, msg, SVM_Q_WAIT); |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 186 | } |
| 187 | |
| 188 | static void |
| 189 | session_mq_disconnected_reply_handler (void *data) |
| 190 | { |
| 191 | session_disconnected_reply_msg_t *mp; |
| 192 | vnet_disconnect_args_t _a, *a = &_a; |
| 193 | application_t *app; |
| 194 | |
| 195 | mp = (session_disconnected_reply_msg_t *) data; |
| 196 | |
| 197 | /* Client objected to disconnecting the session, log and continue */ |
| 198 | if (mp->retval) |
| 199 | { |
| 200 | clib_warning ("client retval %d", mp->retval); |
| 201 | return; |
| 202 | } |
| 203 | |
| 204 | /* Disconnect has been confirmed. Confirm close to transport */ |
| 205 | app = application_lookup (mp->context); |
| 206 | if (app) |
| 207 | { |
| 208 | a->handle = mp->handle; |
Florin Coras | 1553197 | 2018-08-12 23:50:53 -0700 | [diff] [blame] | 209 | a->app_index = app->app_index; |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 210 | vnet_disconnect_session (a); |
| 211 | } |
| 212 | } |
| 213 | |
Florin Coras | 30e79c2 | 2019-01-02 19:31:22 -0800 | [diff] [blame] | 214 | static void |
| 215 | session_mq_worker_update_handler (void *data) |
| 216 | { |
| 217 | session_worker_update_msg_t *mp = (session_worker_update_msg_t *) data; |
| 218 | session_worker_update_reply_msg_t *rmp; |
| 219 | svm_msg_q_msg_t _msg, *msg = &_msg; |
| 220 | app_worker_t *app_wrk; |
| 221 | u32 owner_app_wrk_map; |
| 222 | session_event_t *evt; |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 223 | session_t *s; |
Florin Coras | 30e79c2 | 2019-01-02 19:31:22 -0800 | [diff] [blame] | 224 | application_t *app; |
| 225 | |
| 226 | app = application_lookup (mp->client_index); |
| 227 | if (!app) |
| 228 | return; |
| 229 | if (!(s = session_get_from_handle_if_valid (mp->handle))) |
| 230 | { |
| 231 | clib_warning ("invalid handle %llu", mp->handle); |
| 232 | return; |
| 233 | } |
| 234 | app_wrk = app_worker_get (s->app_wrk_index); |
| 235 | if (app_wrk->app_index != app->app_index) |
| 236 | { |
| 237 | clib_warning ("app %u does not own session %llu", app->app_index, |
| 238 | mp->handle); |
| 239 | return; |
| 240 | } |
| 241 | owner_app_wrk_map = app_wrk->wrk_map_index; |
| 242 | app_wrk = application_get_worker (app, mp->wrk_index); |
| 243 | |
| 244 | /* This needs to come from the new owner */ |
| 245 | if (mp->req_wrk_index == owner_app_wrk_map) |
| 246 | { |
| 247 | session_req_worker_update_msg_t *wump; |
| 248 | |
| 249 | svm_msg_q_lock_and_alloc_msg_w_ring (app_wrk->event_queue, |
| 250 | SESSION_MQ_CTRL_EVT_RING, |
| 251 | SVM_Q_WAIT, msg); |
| 252 | svm_msg_q_unlock (app_wrk->event_queue); |
| 253 | evt = svm_msg_q_msg_data (app_wrk->event_queue, msg); |
| 254 | clib_memset (evt, 0, sizeof (*evt)); |
| 255 | evt->event_type = SESSION_CTRL_EVT_REQ_WORKER_UPDATE; |
| 256 | wump = (session_req_worker_update_msg_t *) evt->data; |
| 257 | wump->session_handle = mp->handle; |
| 258 | svm_msg_q_add (app_wrk->event_queue, msg, SVM_Q_WAIT); |
| 259 | return; |
| 260 | } |
| 261 | |
| 262 | app_worker_own_session (app_wrk, s); |
| 263 | |
| 264 | /* |
| 265 | * Send reply |
| 266 | */ |
| 267 | svm_msg_q_lock_and_alloc_msg_w_ring (app_wrk->event_queue, |
| 268 | SESSION_MQ_CTRL_EVT_RING, |
| 269 | SVM_Q_WAIT, msg); |
| 270 | svm_msg_q_unlock (app_wrk->event_queue); |
| 271 | evt = svm_msg_q_msg_data (app_wrk->event_queue, msg); |
| 272 | clib_memset (evt, 0, sizeof (*evt)); |
| 273 | evt->event_type = SESSION_CTRL_EVT_WORKER_UPDATE_REPLY; |
| 274 | rmp = (session_worker_update_reply_msg_t *) evt->data; |
| 275 | rmp->handle = mp->handle; |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 276 | rmp->rx_fifo = pointer_to_uword (s->rx_fifo); |
| 277 | rmp->tx_fifo = pointer_to_uword (s->tx_fifo); |
Florin Coras | 30e79c2 | 2019-01-02 19:31:22 -0800 | [diff] [blame] | 278 | rmp->segment_handle = session_segment_handle (s); |
| 279 | svm_msg_q_add (app_wrk->event_queue, msg, SVM_Q_WAIT); |
| 280 | |
| 281 | /* |
| 282 | * Retransmit messages that may have been lost |
| 283 | */ |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 284 | if (s->tx_fifo && !svm_fifo_is_empty (s->tx_fifo)) |
Florin Coras | f6c4313 | 2019-03-01 12:41:21 -0800 | [diff] [blame] | 285 | session_send_io_evt_to_thread (s->tx_fifo, SESSION_IO_EVT_TX); |
Florin Coras | 30e79c2 | 2019-01-02 19:31:22 -0800 | [diff] [blame] | 286 | |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 287 | if (s->rx_fifo && !svm_fifo_is_empty (s->rx_fifo)) |
Florin Coras | f6c4313 | 2019-03-01 12:41:21 -0800 | [diff] [blame] | 288 | app_worker_lock_and_send_event (app_wrk, s, SESSION_IO_EVT_RX); |
Florin Coras | 30e79c2 | 2019-01-02 19:31:22 -0800 | [diff] [blame] | 289 | |
| 290 | if (s->session_state >= SESSION_STATE_TRANSPORT_CLOSING) |
Florin Coras | 69b68ef | 2019-04-02 11:38:51 -0700 | [diff] [blame] | 291 | app_worker_close_notify (app_wrk, s); |
Florin Coras | 30e79c2 | 2019-01-02 19:31:22 -0800 | [diff] [blame] | 292 | } |
| 293 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 294 | vlib_node_registration_t session_queue_node; |
| 295 | |
| 296 | typedef struct |
| 297 | { |
| 298 | u32 session_index; |
| 299 | u32 server_thread_index; |
| 300 | } session_queue_trace_t; |
| 301 | |
| 302 | /* packet trace format function */ |
| 303 | static u8 * |
| 304 | format_session_queue_trace (u8 * s, va_list * args) |
| 305 | { |
| 306 | CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); |
| 307 | CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); |
| 308 | session_queue_trace_t *t = va_arg (*args, session_queue_trace_t *); |
| 309 | |
| 310 | s = format (s, "SESSION_QUEUE: session index %d, server thread index %d", |
| 311 | t->session_index, t->server_thread_index); |
| 312 | return s; |
| 313 | } |
| 314 | |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 315 | #define foreach_session_queue_error \ |
| 316 | _(TX, "Packets transmitted") \ |
Florin Coras | 93992a9 | 2017-05-24 18:03:56 -0700 | [diff] [blame] | 317 | _(TIMER, "Timer events") \ |
| 318 | _(NO_BUFFER, "Out of buffers") |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 319 | |
| 320 | typedef enum |
| 321 | { |
| 322 | #define _(sym,str) SESSION_QUEUE_ERROR_##sym, |
| 323 | foreach_session_queue_error |
| 324 | #undef _ |
| 325 | SESSION_QUEUE_N_ERROR, |
| 326 | } session_queue_error_t; |
| 327 | |
| 328 | static char *session_queue_error_strings[] = { |
| 329 | #define _(sym,string) string, |
| 330 | foreach_session_queue_error |
| 331 | #undef _ |
| 332 | }; |
| 333 | |
Florin Coras | 0d60a0f | 2018-06-29 02:02:08 -0700 | [diff] [blame] | 334 | enum |
| 335 | { |
| 336 | SESSION_TX_NO_BUFFERS = -2, |
| 337 | SESSION_TX_NO_DATA, |
| 338 | SESSION_TX_OK |
| 339 | }; |
| 340 | |
Florin Coras | 66cf5e0 | 2018-06-08 09:17:39 -0700 | [diff] [blame] | 341 | static void |
| 342 | session_tx_trace_frame (vlib_main_t * vm, vlib_node_runtime_t * node, |
| 343 | u32 next_index, u32 * to_next, u16 n_segs, |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 344 | session_t * s, u32 n_trace) |
Florin Coras | f6d68ed | 2017-05-07 19:12:02 -0700 | [diff] [blame] | 345 | { |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 346 | session_queue_trace_t *t; |
Florin Coras | 66cf5e0 | 2018-06-08 09:17:39 -0700 | [diff] [blame] | 347 | vlib_buffer_t *b; |
| 348 | int i; |
| 349 | |
| 350 | for (i = 0; i < clib_min (n_trace, n_segs); i++) |
| 351 | { |
| 352 | b = vlib_get_buffer (vm, to_next[i - n_segs]); |
| 353 | vlib_trace_buffer (vm, node, next_index, b, 1 /* follow_chain */ ); |
Florin Coras | 66cf5e0 | 2018-06-08 09:17:39 -0700 | [diff] [blame] | 354 | t = vlib_add_trace (vm, node, b, sizeof (*t)); |
| 355 | t->session_index = s->session_index; |
| 356 | t->server_thread_index = s->thread_index; |
| 357 | } |
Florin Coras | 4df3871 | 2018-06-20 12:44:16 -0700 | [diff] [blame] | 358 | vlib_set_trace_count (vm, node, n_trace - i); |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 359 | } |
Florin Coras | f6d68ed | 2017-05-07 19:12:02 -0700 | [diff] [blame] | 360 | |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 361 | always_inline void |
| 362 | session_tx_fifo_chain_tail (vlib_main_t * vm, session_tx_context_t * ctx, |
| 363 | vlib_buffer_t * b, u16 * n_bufs, u8 peek_data) |
| 364 | { |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 365 | vlib_buffer_t *chain_b, *prev_b; |
| 366 | u32 chain_bi0, to_deq, left_from_seg; |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 367 | session_worker_t *wrk; |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 368 | u16 len_to_deq, n_bytes_read; |
| 369 | u8 *data, j; |
Florin Coras | b2215d6 | 2017-08-01 16:56:58 -0700 | [diff] [blame] | 370 | |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 371 | wrk = session_main_get_worker (ctx->s->thread_index); |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 372 | b->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; |
| 373 | b->total_length_not_including_first_buffer = 0; |
| 374 | |
| 375 | chain_b = b; |
| 376 | left_from_seg = clib_min (ctx->snd_mss - b->current_length, |
| 377 | ctx->left_to_snd); |
Florin Coras | b2215d6 | 2017-08-01 16:56:58 -0700 | [diff] [blame] | 378 | to_deq = left_from_seg; |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 379 | for (j = 1; j < ctx->n_bufs_per_seg; j++) |
Florin Coras | f6d68ed | 2017-05-07 19:12:02 -0700 | [diff] [blame] | 380 | { |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 381 | prev_b = chain_b; |
| 382 | len_to_deq = clib_min (to_deq, ctx->deq_per_buf); |
Florin Coras | f6d68ed | 2017-05-07 19:12:02 -0700 | [diff] [blame] | 383 | |
| 384 | *n_bufs -= 1; |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 385 | chain_bi0 = wrk->tx_buffers[*n_bufs]; |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 386 | chain_b = vlib_get_buffer (vm, chain_bi0); |
| 387 | chain_b->current_data = 0; |
| 388 | data = vlib_buffer_get_current (chain_b); |
Florin Coras | f6d68ed | 2017-05-07 19:12:02 -0700 | [diff] [blame] | 389 | if (peek_data) |
| 390 | { |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 391 | n_bytes_read = svm_fifo_peek (ctx->s->tx_fifo, |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 392 | ctx->tx_offset, len_to_deq, data); |
| 393 | ctx->tx_offset += n_bytes_read; |
Florin Coras | f6d68ed | 2017-05-07 19:12:02 -0700 | [diff] [blame] | 394 | } |
| 395 | else |
| 396 | { |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 397 | if (ctx->transport_vft->tx_type == TRANSPORT_TX_DGRAM) |
Florin Coras | 7fb0fe1 | 2018-04-09 09:24:52 -0700 | [diff] [blame] | 398 | { |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 399 | svm_fifo_t *f = ctx->s->tx_fifo; |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 400 | session_dgram_hdr_t *hdr = &ctx->hdr; |
Florin Coras | 7fb0fe1 | 2018-04-09 09:24:52 -0700 | [diff] [blame] | 401 | u16 deq_now; |
Florin Coras | 7fb0fe1 | 2018-04-09 09:24:52 -0700 | [diff] [blame] | 402 | deq_now = clib_min (hdr->data_length - hdr->data_offset, |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 403 | len_to_deq); |
| 404 | n_bytes_read = svm_fifo_peek (f, hdr->data_offset, deq_now, |
| 405 | data); |
Florin Coras | 7fb0fe1 | 2018-04-09 09:24:52 -0700 | [diff] [blame] | 406 | ASSERT (n_bytes_read > 0); |
| 407 | |
| 408 | hdr->data_offset += n_bytes_read; |
| 409 | if (hdr->data_offset == hdr->data_length) |
shubing guo | aea5f39 | 2018-08-30 13:29:49 +0800 | [diff] [blame] | 410 | { |
| 411 | u32 offset = hdr->data_length + SESSION_CONN_HDR_LEN; |
| 412 | svm_fifo_dequeue_drop (f, offset); |
| 413 | } |
Florin Coras | 7fb0fe1 | 2018-04-09 09:24:52 -0700 | [diff] [blame] | 414 | } |
| 415 | else |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 416 | n_bytes_read = svm_fifo_dequeue_nowait (ctx->s->tx_fifo, |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 417 | len_to_deq, data); |
Florin Coras | f6d68ed | 2017-05-07 19:12:02 -0700 | [diff] [blame] | 418 | } |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 419 | ASSERT (n_bytes_read == len_to_deq); |
| 420 | chain_b->current_length = n_bytes_read; |
| 421 | b->total_length_not_including_first_buffer += chain_b->current_length; |
Florin Coras | f6d68ed | 2017-05-07 19:12:02 -0700 | [diff] [blame] | 422 | |
| 423 | /* update previous buffer */ |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 424 | prev_b->next_buffer = chain_bi0; |
| 425 | prev_b->flags |= VLIB_BUFFER_NEXT_PRESENT; |
Florin Coras | f6d68ed | 2017-05-07 19:12:02 -0700 | [diff] [blame] | 426 | |
| 427 | /* update current buffer */ |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 428 | chain_b->next_buffer = 0; |
Florin Coras | f6d68ed | 2017-05-07 19:12:02 -0700 | [diff] [blame] | 429 | |
Florin Coras | b2215d6 | 2017-08-01 16:56:58 -0700 | [diff] [blame] | 430 | to_deq -= n_bytes_read; |
| 431 | if (to_deq == 0) |
Florin Coras | f6d68ed | 2017-05-07 19:12:02 -0700 | [diff] [blame] | 432 | break; |
| 433 | } |
Florin Coras | 1f152cd | 2017-08-18 19:28:03 -0700 | [diff] [blame] | 434 | ASSERT (to_deq == 0 |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 435 | && b->total_length_not_including_first_buffer == left_from_seg); |
| 436 | ctx->left_to_snd -= left_from_seg; |
Florin Coras | f6d68ed | 2017-05-07 19:12:02 -0700 | [diff] [blame] | 437 | } |
| 438 | |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 439 | always_inline void |
| 440 | session_tx_fill_buffer (vlib_main_t * vm, session_tx_context_t * ctx, |
| 441 | vlib_buffer_t * b, u16 * n_bufs, u8 peek_data) |
| 442 | { |
| 443 | u32 len_to_deq; |
| 444 | u8 *data0; |
| 445 | int n_bytes_read; |
| 446 | |
| 447 | /* |
| 448 | * Start with the first buffer in chain |
| 449 | */ |
| 450 | b->error = 0; |
| 451 | b->flags = VNET_BUFFER_F_LOCALLY_ORIGINATED; |
| 452 | b->current_data = 0; |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 453 | |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 454 | data0 = vlib_buffer_make_headroom (b, TRANSPORT_MAX_HDRS_LEN); |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 455 | len_to_deq = clib_min (ctx->left_to_snd, ctx->deq_per_first_buf); |
| 456 | |
Florin Coras | 7fb0fe1 | 2018-04-09 09:24:52 -0700 | [diff] [blame] | 457 | if (peek_data) |
| 458 | { |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 459 | n_bytes_read = svm_fifo_peek (ctx->s->tx_fifo, ctx->tx_offset, |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 460 | len_to_deq, data0); |
| 461 | ASSERT (n_bytes_read > 0); |
| 462 | /* Keep track of progress locally, transport is also supposed to |
| 463 | * increment it independently when pushing the header */ |
| 464 | ctx->tx_offset += n_bytes_read; |
Florin Coras | 7fb0fe1 | 2018-04-09 09:24:52 -0700 | [diff] [blame] | 465 | } |
| 466 | else |
| 467 | { |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 468 | if (ctx->transport_vft->tx_type == TRANSPORT_TX_DGRAM) |
| 469 | { |
| 470 | session_dgram_hdr_t *hdr = &ctx->hdr; |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 471 | svm_fifo_t *f = ctx->s->tx_fifo; |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 472 | u16 deq_now; |
| 473 | u32 offset; |
| 474 | |
| 475 | ASSERT (hdr->data_length > hdr->data_offset); |
| 476 | deq_now = clib_min (hdr->data_length - hdr->data_offset, |
| 477 | len_to_deq); |
| 478 | offset = hdr->data_offset + SESSION_CONN_HDR_LEN; |
| 479 | n_bytes_read = svm_fifo_peek (f, offset, deq_now, data0); |
| 480 | ASSERT (n_bytes_read > 0); |
| 481 | |
| 482 | if (ctx->s->session_state == SESSION_STATE_LISTENING) |
| 483 | { |
| 484 | ip_copy (&ctx->tc->rmt_ip, &hdr->rmt_ip, ctx->tc->is_ip4); |
| 485 | ctx->tc->rmt_port = hdr->rmt_port; |
| 486 | } |
| 487 | hdr->data_offset += n_bytes_read; |
| 488 | if (hdr->data_offset == hdr->data_length) |
| 489 | { |
| 490 | offset = hdr->data_length + SESSION_CONN_HDR_LEN; |
| 491 | svm_fifo_dequeue_drop (f, offset); |
| 492 | } |
| 493 | } |
Florin Coras | 7fb0fe1 | 2018-04-09 09:24:52 -0700 | [diff] [blame] | 494 | else |
| 495 | { |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 496 | n_bytes_read = svm_fifo_dequeue_nowait (ctx->s->tx_fifo, |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 497 | len_to_deq, data0); |
| 498 | ASSERT (n_bytes_read > 0); |
Florin Coras | 7fb0fe1 | 2018-04-09 09:24:52 -0700 | [diff] [blame] | 499 | } |
| 500 | } |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 501 | b->current_length = n_bytes_read; |
| 502 | ctx->left_to_snd -= n_bytes_read; |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 503 | |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 504 | /* |
| 505 | * Fill in the remaining buffers in the chain, if any |
| 506 | */ |
| 507 | if (PREDICT_FALSE (ctx->n_bufs_per_seg > 1 && ctx->left_to_snd)) |
| 508 | session_tx_fifo_chain_tail (vm, ctx, b, n_bufs, peek_data); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 509 | |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 510 | /* *INDENT-OFF* */ |
Florin Coras | 8b20bf5 | 2018-06-14 14:55:50 -0700 | [diff] [blame] | 511 | SESSION_EVT_DBG(SESSION_EVT_DEQ, ctx->s, ({ |
Florin Coras | f6c4313 | 2019-03-01 12:41:21 -0800 | [diff] [blame] | 512 | ed->data[0] = SESSION_IO_EVT_TX; |
Florin Coras | 8b20bf5 | 2018-06-14 14:55:50 -0700 | [diff] [blame] | 513 | ed->data[1] = ctx->max_dequeue; |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 514 | ed->data[2] = len_to_deq; |
Florin Coras | 8b20bf5 | 2018-06-14 14:55:50 -0700 | [diff] [blame] | 515 | ed->data[3] = ctx->left_to_snd; |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 516 | })); |
| 517 | /* *INDENT-ON* */ |
| 518 | } |
| 519 | |
| 520 | always_inline u8 |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 521 | session_tx_not_ready (session_t * s, u8 peek_data) |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 522 | { |
| 523 | if (peek_data) |
Florin Coras | e04c299 | 2017-03-01 08:17:34 -0800 | [diff] [blame] | 524 | { |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 525 | /* Can retransmit for closed sessions but can't send new data if |
| 526 | * session is not ready or closed */ |
| 527 | if (s->session_state < SESSION_STATE_READY) |
| 528 | return 1; |
Florin Coras | 5a2ec8f | 2018-12-27 11:53:11 -0800 | [diff] [blame] | 529 | if (s->session_state >= SESSION_STATE_TRANSPORT_CLOSED) |
Florin Coras | ca1c8f3 | 2018-05-23 21:01:30 -0700 | [diff] [blame] | 530 | return 2; |
Florin Coras | e04c299 | 2017-03-01 08:17:34 -0800 | [diff] [blame] | 531 | } |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 532 | return 0; |
| 533 | } |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 534 | |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 535 | always_inline transport_connection_t * |
| 536 | session_tx_get_transport (session_tx_context_t * ctx, u8 peek_data) |
| 537 | { |
| 538 | if (peek_data) |
| 539 | { |
| 540 | return ctx->transport_vft->get_connection (ctx->s->connection_index, |
| 541 | ctx->s->thread_index); |
| 542 | } |
| 543 | else |
| 544 | { |
| 545 | if (ctx->s->session_state == SESSION_STATE_LISTENING) |
| 546 | return ctx->transport_vft->get_listener (ctx->s->connection_index); |
| 547 | else |
| 548 | { |
| 549 | return ctx->transport_vft->get_connection (ctx->s->connection_index, |
| 550 | ctx->s->thread_index); |
| 551 | } |
| 552 | } |
| 553 | } |
Florin Coras | 6a9b68b | 2017-11-21 04:20:42 -0800 | [diff] [blame] | 554 | |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 555 | always_inline void |
| 556 | session_tx_set_dequeue_params (vlib_main_t * vm, session_tx_context_t * ctx, |
Florin Coras | f08f26d | 2018-05-10 13:20:47 -0700 | [diff] [blame] | 557 | u32 max_segs, u8 peek_data) |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 558 | { |
| 559 | u32 n_bytes_per_buf, n_bytes_per_seg; |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 560 | ctx->max_dequeue = svm_fifo_max_dequeue (ctx->s->tx_fifo); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 561 | if (peek_data) |
| 562 | { |
Florin Coras | 9d06304 | 2017-09-14 03:08:00 -0400 | [diff] [blame] | 563 | /* Offset in rx fifo from where to peek data */ |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 564 | ctx->tx_offset = ctx->transport_vft->tx_fifo_offset (ctx->tc); |
| 565 | if (PREDICT_FALSE (ctx->tx_offset >= ctx->max_dequeue)) |
| 566 | { |
| 567 | ctx->max_len_to_snd = 0; |
| 568 | return; |
| 569 | } |
| 570 | ctx->max_dequeue -= ctx->tx_offset; |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 571 | } |
Florin Coras | 7fb0fe1 | 2018-04-09 09:24:52 -0700 | [diff] [blame] | 572 | else |
| 573 | { |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 574 | if (ctx->transport_vft->tx_type == TRANSPORT_TX_DGRAM) |
Florin Coras | 7fb0fe1 | 2018-04-09 09:24:52 -0700 | [diff] [blame] | 575 | { |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 576 | if (ctx->max_dequeue <= sizeof (ctx->hdr)) |
| 577 | { |
| 578 | ctx->max_len_to_snd = 0; |
| 579 | return; |
| 580 | } |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 581 | svm_fifo_peek (ctx->s->tx_fifo, 0, sizeof (ctx->hdr), |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 582 | (u8 *) & ctx->hdr); |
| 583 | ASSERT (ctx->hdr.data_length > ctx->hdr.data_offset); |
| 584 | ctx->max_dequeue = ctx->hdr.data_length - ctx->hdr.data_offset; |
Florin Coras | 7fb0fe1 | 2018-04-09 09:24:52 -0700 | [diff] [blame] | 585 | } |
| 586 | } |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 587 | ASSERT (ctx->max_dequeue > 0); |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 588 | |
| 589 | /* Ensure we're not writing more than transport window allows */ |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 590 | if (ctx->max_dequeue < ctx->snd_space) |
Florin Coras | 3e350af | 2017-03-30 02:54:28 -0700 | [diff] [blame] | 591 | { |
| 592 | /* Constrained by tx queue. Try to send only fully formed segments */ |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 593 | ctx->max_len_to_snd = |
| 594 | (ctx->max_dequeue > ctx->snd_mss) ? |
| 595 | ctx->max_dequeue - ctx->max_dequeue % ctx->snd_mss : ctx->max_dequeue; |
Florin Coras | 3e350af | 2017-03-30 02:54:28 -0700 | [diff] [blame] | 596 | /* TODO Nagle ? */ |
| 597 | } |
| 598 | else |
| 599 | { |
Florin Coras | 1f152cd | 2017-08-18 19:28:03 -0700 | [diff] [blame] | 600 | /* Expectation is that snd_space0 is already a multiple of snd_mss */ |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 601 | ctx->max_len_to_snd = ctx->snd_space; |
Florin Coras | 3e350af | 2017-03-30 02:54:28 -0700 | [diff] [blame] | 602 | } |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 603 | |
Florin Coras | f08f26d | 2018-05-10 13:20:47 -0700 | [diff] [blame] | 604 | /* Check if we're tx constrained by the node */ |
| 605 | ctx->n_segs_per_evt = ceil ((f64) ctx->max_len_to_snd / ctx->snd_mss); |
| 606 | if (ctx->n_segs_per_evt > max_segs) |
| 607 | { |
| 608 | ctx->n_segs_per_evt = max_segs; |
| 609 | ctx->max_len_to_snd = max_segs * ctx->snd_mss; |
| 610 | } |
| 611 | |
Damjan Marion | 8934a04 | 2019-02-09 23:29:26 +0100 | [diff] [blame] | 612 | n_bytes_per_buf = vlib_buffer_get_default_data_size (vm); |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 613 | ASSERT (n_bytes_per_buf > TRANSPORT_MAX_HDRS_LEN); |
| 614 | n_bytes_per_seg = TRANSPORT_MAX_HDRS_LEN + ctx->snd_mss; |
Florin Coras | f08f26d | 2018-05-10 13:20:47 -0700 | [diff] [blame] | 615 | ctx->n_bufs_per_seg = ceil ((f64) n_bytes_per_seg / n_bytes_per_buf); |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 616 | ctx->deq_per_buf = clib_min (ctx->snd_mss, n_bytes_per_buf); |
| 617 | ctx->deq_per_first_buf = clib_min (ctx->snd_mss, |
Florin Coras | 1ee7830 | 2019-02-05 15:51:15 -0800 | [diff] [blame] | 618 | n_bytes_per_buf - |
| 619 | TRANSPORT_MAX_HDRS_LEN); |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 620 | } |
Florin Coras | f6d68ed | 2017-05-07 19:12:02 -0700 | [diff] [blame] | 621 | |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 622 | always_inline int |
| 623 | session_tx_fifo_read_and_snd_i (vlib_main_t * vm, vlib_node_runtime_t * node, |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 624 | session_worker_t * wrk, |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 625 | session_event_t * e, int *n_tx_packets, |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 626 | u8 peek_data) |
| 627 | { |
Florin Coras | 2068fd4 | 2019-01-31 14:35:50 -0800 | [diff] [blame] | 628 | u32 next_index, next0, next1, *to_next, n_left_to_next, n_left, pbi; |
Florin Coras | f08f26d | 2018-05-10 13:20:47 -0700 | [diff] [blame] | 629 | u32 n_trace = vlib_get_trace_count (vm, node), n_bufs_needed = 0; |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 630 | session_main_t *smm = &session_main; |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 631 | session_tx_context_t *ctx = &wrk->ctx; |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 632 | transport_proto_t tp; |
| 633 | vlib_buffer_t *pb; |
Florin Coras | ca1c8f3 | 2018-05-23 21:01:30 -0700 | [diff] [blame] | 634 | u16 n_bufs, rv; |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 635 | |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 636 | if (PREDICT_FALSE ((rv = session_tx_not_ready (ctx->s, peek_data)))) |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 637 | { |
Florin Coras | ca1c8f3 | 2018-05-23 21:01:30 -0700 | [diff] [blame] | 638 | if (rv < 2) |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 639 | vec_add1 (wrk->pending_event_vector, *e); |
Florin Coras | 0d60a0f | 2018-06-29 02:02:08 -0700 | [diff] [blame] | 640 | return SESSION_TX_NO_DATA; |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 641 | } |
| 642 | |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 643 | next_index = smm->session_type_to_next[ctx->s->session_type]; |
Florin Coras | 66cf5e0 | 2018-06-08 09:17:39 -0700 | [diff] [blame] | 644 | next0 = next1 = next_index; |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 645 | |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 646 | tp = session_get_transport_proto (ctx->s); |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 647 | ctx->transport_vft = transport_protocol_get_vft (tp); |
| 648 | ctx->tc = session_tx_get_transport (ctx, peek_data); |
| 649 | ctx->snd_mss = ctx->transport_vft->send_mss (ctx->tc); |
Florin Coras | 42ceddb | 2018-12-12 10:56:01 -0800 | [diff] [blame] | 650 | |
| 651 | if (PREDICT_FALSE (e->event_type == SESSION_IO_EVT_TX_FLUSH)) |
| 652 | { |
| 653 | if (ctx->transport_vft->flush_data) |
| 654 | ctx->transport_vft->flush_data (ctx->tc); |
| 655 | } |
| 656 | |
| 657 | ctx->snd_space = transport_connection_snd_space (ctx->tc, |
David Johnson | d9818dd | 2018-12-14 14:53:41 -0500 | [diff] [blame] | 658 | vm->clib_time. |
| 659 | last_cpu_time, |
Florin Coras | 42ceddb | 2018-12-12 10:56:01 -0800 | [diff] [blame] | 660 | ctx->snd_mss); |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 661 | if (ctx->snd_space == 0 || ctx->snd_mss == 0) |
| 662 | { |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 663 | vec_add1 (wrk->pending_event_vector, *e); |
Florin Coras | 0d60a0f | 2018-06-29 02:02:08 -0700 | [diff] [blame] | 664 | return SESSION_TX_NO_DATA; |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 665 | } |
| 666 | |
| 667 | /* Allow enqueuing of a new event */ |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 668 | svm_fifo_unset_event (ctx->s->tx_fifo); |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 669 | |
| 670 | /* Check how much we can pull. */ |
Florin Coras | f08f26d | 2018-05-10 13:20:47 -0700 | [diff] [blame] | 671 | session_tx_set_dequeue_params (vm, ctx, VLIB_FRAME_SIZE - *n_tx_packets, |
| 672 | peek_data); |
| 673 | |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 674 | if (PREDICT_FALSE (!ctx->max_len_to_snd)) |
Florin Coras | 0d60a0f | 2018-06-29 02:02:08 -0700 | [diff] [blame] | 675 | return SESSION_TX_NO_DATA; |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 676 | |
Florin Coras | f08f26d | 2018-05-10 13:20:47 -0700 | [diff] [blame] | 677 | n_bufs_needed = ctx->n_segs_per_evt * ctx->n_bufs_per_seg; |
Florin Coras | 2068fd4 | 2019-01-31 14:35:50 -0800 | [diff] [blame] | 678 | vec_validate_aligned (wrk->tx_buffers, n_bufs_needed - 1, |
| 679 | CLIB_CACHE_LINE_BYTES); |
| 680 | n_bufs = vlib_buffer_alloc (vm, wrk->tx_buffers, n_bufs_needed); |
| 681 | if (PREDICT_FALSE (n_bufs < n_bufs_needed)) |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 682 | { |
Florin Coras | 2068fd4 | 2019-01-31 14:35:50 -0800 | [diff] [blame] | 683 | if (n_bufs) |
| 684 | vlib_buffer_free (vm, wrk->tx_buffers, n_bufs); |
| 685 | vec_add1 (wrk->pending_event_vector, *e); |
| 686 | return SESSION_TX_NO_BUFFERS; |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 687 | } |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 688 | |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 689 | /* |
| 690 | * Write until we fill up a frame |
| 691 | */ |
| 692 | vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); |
Florin Coras | f08f26d | 2018-05-10 13:20:47 -0700 | [diff] [blame] | 693 | if (PREDICT_FALSE (ctx->n_segs_per_evt > n_left_to_next)) |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 694 | { |
Florin Coras | f08f26d | 2018-05-10 13:20:47 -0700 | [diff] [blame] | 695 | ctx->n_segs_per_evt = n_left_to_next; |
| 696 | ctx->max_len_to_snd = ctx->snd_mss * n_left_to_next; |
| 697 | } |
| 698 | ctx->left_to_snd = ctx->max_len_to_snd; |
| 699 | n_left = ctx->n_segs_per_evt; |
Florin Coras | 66cf5e0 | 2018-06-08 09:17:39 -0700 | [diff] [blame] | 700 | |
| 701 | while (n_left >= 4) |
| 702 | { |
| 703 | vlib_buffer_t *b0, *b1; |
| 704 | u32 bi0, bi1; |
| 705 | |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 706 | pbi = wrk->tx_buffers[n_bufs - 3]; |
Florin Coras | 66cf5e0 | 2018-06-08 09:17:39 -0700 | [diff] [blame] | 707 | pb = vlib_get_buffer (vm, pbi); |
| 708 | vlib_prefetch_buffer_header (pb, STORE); |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 709 | pbi = wrk->tx_buffers[n_bufs - 4]; |
Florin Coras | 66cf5e0 | 2018-06-08 09:17:39 -0700 | [diff] [blame] | 710 | pb = vlib_get_buffer (vm, pbi); |
| 711 | vlib_prefetch_buffer_header (pb, STORE); |
| 712 | |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 713 | to_next[0] = bi0 = wrk->tx_buffers[--n_bufs]; |
| 714 | to_next[1] = bi1 = wrk->tx_buffers[--n_bufs]; |
Florin Coras | 66cf5e0 | 2018-06-08 09:17:39 -0700 | [diff] [blame] | 715 | |
| 716 | b0 = vlib_get_buffer (vm, bi0); |
| 717 | b1 = vlib_get_buffer (vm, bi1); |
| 718 | |
| 719 | session_tx_fill_buffer (vm, ctx, b0, &n_bufs, peek_data); |
| 720 | session_tx_fill_buffer (vm, ctx, b1, &n_bufs, peek_data); |
| 721 | |
| 722 | ctx->transport_vft->push_header (ctx->tc, b0); |
| 723 | ctx->transport_vft->push_header (ctx->tc, b1); |
| 724 | |
| 725 | to_next += 2; |
| 726 | n_left_to_next -= 2; |
| 727 | n_left -= 2; |
| 728 | |
| 729 | VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0); |
| 730 | VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b1); |
| 731 | |
| 732 | vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next, |
| 733 | n_left_to_next, bi0, bi1, next0, |
| 734 | next1); |
| 735 | } |
Florin Coras | f08f26d | 2018-05-10 13:20:47 -0700 | [diff] [blame] | 736 | while (n_left) |
| 737 | { |
Florin Coras | 66cf5e0 | 2018-06-08 09:17:39 -0700 | [diff] [blame] | 738 | vlib_buffer_t *b0; |
| 739 | u32 bi0; |
Florin Coras | f6d68ed | 2017-05-07 19:12:02 -0700 | [diff] [blame] | 740 | |
Florin Coras | 91ca462 | 2018-06-30 11:27:59 -0700 | [diff] [blame] | 741 | if (n_left > 1) |
| 742 | { |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 743 | pbi = wrk->tx_buffers[n_bufs - 2]; |
Florin Coras | 91ca462 | 2018-06-30 11:27:59 -0700 | [diff] [blame] | 744 | pb = vlib_get_buffer (vm, pbi); |
| 745 | vlib_prefetch_buffer_header (pb, STORE); |
| 746 | } |
| 747 | |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 748 | to_next[0] = bi0 = wrk->tx_buffers[--n_bufs]; |
Florin Coras | 66cf5e0 | 2018-06-08 09:17:39 -0700 | [diff] [blame] | 749 | b0 = vlib_get_buffer (vm, bi0); |
| 750 | session_tx_fill_buffer (vm, ctx, b0, &n_bufs, peek_data); |
Florin Coras | 81a13db | 2018-03-16 08:48:31 -0700 | [diff] [blame] | 751 | |
Florin Coras | 66cf5e0 | 2018-06-08 09:17:39 -0700 | [diff] [blame] | 752 | /* Ask transport to push header after current_length and |
| 753 | * total_length_not_including_first_buffer are updated */ |
| 754 | ctx->transport_vft->push_header (ctx->tc, b0); |
Florin Coras | f6359c8 | 2017-06-19 12:26:09 -0400 | [diff] [blame] | 755 | |
Florin Coras | 66cf5e0 | 2018-06-08 09:17:39 -0700 | [diff] [blame] | 756 | to_next += 1; |
| 757 | n_left_to_next -= 1; |
| 758 | n_left -= 1; |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 759 | |
Florin Coras | 66cf5e0 | 2018-06-08 09:17:39 -0700 | [diff] [blame] | 760 | VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0); |
Florin Coras | 7fb0fe1 | 2018-04-09 09:24:52 -0700 | [diff] [blame] | 761 | |
Florin Coras | 66cf5e0 | 2018-06-08 09:17:39 -0700 | [diff] [blame] | 762 | vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, |
| 763 | n_left_to_next, bi0, next0); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 764 | } |
Florin Coras | 66cf5e0 | 2018-06-08 09:17:39 -0700 | [diff] [blame] | 765 | |
| 766 | if (PREDICT_FALSE (n_trace > 0)) |
| 767 | session_tx_trace_frame (vm, node, next_index, to_next, |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 768 | ctx->n_segs_per_evt, ctx->s, n_trace); |
Florin Coras | 2068fd4 | 2019-01-31 14:35:50 -0800 | [diff] [blame] | 769 | if (PREDICT_FALSE (n_bufs)) |
| 770 | { |
| 771 | clib_warning ("not all buffers consumed"); |
| 772 | vlib_buffer_free (vm, wrk->tx_buffers, n_bufs); |
| 773 | } |
Florin Coras | f08f26d | 2018-05-10 13:20:47 -0700 | [diff] [blame] | 774 | *n_tx_packets += ctx->n_segs_per_evt; |
Florin Coras | d67f112 | 2018-05-21 17:47:40 -0700 | [diff] [blame] | 775 | transport_connection_update_tx_stats (ctx->tc, ctx->max_len_to_snd); |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 776 | vlib_put_next_frame (vm, node, next_index, n_left_to_next); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 777 | |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 778 | /* If we couldn't dequeue all bytes mark as partially read */ |
Florin Coras | f08f26d | 2018-05-10 13:20:47 -0700 | [diff] [blame] | 779 | ASSERT (ctx->left_to_snd == 0); |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 780 | if (ctx->max_len_to_snd < ctx->max_dequeue) |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 781 | if (svm_fifo_set_event (ctx->s->tx_fifo)) |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 782 | vec_add1 (wrk->pending_event_vector, *e); |
Florin Coras | 7fb0fe1 | 2018-04-09 09:24:52 -0700 | [diff] [blame] | 783 | |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 784 | if (!peek_data && ctx->transport_vft->tx_type == TRANSPORT_TX_DGRAM) |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 785 | { |
Florin Coras | 7fb0fe1 | 2018-04-09 09:24:52 -0700 | [diff] [blame] | 786 | /* Fix dgram pre header */ |
Florin Coras | 8e43d04 | 2018-05-04 15:46:57 -0700 | [diff] [blame] | 787 | if (ctx->max_len_to_snd < ctx->max_dequeue) |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 788 | svm_fifo_overwrite_head (ctx->s->tx_fifo, (u8 *) & ctx->hdr, |
Florin Coras | 7fb0fe1 | 2018-04-09 09:24:52 -0700 | [diff] [blame] | 789 | sizeof (session_dgram_pre_hdr_t)); |
| 790 | /* More data needs to be read */ |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 791 | else if (svm_fifo_max_dequeue (ctx->s->tx_fifo) > 0) |
| 792 | if (svm_fifo_set_event (ctx->s->tx_fifo)) |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 793 | vec_add1 (wrk->pending_event_vector, *e); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 794 | } |
Florin Coras | 0d60a0f | 2018-06-29 02:02:08 -0700 | [diff] [blame] | 795 | return SESSION_TX_OK; |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 796 | } |
| 797 | |
| 798 | int |
Florin Coras | d79b41e | 2017-03-04 05:37:52 -0800 | [diff] [blame] | 799 | session_tx_fifo_peek_and_snd (vlib_main_t * vm, vlib_node_runtime_t * node, |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 800 | session_worker_t * wrk, |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 801 | session_event_t * e, int *n_tx_pkts) |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 802 | { |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 803 | return session_tx_fifo_read_and_snd_i (vm, node, wrk, e, n_tx_pkts, 1); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 804 | } |
| 805 | |
| 806 | int |
Florin Coras | d79b41e | 2017-03-04 05:37:52 -0800 | [diff] [blame] | 807 | session_tx_fifo_dequeue_and_snd (vlib_main_t * vm, vlib_node_runtime_t * node, |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 808 | session_worker_t * wrk, |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 809 | session_event_t * e, int *n_tx_pkts) |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 810 | { |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 811 | return session_tx_fifo_read_and_snd_i (vm, node, wrk, e, n_tx_pkts, 0); |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 812 | } |
| 813 | |
Florin Coras | 371ca50 | 2018-02-21 12:07:41 -0800 | [diff] [blame] | 814 | int |
| 815 | session_tx_fifo_dequeue_internal (vlib_main_t * vm, |
| 816 | vlib_node_runtime_t * node, |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 817 | session_worker_t * wrk, |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 818 | session_event_t * e, int *n_tx_pkts) |
Florin Coras | 371ca50 | 2018-02-21 12:07:41 -0800 | [diff] [blame] | 819 | { |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 820 | session_t *s = wrk->ctx.s; |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 821 | |
Florin Coras | c0737e9 | 2019-03-04 14:19:39 -0800 | [diff] [blame] | 822 | if (PREDICT_FALSE (s->session_state >= SESSION_STATE_TRANSPORT_CLOSED)) |
Florin Coras | ef91534 | 2018-09-29 10:23:06 -0700 | [diff] [blame] | 823 | return 0; |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 824 | svm_fifo_unset_event (s->tx_fifo); |
Florin Coras | f940f8a | 2019-03-06 10:44:38 -0800 | [diff] [blame] | 825 | return transport_custom_tx (session_get_transport_proto (s), s); |
Florin Coras | 371ca50 | 2018-02-21 12:07:41 -0800 | [diff] [blame] | 826 | } |
| 827 | |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 828 | always_inline session_t * |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 829 | session_event_get_session (session_event_t * e, u8 thread_index) |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 830 | { |
Florin Coras | c0737e9 | 2019-03-04 14:19:39 -0800 | [diff] [blame] | 831 | return session_get_if_valid (e->session_index, thread_index); |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 832 | } |
| 833 | |
Florin Coras | d67f112 | 2018-05-21 17:47:40 -0700 | [diff] [blame] | 834 | static void |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 835 | session_update_dispatch_period (session_worker_t * wrk, f64 now, |
Florin Coras | d67f112 | 2018-05-21 17:47:40 -0700 | [diff] [blame] | 836 | u32 thread_index) |
| 837 | { |
Florin Coras | efefc6b | 2018-11-07 12:49:19 -0800 | [diff] [blame] | 838 | if (wrk->last_tx_packets) |
Florin Coras | c44a558 | 2018-11-01 16:30:54 -0700 | [diff] [blame] | 839 | { |
| 840 | f64 sample = now - wrk->last_vlib_time; |
Florin Coras | c44a558 | 2018-11-01 16:30:54 -0700 | [diff] [blame] | 841 | wrk->dispatch_period = (wrk->dispatch_period + sample) * 0.5; |
| 842 | } |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 843 | wrk->last_vlib_time = now; |
Florin Coras | d67f112 | 2018-05-21 17:47:40 -0700 | [diff] [blame] | 844 | } |
| 845 | |
Florin Coras | 0d60a0f | 2018-06-29 02:02:08 -0700 | [diff] [blame] | 846 | static uword |
| 847 | session_queue_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node, |
| 848 | vlib_frame_t * frame) |
| 849 | { |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 850 | session_main_t *smm = vnet_get_session_main (); |
Florin Coras | 3c2fed5 | 2018-07-04 04:15:05 -0700 | [diff] [blame] | 851 | u32 thread_index = vm->thread_index, n_to_dequeue, n_events; |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 852 | session_worker_t *wrk = &smm->wrk[thread_index]; |
Florin Coras | 5f56d73 | 2018-10-30 10:21:59 -0700 | [diff] [blame] | 853 | session_event_t *e, *fifo_events; |
Florin Coras | 3c2fed5 | 2018-07-04 04:15:05 -0700 | [diff] [blame] | 854 | svm_msg_q_msg_t _msg, *msg = &_msg; |
Florin Coras | 0d60a0f | 2018-06-29 02:02:08 -0700 | [diff] [blame] | 855 | f64 now = vlib_time_now (vm); |
Florin Coras | 3c2fed5 | 2018-07-04 04:15:05 -0700 | [diff] [blame] | 856 | int n_tx_packets = 0, i, rv; |
Florin Coras | 1553197 | 2018-08-12 23:50:53 -0700 | [diff] [blame] | 857 | app_worker_t *app_wrk; |
Florin Coras | 3c2fed5 | 2018-07-04 04:15:05 -0700 | [diff] [blame] | 858 | svm_msg_q_t *mq; |
Florin Coras | 0d60a0f | 2018-06-29 02:02:08 -0700 | [diff] [blame] | 859 | void (*fp) (void *); |
| 860 | |
| 861 | SESSION_EVT_DBG (SESSION_EVT_POLL_GAP_TRACK, smm, thread_index); |
| 862 | |
| 863 | /* |
| 864 | * Update transport time |
| 865 | */ |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 866 | session_update_dispatch_period (wrk, now, thread_index); |
Florin Coras | 0d60a0f | 2018-06-29 02:02:08 -0700 | [diff] [blame] | 867 | transport_update_time (now, thread_index); |
| 868 | |
Florin Coras | 0d60a0f | 2018-06-29 02:02:08 -0700 | [diff] [blame] | 869 | SESSION_EVT_DBG (SESSION_EVT_DEQ_NODE, 0); |
| 870 | |
Florin Coras | 5f56d73 | 2018-10-30 10:21:59 -0700 | [diff] [blame] | 871 | /* Make sure postponed events are handled first */ |
| 872 | fifo_events = wrk->free_event_vector; |
| 873 | vec_append (fifo_events, wrk->postponed_event_vector); |
Florin Coras | 36ee9f1 | 2018-11-02 12:52:10 -0700 | [diff] [blame] | 874 | _vec_len (wrk->postponed_event_vector) = 0; |
Florin Coras | 0d60a0f | 2018-06-29 02:02:08 -0700 | [diff] [blame] | 875 | |
Florin Coras | 5f56d73 | 2018-10-30 10:21:59 -0700 | [diff] [blame] | 876 | /* Try to dequeue what is available. Don't wait for lock. |
| 877 | * XXX: we may need priorities here */ |
| 878 | mq = wrk->vpp_event_queue; |
| 879 | n_to_dequeue = svm_msg_q_size (mq); |
| 880 | if (n_to_dequeue && svm_msg_q_try_lock (mq) == 0) |
| 881 | { |
| 882 | for (i = 0; i < n_to_dequeue; i++) |
| 883 | { |
| 884 | vec_add2 (fifo_events, e, 1); |
| 885 | svm_msg_q_sub_w_lock (mq, msg); |
Florin Coras | 3c7d4f9 | 2018-12-14 11:28:43 -0800 | [diff] [blame] | 886 | /* Works because reply messages are smaller than a session evt. |
| 887 | * If we ever need to support bigger messages this needs to be |
| 888 | * fixed */ |
Dave Barach | 178cf49 | 2018-11-13 16:34:13 -0500 | [diff] [blame] | 889 | clib_memcpy_fast (e, svm_msg_q_msg_data (mq, msg), sizeof (*e)); |
Florin Coras | 5f56d73 | 2018-10-30 10:21:59 -0700 | [diff] [blame] | 890 | svm_msg_q_free_msg (mq, msg); |
| 891 | } |
| 892 | svm_msg_q_unlock (mq); |
| 893 | } |
| 894 | |
| 895 | vec_append (fifo_events, wrk->pending_event_vector); |
| 896 | vec_append (fifo_events, wrk->pending_disconnects); |
| 897 | |
Florin Coras | 5f56d73 | 2018-10-30 10:21:59 -0700 | [diff] [blame] | 898 | _vec_len (wrk->pending_event_vector) = 0; |
Florin Coras | 36ee9f1 | 2018-11-02 12:52:10 -0700 | [diff] [blame] | 899 | _vec_len (wrk->pending_disconnects) = 0; |
Florin Coras | 5f56d73 | 2018-10-30 10:21:59 -0700 | [diff] [blame] | 900 | |
Florin Coras | 0d60a0f | 2018-06-29 02:02:08 -0700 | [diff] [blame] | 901 | n_events = vec_len (fifo_events); |
Florin Coras | 5f56d73 | 2018-10-30 10:21:59 -0700 | [diff] [blame] | 902 | if (PREDICT_FALSE (!n_events)) |
| 903 | return 0; |
| 904 | |
Florin Coras | 0d60a0f | 2018-06-29 02:02:08 -0700 | [diff] [blame] | 905 | for (i = 0; i < n_events; i++) |
| 906 | { |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 907 | session_t *s; /* $$$ prefetch 1 ahead maybe */ |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 908 | session_event_t *e; |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 909 | u8 need_tx_ntf; |
Florin Coras | 0d60a0f | 2018-06-29 02:02:08 -0700 | [diff] [blame] | 910 | |
| 911 | e = &fifo_events[i]; |
| 912 | switch (e->event_type) |
| 913 | { |
Florin Coras | 42ceddb | 2018-12-12 10:56:01 -0800 | [diff] [blame] | 914 | case SESSION_IO_EVT_TX_FLUSH: |
Florin Coras | f6c4313 | 2019-03-01 12:41:21 -0800 | [diff] [blame] | 915 | case SESSION_IO_EVT_TX: |
Florin Coras | 0d60a0f | 2018-06-29 02:02:08 -0700 | [diff] [blame] | 916 | /* Don't try to send more that one frame per dispatch cycle */ |
| 917 | if (n_tx_packets == VLIB_FRAME_SIZE) |
| 918 | { |
Florin Coras | 5f56d73 | 2018-10-30 10:21:59 -0700 | [diff] [blame] | 919 | vec_add1 (wrk->postponed_event_vector, *e); |
Florin Coras | 0d60a0f | 2018-06-29 02:02:08 -0700 | [diff] [blame] | 920 | break; |
| 921 | } |
| 922 | |
| 923 | s = session_event_get_session (e, thread_index); |
| 924 | if (PREDICT_FALSE (!s)) |
| 925 | { |
Florin Coras | 326b81e | 2018-10-04 19:03:05 -0700 | [diff] [blame] | 926 | clib_warning ("session was freed!"); |
Florin Coras | 0d60a0f | 2018-06-29 02:02:08 -0700 | [diff] [blame] | 927 | continue; |
| 928 | } |
Florin Coras | 14ed6df | 2019-03-06 21:13:42 -0800 | [diff] [blame] | 929 | CLIB_PREFETCH (s->tx_fifo, 2 * CLIB_CACHE_LINE_BYTES, LOAD); |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 930 | wrk->ctx.s = s; |
Florin Coras | 0d60a0f | 2018-06-29 02:02:08 -0700 | [diff] [blame] | 931 | /* Spray packets in per session type frames, since they go to |
| 932 | * different nodes */ |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 933 | rv = (smm->session_tx_fns[s->session_type]) (vm, node, wrk, e, |
Florin Coras | 0d60a0f | 2018-06-29 02:02:08 -0700 | [diff] [blame] | 934 | &n_tx_packets); |
| 935 | if (PREDICT_TRUE (rv == SESSION_TX_OK)) |
| 936 | { |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 937 | need_tx_ntf = svm_fifo_needs_tx_ntf (s->tx_fifo, |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 938 | wrk->ctx.max_len_to_snd); |
| 939 | if (PREDICT_FALSE (need_tx_ntf)) |
| 940 | session_dequeue_notify (s); |
Florin Coras | 0d60a0f | 2018-06-29 02:02:08 -0700 | [diff] [blame] | 941 | } |
| 942 | else if (PREDICT_FALSE (rv == SESSION_TX_NO_BUFFERS)) |
| 943 | { |
| 944 | vlib_node_increment_counter (vm, node->node_index, |
| 945 | SESSION_QUEUE_ERROR_NO_BUFFER, 1); |
| 946 | continue; |
| 947 | } |
| 948 | break; |
Florin Coras | f6c4313 | 2019-03-01 12:41:21 -0800 | [diff] [blame] | 949 | case SESSION_CTRL_EVT_CLOSE: |
Florin Coras | 3c7d4f9 | 2018-12-14 11:28:43 -0800 | [diff] [blame] | 950 | s = session_get_from_handle_if_valid (e->session_handle); |
| 951 | if (PREDICT_FALSE (!s)) |
| 952 | break; |
| 953 | |
Florin Coras | e96bf63 | 2018-12-18 22:44:27 -0800 | [diff] [blame] | 954 | /* Make sure session disconnects run after the pending list is |
| 955 | * drained, i.e., postpone if the first time. If not the first |
| 956 | * and the tx queue is still not empty, try to wait for some |
| 957 | * dispatch cycles */ |
| 958 | if (!e->postponed |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 959 | || (e->postponed < 200 && svm_fifo_max_dequeue (s->tx_fifo))) |
Florin Coras | 0d60a0f | 2018-06-29 02:02:08 -0700 | [diff] [blame] | 960 | { |
Florin Coras | e96bf63 | 2018-12-18 22:44:27 -0800 | [diff] [blame] | 961 | e->postponed += 1; |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 962 | vec_add1 (wrk->pending_disconnects, *e); |
Florin Coras | 0d60a0f | 2018-06-29 02:02:08 -0700 | [diff] [blame] | 963 | continue; |
| 964 | } |
| 965 | |
Florin Coras | 5a2ec8f | 2018-12-27 11:53:11 -0800 | [diff] [blame] | 966 | session_transport_close (s); |
Florin Coras | 0d60a0f | 2018-06-29 02:02:08 -0700 | [diff] [blame] | 967 | break; |
Florin Coras | f6c4313 | 2019-03-01 12:41:21 -0800 | [diff] [blame] | 968 | case SESSION_IO_EVT_BUILTIN_RX: |
Florin Coras | 0d60a0f | 2018-06-29 02:02:08 -0700 | [diff] [blame] | 969 | s = session_event_get_session (e, thread_index); |
Florin Coras | ef91534 | 2018-09-29 10:23:06 -0700 | [diff] [blame] | 970 | if (PREDICT_FALSE (!s || s->session_state >= SESSION_STATE_CLOSING)) |
Florin Coras | 0d60a0f | 2018-06-29 02:02:08 -0700 | [diff] [blame] | 971 | continue; |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 972 | svm_fifo_unset_event (s->rx_fifo); |
Florin Coras | 1553197 | 2018-08-12 23:50:53 -0700 | [diff] [blame] | 973 | app_wrk = app_worker_get (s->app_wrk_index); |
Florin Coras | bf7ce2c | 2019-03-06 14:44:42 -0800 | [diff] [blame] | 974 | app_worker_builtin_rx (app_wrk, s); |
Florin Coras | 0d60a0f | 2018-06-29 02:02:08 -0700 | [diff] [blame] | 975 | break; |
Florin Coras | f6c4313 | 2019-03-01 12:41:21 -0800 | [diff] [blame] | 976 | case SESSION_IO_EVT_BUILTIN_TX: |
Florin Coras | ef91534 | 2018-09-29 10:23:06 -0700 | [diff] [blame] | 977 | s = session_get_from_handle_if_valid (e->session_handle); |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 978 | wrk->ctx.s = s; |
Florin Coras | ef91534 | 2018-09-29 10:23:06 -0700 | [diff] [blame] | 979 | if (PREDICT_TRUE (s != 0)) |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 980 | session_tx_fifo_dequeue_internal (vm, node, wrk, e, |
| 981 | &n_tx_packets); |
Florin Coras | ef91534 | 2018-09-29 10:23:06 -0700 | [diff] [blame] | 982 | break; |
Florin Coras | f6c4313 | 2019-03-01 12:41:21 -0800 | [diff] [blame] | 983 | case SESSION_CTRL_EVT_RPC: |
Florin Coras | 0d60a0f | 2018-06-29 02:02:08 -0700 | [diff] [blame] | 984 | fp = e->rpc_args.fp; |
| 985 | (*fp) (e->rpc_args.arg); |
| 986 | break; |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 987 | case SESSION_CTRL_EVT_DISCONNECTED: |
| 988 | session_mq_disconnected_handler (e->data); |
| 989 | break; |
| 990 | case SESSION_CTRL_EVT_ACCEPTED_REPLY: |
| 991 | session_mq_accepted_reply_handler (e->data); |
| 992 | break; |
| 993 | case SESSION_CTRL_EVT_CONNECTED_REPLY: |
| 994 | break; |
| 995 | case SESSION_CTRL_EVT_DISCONNECTED_REPLY: |
| 996 | session_mq_disconnected_reply_handler (e->data); |
| 997 | break; |
| 998 | case SESSION_CTRL_EVT_RESET_REPLY: |
| 999 | session_mq_reset_reply_handler (e->data); |
| 1000 | break; |
Florin Coras | 30e79c2 | 2019-01-02 19:31:22 -0800 | [diff] [blame] | 1001 | case SESSION_CTRL_EVT_WORKER_UPDATE: |
| 1002 | session_mq_worker_update_handler (e->data); |
| 1003 | break; |
Florin Coras | 0d60a0f | 2018-06-29 02:02:08 -0700 | [diff] [blame] | 1004 | default: |
| 1005 | clib_warning ("unhandled event type %d", e->event_type); |
| 1006 | } |
| 1007 | } |
| 1008 | |
| 1009 | _vec_len (fifo_events) = 0; |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 1010 | wrk->free_event_vector = fifo_events; |
Florin Coras | c44a558 | 2018-11-01 16:30:54 -0700 | [diff] [blame] | 1011 | wrk->last_tx_packets = n_tx_packets; |
Florin Coras | 0d60a0f | 2018-06-29 02:02:08 -0700 | [diff] [blame] | 1012 | |
| 1013 | vlib_node_increment_counter (vm, session_queue_node.index, |
| 1014 | SESSION_QUEUE_ERROR_TX, n_tx_packets); |
| 1015 | |
| 1016 | SESSION_EVT_DBG (SESSION_EVT_DISPATCH_END, smm, thread_index); |
| 1017 | |
| 1018 | return n_tx_packets; |
| 1019 | } |
| 1020 | |
| 1021 | /* *INDENT-OFF* */ |
| 1022 | VLIB_REGISTER_NODE (session_queue_node) = |
| 1023 | { |
| 1024 | .function = session_queue_node_fn, |
| 1025 | .name = "session-queue", |
| 1026 | .format_trace = format_session_queue_trace, |
| 1027 | .type = VLIB_NODE_TYPE_INPUT, |
| 1028 | .n_errors = ARRAY_LEN (session_queue_error_strings), |
| 1029 | .error_strings = session_queue_error_strings, |
| 1030 | .state = VLIB_NODE_STATE_DISABLED, |
| 1031 | }; |
| 1032 | /* *INDENT-ON* */ |
| 1033 | |
Dave Barach | acd2a6a | 2017-05-16 17:41:34 -0400 | [diff] [blame] | 1034 | void |
| 1035 | dump_thread_0_event_queue (void) |
| 1036 | { |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 1037 | session_main_t *smm = vnet_get_session_main (); |
Dave Barach | acd2a6a | 2017-05-16 17:41:34 -0400 | [diff] [blame] | 1038 | vlib_main_t *vm = &vlib_global_main; |
| 1039 | u32 my_thread_index = vm->thread_index; |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 1040 | session_event_t _e, *e = &_e; |
Florin Coras | 3c2fed5 | 2018-07-04 04:15:05 -0700 | [diff] [blame] | 1041 | svm_msg_q_ring_t *ring; |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 1042 | session_t *s0; |
Florin Coras | 3c2fed5 | 2018-07-04 04:15:05 -0700 | [diff] [blame] | 1043 | svm_msg_q_msg_t *msg; |
| 1044 | svm_msg_q_t *mq; |
Dave Barach | acd2a6a | 2017-05-16 17:41:34 -0400 | [diff] [blame] | 1045 | int i, index; |
Dave Barach | acd2a6a | 2017-05-16 17:41:34 -0400 | [diff] [blame] | 1046 | |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 1047 | mq = smm->wrk[my_thread_index].vpp_event_queue; |
Florin Coras | 3c2fed5 | 2018-07-04 04:15:05 -0700 | [diff] [blame] | 1048 | index = mq->q->head; |
Dave Barach | acd2a6a | 2017-05-16 17:41:34 -0400 | [diff] [blame] | 1049 | |
Florin Coras | 3c2fed5 | 2018-07-04 04:15:05 -0700 | [diff] [blame] | 1050 | for (i = 0; i < mq->q->cursize; i++) |
Dave Barach | acd2a6a | 2017-05-16 17:41:34 -0400 | [diff] [blame] | 1051 | { |
Florin Coras | 3c2fed5 | 2018-07-04 04:15:05 -0700 | [diff] [blame] | 1052 | msg = (svm_msg_q_msg_t *) (&mq->q->data[0] + mq->q->elsize * index); |
| 1053 | ring = svm_msg_q_ring (mq, msg->ring_index); |
Dave Barach | 178cf49 | 2018-11-13 16:34:13 -0500 | [diff] [blame] | 1054 | clib_memcpy_fast (e, svm_msg_q_msg_data (mq, msg), ring->elsize); |
Dave Barach | acd2a6a | 2017-05-16 17:41:34 -0400 | [diff] [blame] | 1055 | |
| 1056 | switch (e->event_type) |
| 1057 | { |
Florin Coras | f6c4313 | 2019-03-01 12:41:21 -0800 | [diff] [blame] | 1058 | case SESSION_IO_EVT_TX: |
Dave Barach | acd2a6a | 2017-05-16 17:41:34 -0400 | [diff] [blame] | 1059 | s0 = session_event_get_session (e, my_thread_index); |
| 1060 | fformat (stdout, "[%04d] TX session %d\n", i, s0->session_index); |
| 1061 | break; |
| 1062 | |
Florin Coras | f6c4313 | 2019-03-01 12:41:21 -0800 | [diff] [blame] | 1063 | case SESSION_CTRL_EVT_CLOSE: |
Florin Coras | cea194d | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 1064 | s0 = session_get_from_handle (e->session_handle); |
Dave Barach | acd2a6a | 2017-05-16 17:41:34 -0400 | [diff] [blame] | 1065 | fformat (stdout, "[%04d] disconnect session %d\n", i, |
| 1066 | s0->session_index); |
| 1067 | break; |
| 1068 | |
Florin Coras | f6c4313 | 2019-03-01 12:41:21 -0800 | [diff] [blame] | 1069 | case SESSION_IO_EVT_BUILTIN_RX: |
Dave Barach | acd2a6a | 2017-05-16 17:41:34 -0400 | [diff] [blame] | 1070 | s0 = session_event_get_session (e, my_thread_index); |
| 1071 | fformat (stdout, "[%04d] builtin_rx %d\n", i, s0->session_index); |
| 1072 | break; |
| 1073 | |
Florin Coras | f6c4313 | 2019-03-01 12:41:21 -0800 | [diff] [blame] | 1074 | case SESSION_CTRL_EVT_RPC: |
Dave Barach | acd2a6a | 2017-05-16 17:41:34 -0400 | [diff] [blame] | 1075 | fformat (stdout, "[%04d] RPC call %llx with %llx\n", |
David Johnson | d9818dd | 2018-12-14 14:53:41 -0500 | [diff] [blame] | 1076 | i, (u64) (uword) (e->rpc_args.fp), |
| 1077 | (u64) (uword) (e->rpc_args.arg)); |
Dave Barach | acd2a6a | 2017-05-16 17:41:34 -0400 | [diff] [blame] | 1078 | break; |
| 1079 | |
| 1080 | default: |
| 1081 | fformat (stdout, "[%04d] unhandled event type %d\n", |
| 1082 | i, e->event_type); |
| 1083 | break; |
| 1084 | } |
| 1085 | |
| 1086 | index++; |
| 1087 | |
Florin Coras | 3c2fed5 | 2018-07-04 04:15:05 -0700 | [diff] [blame] | 1088 | if (index == mq->q->maxsize) |
Dave Barach | acd2a6a | 2017-05-16 17:41:34 -0400 | [diff] [blame] | 1089 | index = 0; |
| 1090 | } |
| 1091 | } |
| 1092 | |
Florin Coras | 6534b7a | 2017-07-18 05:38:03 -0400 | [diff] [blame] | 1093 | static u8 |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 1094 | session_node_cmp_event (session_event_t * e, svm_fifo_t * f) |
Florin Coras | 6534b7a | 2017-07-18 05:38:03 -0400 | [diff] [blame] | 1095 | { |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 1096 | session_t *s; |
Florin Coras | 6534b7a | 2017-07-18 05:38:03 -0400 | [diff] [blame] | 1097 | switch (e->event_type) |
| 1098 | { |
Florin Coras | f6c4313 | 2019-03-01 12:41:21 -0800 | [diff] [blame] | 1099 | case SESSION_IO_EVT_RX: |
| 1100 | case SESSION_IO_EVT_TX: |
| 1101 | case SESSION_IO_EVT_BUILTIN_RX: |
Florin Coras | c0737e9 | 2019-03-04 14:19:39 -0800 | [diff] [blame] | 1102 | if (e->session_index == f->master_session_index) |
Florin Coras | 6534b7a | 2017-07-18 05:38:03 -0400 | [diff] [blame] | 1103 | return 1; |
| 1104 | break; |
Florin Coras | f6c4313 | 2019-03-01 12:41:21 -0800 | [diff] [blame] | 1105 | case SESSION_CTRL_EVT_CLOSE: |
Florin Coras | 6534b7a | 2017-07-18 05:38:03 -0400 | [diff] [blame] | 1106 | break; |
Florin Coras | f6c4313 | 2019-03-01 12:41:21 -0800 | [diff] [blame] | 1107 | case SESSION_CTRL_EVT_RPC: |
Florin Coras | cea194d | 2017-10-02 00:18:51 -0700 | [diff] [blame] | 1108 | s = session_get_from_handle (e->session_handle); |
Florin Coras | 6534b7a | 2017-07-18 05:38:03 -0400 | [diff] [blame] | 1109 | if (!s) |
| 1110 | { |
| 1111 | clib_warning ("session has event but doesn't exist!"); |
| 1112 | break; |
| 1113 | } |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 1114 | if (s->rx_fifo == f || s->tx_fifo == f) |
Florin Coras | 6534b7a | 2017-07-18 05:38:03 -0400 | [diff] [blame] | 1115 | return 1; |
| 1116 | break; |
| 1117 | default: |
| 1118 | break; |
| 1119 | } |
| 1120 | return 0; |
| 1121 | } |
| 1122 | |
| 1123 | u8 |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 1124 | session_node_lookup_fifo_event (svm_fifo_t * f, session_event_t * e) |
Florin Coras | 6534b7a | 2017-07-18 05:38:03 -0400 | [diff] [blame] | 1125 | { |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 1126 | session_event_t *pending_event_vector, *evt; |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 1127 | session_worker_t *wrk; |
Florin Coras | 6534b7a | 2017-07-18 05:38:03 -0400 | [diff] [blame] | 1128 | int i, index, found = 0; |
Florin Coras | 3c2fed5 | 2018-07-04 04:15:05 -0700 | [diff] [blame] | 1129 | svm_msg_q_msg_t *msg; |
| 1130 | svm_msg_q_ring_t *ring; |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 1131 | svm_msg_q_t *mq; |
Florin Coras | 6534b7a | 2017-07-18 05:38:03 -0400 | [diff] [blame] | 1132 | u8 thread_index; |
| 1133 | |
| 1134 | ASSERT (e); |
| 1135 | thread_index = f->master_thread_index; |
Florin Coras | 31c9955 | 2019-03-01 13:00:58 -0800 | [diff] [blame] | 1136 | wrk = session_main_get_worker (thread_index); |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 1137 | |
Florin Coras | 6534b7a | 2017-07-18 05:38:03 -0400 | [diff] [blame] | 1138 | /* |
| 1139 | * Search evt queue |
| 1140 | */ |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 1141 | mq = wrk->vpp_event_queue; |
Florin Coras | 3c2fed5 | 2018-07-04 04:15:05 -0700 | [diff] [blame] | 1142 | index = mq->q->head; |
| 1143 | for (i = 0; i < mq->q->cursize; i++) |
Florin Coras | 6534b7a | 2017-07-18 05:38:03 -0400 | [diff] [blame] | 1144 | { |
Florin Coras | 3c2fed5 | 2018-07-04 04:15:05 -0700 | [diff] [blame] | 1145 | msg = (svm_msg_q_msg_t *) (&mq->q->data[0] + mq->q->elsize * index); |
| 1146 | ring = svm_msg_q_ring (mq, msg->ring_index); |
Dave Barach | 178cf49 | 2018-11-13 16:34:13 -0500 | [diff] [blame] | 1147 | clib_memcpy_fast (e, svm_msg_q_msg_data (mq, msg), ring->elsize); |
Florin Coras | 6534b7a | 2017-07-18 05:38:03 -0400 | [diff] [blame] | 1148 | found = session_node_cmp_event (e, f); |
| 1149 | if (found) |
Florin Coras | 371ca50 | 2018-02-21 12:07:41 -0800 | [diff] [blame] | 1150 | return 1; |
Florin Coras | 3c2fed5 | 2018-07-04 04:15:05 -0700 | [diff] [blame] | 1151 | if (++index == mq->q->maxsize) |
Florin Coras | 6534b7a | 2017-07-18 05:38:03 -0400 | [diff] [blame] | 1152 | index = 0; |
| 1153 | } |
| 1154 | /* |
| 1155 | * Search pending events vector |
| 1156 | */ |
Florin Coras | 5a7ca7b | 2018-10-30 12:01:48 -0700 | [diff] [blame] | 1157 | pending_event_vector = wrk->pending_event_vector; |
Florin Coras | 6534b7a | 2017-07-18 05:38:03 -0400 | [diff] [blame] | 1158 | vec_foreach (evt, pending_event_vector) |
| 1159 | { |
| 1160 | found = session_node_cmp_event (evt, f); |
| 1161 | if (found) |
| 1162 | { |
Dave Barach | 178cf49 | 2018-11-13 16:34:13 -0500 | [diff] [blame] | 1163 | clib_memcpy_fast (e, evt, sizeof (*evt)); |
Florin Coras | 6534b7a | 2017-07-18 05:38:03 -0400 | [diff] [blame] | 1164 | break; |
| 1165 | } |
| 1166 | } |
| 1167 | return found; |
| 1168 | } |
| 1169 | |
Dave Barach | 2a86391 | 2017-11-28 10:11:42 -0500 | [diff] [blame] | 1170 | static clib_error_t * |
| 1171 | session_queue_exit (vlib_main_t * vm) |
| 1172 | { |
| 1173 | if (vec_len (vlib_mains) < 2) |
| 1174 | return 0; |
| 1175 | |
| 1176 | /* |
| 1177 | * Shut off (especially) worker-thread session nodes. |
| 1178 | * Otherwise, vpp can crash as the main thread unmaps the |
| 1179 | * API segment. |
| 1180 | */ |
| 1181 | vlib_worker_thread_barrier_sync (vm); |
| 1182 | session_node_enable_disable (0 /* is_enable */ ); |
| 1183 | vlib_worker_thread_barrier_release (vm); |
| 1184 | return 0; |
| 1185 | } |
| 1186 | |
| 1187 | VLIB_MAIN_LOOP_EXIT_FUNCTION (session_queue_exit); |
| 1188 | |
Florin Coras | fd542f1 | 2018-05-16 19:28:24 -0700 | [diff] [blame] | 1189 | static uword |
| 1190 | session_queue_process (vlib_main_t * vm, vlib_node_runtime_t * rt, |
| 1191 | vlib_frame_t * f) |
| 1192 | { |
| 1193 | f64 now, timeout = 1.0; |
| 1194 | uword *event_data = 0; |
| 1195 | uword event_type; |
| 1196 | |
| 1197 | while (1) |
| 1198 | { |
| 1199 | vlib_process_wait_for_event_or_clock (vm, timeout); |
| 1200 | now = vlib_time_now (vm); |
| 1201 | event_type = vlib_process_get_events (vm, (uword **) & event_data); |
| 1202 | |
| 1203 | switch (event_type) |
| 1204 | { |
| 1205 | case SESSION_Q_PROCESS_FLUSH_FRAMES: |
| 1206 | /* Flush the frames by updating all transports times */ |
| 1207 | transport_update_time (now, 0); |
| 1208 | break; |
| 1209 | case SESSION_Q_PROCESS_STOP: |
| 1210 | timeout = 100000.0; |
| 1211 | break; |
| 1212 | case ~0: |
| 1213 | /* Timed out. Update time for all transports to trigger all |
| 1214 | * outstanding retransmits. */ |
| 1215 | transport_update_time (now, 0); |
| 1216 | break; |
| 1217 | } |
| 1218 | vec_reset_length (event_data); |
| 1219 | } |
| 1220 | return 0; |
| 1221 | } |
| 1222 | |
| 1223 | /* *INDENT-OFF* */ |
| 1224 | VLIB_REGISTER_NODE (session_queue_process_node) = |
| 1225 | { |
| 1226 | .function = session_queue_process, |
| 1227 | .type = VLIB_NODE_TYPE_PROCESS, |
| 1228 | .name = "session-queue-process", |
| 1229 | .state = VLIB_NODE_STATE_DISABLED, |
| 1230 | }; |
| 1231 | /* *INDENT-ON* */ |
| 1232 | |
Florin Coras | 653e43f | 2019-03-04 10:56:23 -0800 | [diff] [blame] | 1233 | static_always_inline uword |
| 1234 | session_queue_pre_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, |
| 1235 | vlib_frame_t * frame) |
| 1236 | { |
| 1237 | session_main_t *sm = &session_main; |
| 1238 | if (!sm->wrk[0].vpp_event_queue) |
| 1239 | return 0; |
| 1240 | return session_queue_node_fn (vm, node, frame); |
| 1241 | } |
| 1242 | |
| 1243 | /* *INDENT-OFF* */ |
| 1244 | VLIB_REGISTER_NODE (session_queue_pre_input_node) = |
| 1245 | { |
| 1246 | .function = session_queue_pre_input_inline, |
| 1247 | .type = VLIB_NODE_TYPE_PRE_INPUT, |
| 1248 | .name = "session-queue-main", |
| 1249 | .state = VLIB_NODE_STATE_DISABLED, |
| 1250 | }; |
| 1251 | /* *INDENT-ON* */ |
| 1252 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 1253 | /* |
| 1254 | * fd.io coding-style-patch-verification: ON |
| 1255 | * |
| 1256 | * Local Variables: |
| 1257 | * eval: (c-set-style "gnu") |
| 1258 | * End: |
| 1259 | */ |