Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 1 | /* |
Florin Coras | 5e06257 | 2019-03-14 19:07:51 -0700 | [diff] [blame] | 2 | * Copyright (c) 2018-2019 Cisco and/or its affiliates. |
Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 3 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | * you may not use this |
| 5 | * You may obtain a copy of the License at: |
| 6 | * |
| 7 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | * |
| 9 | * Unless required by applicable law or agreed to in writing, software |
| 10 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | * See the License for the specific language governing permissions and |
| 13 | * limitations under the License. |
| 14 | */ |
| 15 | |
| 16 | #include <vcl/vcl_private.h> |
| 17 | |
Florin Coras | 3348a4c | 2018-09-07 17:09:35 -0700 | [diff] [blame] | 18 | static pthread_key_t vcl_worker_stop_key; |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 19 | |
Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 20 | vcl_mq_evt_conn_t * |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 21 | vcl_mq_evt_conn_alloc (vcl_worker_t * wrk) |
Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 22 | { |
| 23 | vcl_mq_evt_conn_t *mqc; |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 24 | pool_get (wrk->mq_evt_conns, mqc); |
Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 25 | memset (mqc, 0, sizeof (*mqc)); |
| 26 | return mqc; |
| 27 | } |
| 28 | |
| 29 | u32 |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 30 | vcl_mq_evt_conn_index (vcl_worker_t * wrk, vcl_mq_evt_conn_t * mqc) |
Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 31 | { |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 32 | return (mqc - wrk->mq_evt_conns); |
Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 33 | } |
| 34 | |
| 35 | vcl_mq_evt_conn_t * |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 36 | vcl_mq_evt_conn_get (vcl_worker_t * wrk, u32 mq_conn_idx) |
Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 37 | { |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 38 | return pool_elt_at_index (wrk->mq_evt_conns, mq_conn_idx); |
Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 39 | } |
| 40 | |
| 41 | int |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 42 | vcl_mq_epoll_add_evfd (vcl_worker_t * wrk, svm_msg_q_t * mq) |
Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 43 | { |
| 44 | struct epoll_event e = { 0 }; |
| 45 | vcl_mq_evt_conn_t *mqc; |
| 46 | u32 mqc_index; |
| 47 | int mq_fd; |
| 48 | |
| 49 | mq_fd = svm_msg_q_get_consumer_eventfd (mq); |
| 50 | |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 51 | if (wrk->mqs_epfd < 0 || mq_fd == -1) |
Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 52 | return -1; |
| 53 | |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 54 | mqc = vcl_mq_evt_conn_alloc (wrk); |
| 55 | mqc_index = vcl_mq_evt_conn_index (wrk, mqc); |
Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 56 | mqc->mq_fd = mq_fd; |
| 57 | mqc->mq = mq; |
| 58 | |
| 59 | e.events = EPOLLIN; |
| 60 | e.data.u32 = mqc_index; |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 61 | if (epoll_ctl (wrk->mqs_epfd, EPOLL_CTL_ADD, mq_fd, &e) < 0) |
Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 62 | { |
Florin Coras | 5e06257 | 2019-03-14 19:07:51 -0700 | [diff] [blame] | 63 | VDBG (0, "failed to add mq eventfd to mq epoll fd"); |
Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 64 | return -1; |
| 65 | } |
| 66 | |
| 67 | return mqc_index; |
| 68 | } |
| 69 | |
| 70 | int |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 71 | vcl_mq_epoll_del_evfd (vcl_worker_t * wrk, u32 mqc_index) |
Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 72 | { |
| 73 | vcl_mq_evt_conn_t *mqc; |
| 74 | |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 75 | if (wrk->mqs_epfd || mqc_index == ~0) |
Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 76 | return -1; |
| 77 | |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 78 | mqc = vcl_mq_evt_conn_get (wrk, mqc_index); |
| 79 | if (epoll_ctl (wrk->mqs_epfd, EPOLL_CTL_DEL, mqc->mq_fd, 0) < 0) |
Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 80 | { |
Florin Coras | 5e06257 | 2019-03-14 19:07:51 -0700 | [diff] [blame] | 81 | VDBG (0, "failed to del mq eventfd to mq epoll fd"); |
Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 82 | return -1; |
| 83 | } |
| 84 | return 0; |
| 85 | } |
| 86 | |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 87 | static vcl_worker_t * |
| 88 | vcl_worker_alloc (void) |
| 89 | { |
| 90 | vcl_worker_t *wrk; |
| 91 | pool_get (vcm->workers, wrk); |
| 92 | memset (wrk, 0, sizeof (*wrk)); |
| 93 | wrk->wrk_index = wrk - vcm->workers; |
Florin Coras | 01f3f89 | 2018-12-02 12:45:53 -0800 | [diff] [blame] | 94 | wrk->forked_child = ~0; |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 95 | return wrk; |
| 96 | } |
| 97 | |
| 98 | static void |
| 99 | vcl_worker_free (vcl_worker_t * wrk) |
| 100 | { |
| 101 | pool_put (vcm->workers, wrk); |
| 102 | } |
| 103 | |
Florin Coras | 935ce75 | 2020-09-08 22:43:47 -0700 | [diff] [blame] | 104 | int |
| 105 | vcl_api_app_worker_add (void) |
| 106 | { |
| 107 | if (vcm->cfg.vpp_app_socket_api) |
| 108 | return vcl_sapi_app_worker_add (); |
| 109 | |
| 110 | return vcl_bapi_app_worker_add (); |
| 111 | } |
| 112 | |
| 113 | void |
| 114 | vcl_api_app_worker_del (vcl_worker_t * wrk) |
| 115 | { |
| 116 | if (vcm->cfg.vpp_app_socket_api) |
| 117 | return vcl_sapi_app_worker_del (wrk); |
| 118 | |
| 119 | vcl_bapi_app_worker_del (wrk); |
| 120 | } |
| 121 | |
Florin Coras | 47c40e2 | 2018-11-26 17:01:36 -0800 | [diff] [blame] | 122 | void |
Florin Coras | 01f3f89 | 2018-12-02 12:45:53 -0800 | [diff] [blame] | 123 | vcl_worker_cleanup (vcl_worker_t * wrk, u8 notify_vpp) |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 124 | { |
Florin Coras | 47c40e2 | 2018-11-26 17:01:36 -0800 | [diff] [blame] | 125 | clib_spinlock_lock (&vcm->workers_lock); |
Florin Coras | 940f78f | 2018-11-30 12:11:20 -0800 | [diff] [blame] | 126 | if (notify_vpp) |
Florin Coras | 935ce75 | 2020-09-08 22:43:47 -0700 | [diff] [blame] | 127 | vcl_api_app_worker_del (wrk); |
Florin Coras | 64cf459 | 2019-12-12 12:01:24 -0800 | [diff] [blame] | 128 | |
Florin Coras | 940f78f | 2018-11-30 12:11:20 -0800 | [diff] [blame] | 129 | if (wrk->mqs_epfd > 0) |
| 130 | close (wrk->mqs_epfd); |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 131 | hash_free (wrk->session_index_by_vpp_handles); |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 132 | vec_free (wrk->mq_events); |
| 133 | vec_free (wrk->mq_msg_vector); |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 134 | vcl_worker_free (wrk); |
Florin Coras | 47c40e2 | 2018-11-26 17:01:36 -0800 | [diff] [blame] | 135 | clib_spinlock_unlock (&vcm->workers_lock); |
Florin Coras | 47c40e2 | 2018-11-26 17:01:36 -0800 | [diff] [blame] | 136 | } |
| 137 | |
| 138 | static void |
| 139 | vcl_worker_cleanup_cb (void *arg) |
| 140 | { |
Florin Coras | 01f3f89 | 2018-12-02 12:45:53 -0800 | [diff] [blame] | 141 | vcl_worker_t *wrk = vcl_worker_get_current (); |
| 142 | u32 wrk_index = wrk->wrk_index; |
| 143 | vcl_worker_cleanup (wrk, 1 /* notify vpp */ ); |
| 144 | vcl_set_worker_index (~0); |
Florin Coras | 940f78f | 2018-11-30 12:11:20 -0800 | [diff] [blame] | 145 | VDBG (0, "cleaned up worker %u", wrk_index); |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 146 | } |
| 147 | |
| 148 | vcl_worker_t * |
| 149 | vcl_worker_alloc_and_init () |
| 150 | { |
| 151 | vcl_worker_t *wrk; |
| 152 | |
| 153 | /* This was initialized already */ |
| 154 | if (vcl_get_worker_index () != ~0) |
| 155 | return 0; |
| 156 | |
Florin Coras | d2c9e70 | 2019-08-02 10:53:01 -0700 | [diff] [blame] | 157 | /* Use separate heap map entry for worker */ |
| 158 | clib_mem_set_thread_index (); |
| 159 | |
Florin Coras | de9f08b | 2018-09-07 14:32:58 -0700 | [diff] [blame] | 160 | if (pool_elts (vcm->workers) == vcm->cfg.max_workers) |
| 161 | { |
| 162 | VDBG (0, "max-workers %u limit reached", vcm->cfg.max_workers); |
| 163 | return 0; |
| 164 | } |
| 165 | |
| 166 | clib_spinlock_lock (&vcm->workers_lock); |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 167 | wrk = vcl_worker_alloc (); |
| 168 | vcl_set_worker_index (wrk->wrk_index); |
Florin Coras | 47c40e2 | 2018-11-26 17:01:36 -0800 | [diff] [blame] | 169 | wrk->thread_id = pthread_self (); |
| 170 | wrk->current_pid = getpid (); |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 171 | |
| 172 | wrk->mqs_epfd = -1; |
| 173 | if (vcm->cfg.use_mq_eventfd) |
| 174 | { |
hanlin | a3a4896 | 2020-07-13 11:09:15 +0800 | [diff] [blame] | 175 | wrk->vcl_needs_real_epoll = 1; |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 176 | wrk->mqs_epfd = epoll_create (1); |
hanlin | a3a4896 | 2020-07-13 11:09:15 +0800 | [diff] [blame] | 177 | wrk->vcl_needs_real_epoll = 0; |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 178 | if (wrk->mqs_epfd < 0) |
| 179 | { |
| 180 | clib_unix_warning ("epoll_create() returned"); |
Florin Coras | 47c40e2 | 2018-11-26 17:01:36 -0800 | [diff] [blame] | 181 | goto done; |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 182 | } |
| 183 | } |
| 184 | |
| 185 | wrk->session_index_by_vpp_handles = hash_create (0, sizeof (uword)); |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 186 | clib_time_init (&wrk->clib_time); |
| 187 | vec_validate (wrk->mq_events, 64); |
| 188 | vec_validate (wrk->mq_msg_vector, 128); |
| 189 | vec_reset_length (wrk->mq_msg_vector); |
Florin Coras | 86f0450 | 2018-09-12 16:08:01 -0700 | [diff] [blame] | 190 | vec_validate (wrk->unhandled_evts_vector, 128); |
| 191 | vec_reset_length (wrk->unhandled_evts_vector); |
Florin Coras | 47c40e2 | 2018-11-26 17:01:36 -0800 | [diff] [blame] | 192 | clib_spinlock_unlock (&vcm->workers_lock); |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 193 | |
Florin Coras | 47c40e2 | 2018-11-26 17:01:36 -0800 | [diff] [blame] | 194 | done: |
| 195 | return wrk; |
| 196 | } |
| 197 | |
| 198 | int |
| 199 | vcl_worker_register_with_vpp (void) |
| 200 | { |
| 201 | vcl_worker_t *wrk = vcl_worker_get_current (); |
| 202 | |
| 203 | clib_spinlock_lock (&vcm->workers_lock); |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 204 | |
Florin Coras | 935ce75 | 2020-09-08 22:43:47 -0700 | [diff] [blame] | 205 | if (vcl_api_app_worker_add ()) |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 206 | { |
Florin Coras | 5e06257 | 2019-03-14 19:07:51 -0700 | [diff] [blame] | 207 | VDBG (0, "failed to add worker to vpp"); |
Florin Coras | b88de90 | 2020-09-08 16:47:57 -0700 | [diff] [blame] | 208 | clib_spinlock_unlock (&vcm->workers_lock); |
Florin Coras | 47c40e2 | 2018-11-26 17:01:36 -0800 | [diff] [blame] | 209 | return -1; |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 210 | } |
Florin Coras | 47c40e2 | 2018-11-26 17:01:36 -0800 | [diff] [blame] | 211 | if (pthread_key_create (&vcl_worker_stop_key, vcl_worker_cleanup_cb)) |
Florin Coras | 5e06257 | 2019-03-14 19:07:51 -0700 | [diff] [blame] | 212 | VDBG (0, "failed to add pthread cleanup function"); |
Florin Coras | 3348a4c | 2018-09-07 17:09:35 -0700 | [diff] [blame] | 213 | if (pthread_setspecific (vcl_worker_stop_key, &wrk->thread_id)) |
Florin Coras | 5e06257 | 2019-03-14 19:07:51 -0700 | [diff] [blame] | 214 | VDBG (0, "failed to setup key value"); |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 215 | |
Florin Coras | de9f08b | 2018-09-07 14:32:58 -0700 | [diff] [blame] | 216 | clib_spinlock_unlock (&vcm->workers_lock); |
| 217 | |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 218 | VDBG (0, "added worker %u", wrk->wrk_index); |
Florin Coras | 47c40e2 | 2018-11-26 17:01:36 -0800 | [diff] [blame] | 219 | return 0; |
| 220 | } |
Florin Coras | 134a996 | 2018-08-28 11:32:04 -0700 | [diff] [blame] | 221 | |
Florin Coras | 458089b | 2019-08-21 16:20:44 -0700 | [diff] [blame] | 222 | svm_msg_q_t * |
| 223 | vcl_worker_ctrl_mq (vcl_worker_t * wrk) |
| 224 | { |
| 225 | return wrk->ctrl_mq; |
| 226 | } |
| 227 | |
Florin Coras | 0ef8ef2 | 2019-01-18 08:37:13 -0800 | [diff] [blame] | 228 | int |
Florin Coras | c127d5a | 2020-10-14 16:35:58 -0700 | [diff] [blame] | 229 | vcl_session_read_ready (vcl_session_t * s) |
Florin Coras | 0ef8ef2 | 2019-01-18 08:37:13 -0800 | [diff] [blame] | 230 | { |
Florin Coras | 6c3b218 | 2020-10-19 18:36:48 -0700 | [diff] [blame] | 231 | if (PREDICT_FALSE (s->flags & VCL_SESSION_F_IS_VEP)) |
Florin Coras | 0ef8ef2 | 2019-01-18 08:37:13 -0800 | [diff] [blame] | 232 | { |
| 233 | VDBG (0, "ERROR: session %u: cannot read from an epoll session!", |
Florin Coras | c127d5a | 2020-10-14 16:35:58 -0700 | [diff] [blame] | 234 | s->session_index); |
Florin Coras | 0ef8ef2 | 2019-01-18 08:37:13 -0800 | [diff] [blame] | 235 | return VPPCOM_EBADFD; |
| 236 | } |
| 237 | |
Florin Coras | 1bc919e | 2020-10-18 20:17:49 -0700 | [diff] [blame] | 238 | if (vcl_session_is_open (s)) |
Florin Coras | 0ef8ef2 | 2019-01-18 08:37:13 -0800 | [diff] [blame] | 239 | { |
Florin Coras | 1bc919e | 2020-10-18 20:17:49 -0700 | [diff] [blame] | 240 | if (vcl_session_is_ct (s)) |
| 241 | return svm_fifo_max_dequeue_cons (s->ct_rx_fifo); |
Florin Coras | 0ef8ef2 | 2019-01-18 08:37:13 -0800 | [diff] [blame] | 242 | |
Florin Coras | 1bc919e | 2020-10-18 20:17:49 -0700 | [diff] [blame] | 243 | if (s->is_dgram) |
| 244 | { |
| 245 | session_dgram_pre_hdr_t ph; |
| 246 | u32 max_deq; |
| 247 | |
| 248 | max_deq = svm_fifo_max_dequeue_cons (s->rx_fifo); |
| 249 | if (max_deq <= SESSION_CONN_HDR_LEN) |
| 250 | return 0; |
| 251 | if (svm_fifo_peek (s->rx_fifo, 0, sizeof (ph), (u8 *) & ph) < 0) |
| 252 | return 0; |
| 253 | if (ph.data_length + SESSION_CONN_HDR_LEN > max_deq) |
| 254 | return 0; |
| 255 | |
| 256 | return ph.data_length; |
| 257 | } |
| 258 | |
| 259 | return svm_fifo_max_dequeue_cons (s->rx_fifo); |
| 260 | } |
| 261 | else if (s->session_state == VCL_STATE_LISTEN) |
| 262 | { |
| 263 | return clib_fifo_elts (s->accept_evts_fifo); |
| 264 | } |
| 265 | else |
| 266 | { |
| 267 | return (s->session_state == VCL_STATE_DISCONNECT) ? |
Florin Coras | c127d5a | 2020-10-14 16:35:58 -0700 | [diff] [blame] | 268 | VPPCOM_ECONNRESET : VPPCOM_ENOTCONN; |
Florin Coras | 0ef8ef2 | 2019-01-18 08:37:13 -0800 | [diff] [blame] | 269 | } |
Florin Coras | 0ef8ef2 | 2019-01-18 08:37:13 -0800 | [diff] [blame] | 270 | } |
| 271 | |
| 272 | int |
Florin Coras | c127d5a | 2020-10-14 16:35:58 -0700 | [diff] [blame] | 273 | vcl_session_write_ready (vcl_session_t * s) |
Florin Coras | 0ef8ef2 | 2019-01-18 08:37:13 -0800 | [diff] [blame] | 274 | { |
Florin Coras | 6c3b218 | 2020-10-19 18:36:48 -0700 | [diff] [blame] | 275 | if (PREDICT_FALSE (s->flags & VCL_SESSION_F_IS_VEP)) |
Florin Coras | 0ef8ef2 | 2019-01-18 08:37:13 -0800 | [diff] [blame] | 276 | { |
| 277 | VDBG (0, "session %u [0x%llx]: cannot write to an epoll session!", |
Florin Coras | c127d5a | 2020-10-14 16:35:58 -0700 | [diff] [blame] | 278 | s->session_index, s->vpp_handle); |
Florin Coras | 0ef8ef2 | 2019-01-18 08:37:13 -0800 | [diff] [blame] | 279 | return VPPCOM_EBADFD; |
| 280 | } |
| 281 | |
Florin Coras | 1bc919e | 2020-10-18 20:17:49 -0700 | [diff] [blame] | 282 | if (vcl_session_is_open (s)) |
| 283 | { |
| 284 | if (vcl_session_is_ct (s)) |
| 285 | return svm_fifo_max_enqueue_prod (s->ct_tx_fifo); |
| 286 | |
| 287 | if (s->is_dgram) |
| 288 | { |
| 289 | u32 max_enq = svm_fifo_max_enqueue_prod (s->tx_fifo); |
| 290 | |
| 291 | if (max_enq <= sizeof (session_dgram_hdr_t)) |
| 292 | return 0; |
| 293 | return max_enq - sizeof (session_dgram_hdr_t); |
| 294 | } |
| 295 | |
| 296 | return svm_fifo_max_enqueue_prod (s->tx_fifo); |
| 297 | } |
| 298 | else if (s->session_state == VCL_STATE_LISTEN) |
Florin Coras | 0ef8ef2 | 2019-01-18 08:37:13 -0800 | [diff] [blame] | 299 | { |
Florin Coras | c127d5a | 2020-10-14 16:35:58 -0700 | [diff] [blame] | 300 | if (s->tx_fifo) |
| 301 | return svm_fifo_max_enqueue_prod (s->tx_fifo); |
Florin Coras | 0ef8ef2 | 2019-01-18 08:37:13 -0800 | [diff] [blame] | 302 | else |
| 303 | return VPPCOM_EBADFD; |
| 304 | } |
Florin Coras | 1bc919e | 2020-10-18 20:17:49 -0700 | [diff] [blame] | 305 | else |
Florin Coras | 0ef8ef2 | 2019-01-18 08:37:13 -0800 | [diff] [blame] | 306 | { |
Florin Coras | 1bc919e | 2020-10-18 20:17:49 -0700 | [diff] [blame] | 307 | return (s->session_state == VCL_STATE_DISCONNECT) ? |
Florin Coras | c127d5a | 2020-10-14 16:35:58 -0700 | [diff] [blame] | 308 | VPPCOM_ECONNRESET : VPPCOM_ENOTCONN; |
Florin Coras | 0ef8ef2 | 2019-01-18 08:37:13 -0800 | [diff] [blame] | 309 | } |
Florin Coras | 0ef8ef2 | 2019-01-18 08:37:13 -0800 | [diff] [blame] | 310 | } |
| 311 | |
Florin Coras | c4c4cf5 | 2019-08-24 18:17:34 -0700 | [diff] [blame] | 312 | int |
| 313 | vcl_segment_attach (u64 segment_handle, char *name, ssvm_segment_type_t type, |
| 314 | int fd) |
| 315 | { |
| 316 | fifo_segment_create_args_t _a, *a = &_a; |
| 317 | int rv; |
| 318 | |
| 319 | memset (a, 0, sizeof (*a)); |
Jakub Grajciar | b4e5e50 | 2020-01-31 09:35:29 +0100 | [diff] [blame] | 320 | a->segment_name = name; |
Florin Coras | c4c4cf5 | 2019-08-24 18:17:34 -0700 | [diff] [blame] | 321 | a->segment_type = type; |
| 322 | |
| 323 | if (type == SSVM_SEGMENT_MEMFD) |
| 324 | a->memfd_fd = fd; |
| 325 | |
Florin Coras | da8f218 | 2019-12-22 12:48:05 -0800 | [diff] [blame] | 326 | clib_rwlock_writer_lock (&vcm->segment_table_lock); |
| 327 | |
Florin Coras | c4c4cf5 | 2019-08-24 18:17:34 -0700 | [diff] [blame] | 328 | if ((rv = fifo_segment_attach (&vcm->segment_main, a))) |
| 329 | { |
| 330 | clib_warning ("svm_fifo_segment_attach ('%s') failed", name); |
| 331 | return rv; |
| 332 | } |
Florin Coras | da8f218 | 2019-12-22 12:48:05 -0800 | [diff] [blame] | 333 | hash_set (vcm->segment_table, segment_handle, a->new_segment_indices[0]); |
| 334 | |
| 335 | clib_rwlock_writer_unlock (&vcm->segment_table_lock); |
| 336 | |
Florin Coras | c4c4cf5 | 2019-08-24 18:17:34 -0700 | [diff] [blame] | 337 | vec_reset_length (a->new_segment_indices); |
| 338 | return 0; |
| 339 | } |
| 340 | |
Florin Coras | da8f218 | 2019-12-22 12:48:05 -0800 | [diff] [blame] | 341 | u32 |
| 342 | vcl_segment_table_lookup (u64 segment_handle) |
| 343 | { |
| 344 | uword *seg_indexp; |
| 345 | |
| 346 | clib_rwlock_reader_lock (&vcm->segment_table_lock); |
| 347 | seg_indexp = hash_get (vcm->segment_table, segment_handle); |
| 348 | clib_rwlock_reader_unlock (&vcm->segment_table_lock); |
| 349 | |
| 350 | if (!seg_indexp) |
| 351 | return VCL_INVALID_SEGMENT_INDEX; |
| 352 | return ((u32) * seg_indexp); |
| 353 | } |
| 354 | |
Florin Coras | c4c4cf5 | 2019-08-24 18:17:34 -0700 | [diff] [blame] | 355 | void |
| 356 | vcl_segment_detach (u64 segment_handle) |
| 357 | { |
| 358 | fifo_segment_main_t *sm = &vcm->segment_main; |
| 359 | fifo_segment_t *segment; |
| 360 | u32 segment_index; |
| 361 | |
| 362 | segment_index = vcl_segment_table_lookup (segment_handle); |
| 363 | if (segment_index == (u32) ~ 0) |
| 364 | return; |
Florin Coras | da8f218 | 2019-12-22 12:48:05 -0800 | [diff] [blame] | 365 | |
| 366 | clib_rwlock_writer_lock (&vcm->segment_table_lock); |
| 367 | |
Florin Coras | c4c4cf5 | 2019-08-24 18:17:34 -0700 | [diff] [blame] | 368 | segment = fifo_segment_get_segment (sm, segment_index); |
| 369 | fifo_segment_delete (sm, segment); |
Florin Coras | da8f218 | 2019-12-22 12:48:05 -0800 | [diff] [blame] | 370 | hash_unset (vcm->segment_table, segment_handle); |
| 371 | |
| 372 | clib_rwlock_writer_unlock (&vcm->segment_table_lock); |
| 373 | |
Florin Coras | c4c4cf5 | 2019-08-24 18:17:34 -0700 | [diff] [blame] | 374 | VDBG (0, "detached segment %u handle %u", segment_index, segment_handle); |
| 375 | } |
| 376 | |
Florin Coras | c547e91 | 2020-12-08 17:50:45 -0800 | [diff] [blame^] | 377 | int |
| 378 | vcl_segment_attach_session (uword segment_handle, uword rxf_offset, |
| 379 | uword txf_offset, u8 is_ct, vcl_session_t *s) |
| 380 | { |
| 381 | svm_fifo_shared_t *rxsf, *txsf; |
| 382 | svm_fifo_t *rxf, *txf; |
| 383 | fifo_segment_t *fs; |
| 384 | u32 fs_index; |
| 385 | |
| 386 | fs_index = vcl_segment_table_lookup (segment_handle); |
| 387 | if (fs_index == VCL_INVALID_SEGMENT_INDEX) |
| 388 | { |
| 389 | VDBG (0, "ERROR: segment for session %u is not mounted!", |
| 390 | s->session_index); |
| 391 | return -1; |
| 392 | } |
| 393 | |
| 394 | rxsf = uword_to_pointer (rxf_offset, svm_fifo_shared_t *); |
| 395 | txsf = uword_to_pointer (txf_offset, svm_fifo_shared_t *); |
| 396 | |
| 397 | clib_rwlock_reader_lock (&vcm->segment_table_lock); |
| 398 | |
| 399 | fs = fifo_segment_get_segment (&vcm->segment_main, fs_index); |
| 400 | rxf = fifo_segment_alloc_fifo_w_shared (fs, rxsf); |
| 401 | txf = fifo_segment_alloc_fifo_w_shared (fs, txsf); |
| 402 | |
| 403 | clib_rwlock_reader_unlock (&vcm->segment_table_lock); |
| 404 | |
| 405 | if (!is_ct) |
| 406 | { |
| 407 | rxsf->client_session_index = s->session_index; |
| 408 | txsf->client_session_index = s->session_index; |
| 409 | rxf->client_thread_index = vcl_get_worker_index (); |
| 410 | txf->client_thread_index = vcl_get_worker_index (); |
| 411 | s->rx_fifo = rxf; |
| 412 | s->tx_fifo = txf; |
| 413 | } |
| 414 | else |
| 415 | { |
| 416 | s->ct_rx_fifo = rxf; |
| 417 | s->ct_tx_fifo = txf; |
| 418 | } |
| 419 | |
| 420 | return 0; |
| 421 | } |
Florin Coras | c4c4cf5 | 2019-08-24 18:17:34 -0700 | [diff] [blame] | 422 | |
Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 423 | /* |
| 424 | * fd.io coding-style-patch-verification: ON |
| 425 | * |
| 426 | * Local Variables: |
| 427 | * eval: (c-set-style "gnu") |
| 428 | * End: |
| 429 | */ |