Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 1 | /* |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 2 | * Copyright (c) 2017-2019 Cisco and/or its affiliates. |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 3 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | * you may not use this file except in compliance with the License. |
| 5 | * You may obtain a copy of the License at: |
| 6 | * |
| 7 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | * |
| 9 | * Unless required by applicable law or agreed to in writing, software |
| 10 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | * See the License for the specific language governing permissions and |
| 13 | * limitations under the License. |
| 14 | */ |
| 15 | |
| 16 | #include <vnet/session/segment_manager.h> |
| 17 | #include <vnet/session/session.h> |
| 18 | #include <vnet/session/application.h> |
| 19 | |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 20 | typedef struct segment_manager_main_ |
| 21 | { |
| 22 | segment_manager_t *segment_managers; /**< Pool of segment managers */ |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 23 | u32 seg_name_counter; /**< Counter for segment names */ |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 24 | |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 25 | /* |
| 26 | * Configuration |
| 27 | */ |
Ryujiro Shibuya | d8f48e2 | 2020-01-22 12:11:42 +0000 | [diff] [blame] | 28 | u32 default_fifo_size; /**< default rx/tx fifo size */ |
| 29 | u32 default_segment_size; /**< default fifo segment size */ |
| 30 | u32 default_app_mq_size; /**< default app msg q size */ |
| 31 | u32 default_max_fifo_size; /**< default max fifo size */ |
| 32 | u8 default_high_watermark; /**< default high watermark % */ |
| 33 | u8 default_low_watermark; /**< default low watermark % */ |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 34 | } segment_manager_main_t; |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 35 | |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 36 | static segment_manager_main_t sm_main; |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 37 | |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 38 | #define segment_manager_foreach_segment_w_lock(VAR, SM, BODY) \ |
| 39 | do { \ |
| 40 | clib_rwlock_reader_lock (&(SM)->segments_rwlock); \ |
Damjan Marion | b2c31b6 | 2020-12-13 21:47:40 +0100 | [diff] [blame] | 41 | pool_foreach((VAR), ((SM)->segments)) (BODY); \ |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 42 | clib_rwlock_reader_unlock (&(SM)->segments_rwlock); \ |
| 43 | } while (0) |
| 44 | |
| 45 | static segment_manager_props_t * |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 46 | segment_manager_properties_get (segment_manager_t * sm) |
Florin Coras | ad0c77f | 2017-11-09 18:00:15 -0800 | [diff] [blame] | 47 | { |
Florin Coras | ab2f6db | 2018-08-31 14:31:41 -0700 | [diff] [blame] | 48 | app_worker_t *app_wrk = app_worker_get (sm->app_wrk_index); |
| 49 | return application_get_segment_manager_properties (app_wrk->app_index); |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 50 | } |
| 51 | |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 52 | segment_manager_props_t * |
| 53 | segment_manager_props_init (segment_manager_props_t * props) |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 54 | { |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 55 | props->add_segment_size = sm_main.default_segment_size; |
| 56 | props->rx_fifo_size = sm_main.default_fifo_size; |
| 57 | props->tx_fifo_size = sm_main.default_fifo_size; |
| 58 | props->evt_q_size = sm_main.default_app_mq_size; |
Ryujiro Shibuya | d8f48e2 | 2020-01-22 12:11:42 +0000 | [diff] [blame] | 59 | props->max_fifo_size = sm_main.default_max_fifo_size; |
Ryujiro Shibuya | 234fe89 | 2019-12-25 07:40:54 +0000 | [diff] [blame] | 60 | props->high_watermark = sm_main.default_high_watermark; |
| 61 | props->low_watermark = sm_main.default_low_watermark; |
Florin Coras | 62ddc03 | 2019-12-08 18:30:42 -0800 | [diff] [blame] | 62 | props->n_slices = vlib_num_workers () + 1; |
Florin Coras | ad0c77f | 2017-11-09 18:00:15 -0800 | [diff] [blame] | 63 | return props; |
| 64 | } |
| 65 | |
Florin Coras | bf39597 | 2020-04-30 15:05:24 +0000 | [diff] [blame] | 66 | u8 |
Florin Coras | 9d06304 | 2017-09-14 03:08:00 -0400 | [diff] [blame] | 67 | segment_manager_app_detached (segment_manager_t * sm) |
| 68 | { |
Florin Coras | c8e812f | 2020-05-14 05:32:18 +0000 | [diff] [blame] | 69 | return (sm->flags & SEG_MANAGER_F_DETACHED); |
Dave Wallace | 7b749fe | 2017-07-05 14:30:46 -0400 | [diff] [blame] | 70 | } |
| 71 | |
Florin Coras | 4e4531e | 2017-11-06 23:27:56 -0800 | [diff] [blame] | 72 | void |
| 73 | segment_manager_app_detach (segment_manager_t * sm) |
| 74 | { |
Florin Coras | c8e812f | 2020-05-14 05:32:18 +0000 | [diff] [blame] | 75 | sm->flags |= SEG_MANAGER_F_DETACHED; |
Florin Coras | 4e4531e | 2017-11-06 23:27:56 -0800 | [diff] [blame] | 76 | } |
| 77 | |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 78 | always_inline u32 |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 79 | segment_manager_segment_index (segment_manager_t * sm, fifo_segment_t * seg) |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 80 | { |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 81 | return (seg - sm->segments); |
| 82 | } |
| 83 | |
| 84 | /** |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 85 | * Adds segment to segment manager's pool |
| 86 | * |
| 87 | * If needed a writer's lock is acquired before allocating a new segment |
| 88 | * to avoid affecting any of the segments pool readers. |
| 89 | */ |
Florin Coras | 6973c73 | 2021-06-17 15:53:38 -0700 | [diff] [blame] | 90 | static inline int |
| 91 | segment_manager_add_segment_inline (segment_manager_t *sm, uword segment_size, |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 92 | u8 notify_app, u8 flags, u8 need_lock) |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 93 | { |
| 94 | segment_manager_main_t *smm = &sm_main; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 95 | segment_manager_props_t *props; |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 96 | app_worker_t *app_wrk; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 97 | fifo_segment_t *fs; |
Florin Coras | 9a45bd8 | 2020-12-28 16:28:07 -0800 | [diff] [blame] | 98 | u32 fs_index = ~0; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 99 | u8 *seg_name; |
| 100 | int rv; |
| 101 | |
| 102 | props = segment_manager_properties_get (sm); |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 103 | app_wrk = app_worker_get (sm->app_wrk_index); |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 104 | |
| 105 | /* Not configured for addition of new segments and not first */ |
| 106 | if (!props->add_segment && !segment_size) |
| 107 | { |
Filip Tehlar | ac3c8dc | 2023-03-14 08:50:28 +0100 | [diff] [blame] | 108 | SESSION_DBG ("cannot allocate new segment"); |
| 109 | return SESSION_E_INVALID; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 110 | } |
| 111 | |
| 112 | /* |
Florin Coras | f9d4ab4 | 2019-05-11 16:55:53 -0700 | [diff] [blame] | 113 | * Allocate fifo segment and grab lock if needed |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 114 | */ |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 115 | if (need_lock) |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 116 | clib_rwlock_writer_lock (&sm->segments_rwlock); |
| 117 | |
| 118 | pool_get_zero (sm->segments, fs); |
| 119 | |
| 120 | /* |
Florin Coras | f9d4ab4 | 2019-05-11 16:55:53 -0700 | [diff] [blame] | 121 | * Allocate ssvm segment |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 122 | */ |
| 123 | segment_size = segment_size ? segment_size : props->add_segment_size; |
Filip Tehlar | 2711ca7 | 2021-11-24 10:30:59 +0000 | [diff] [blame] | 124 | /* add overhead to ensure the result segment size is at least |
| 125 | * of that requested */ |
| 126 | segment_size += |
| 127 | sizeof (fifo_segment_header_t) + |
| 128 | vlib_thread_main.n_vlib_mains * sizeof (fifo_segment_slice_t) + |
| 129 | FIFO_SEGMENT_ALLOC_OVERHEAD; |
Junfeng Wang | c795b88 | 2022-08-12 16:24:46 +0800 | [diff] [blame] | 130 | |
| 131 | if (props->huge_page) |
| 132 | { |
| 133 | uword hugepage_size = clib_mem_get_default_hugepage_size (); |
| 134 | segment_size = round_pow2 (segment_size, hugepage_size); |
| 135 | fs->ssvm.huge_page = 1; |
| 136 | } |
| 137 | else |
| 138 | segment_size = round_pow2 (segment_size, clib_mem_get_page_size ()); |
Florin Coras | f9d4ab4 | 2019-05-11 16:55:53 -0700 | [diff] [blame] | 139 | |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 140 | seg_name = format (0, "seg-%u-%u-%u%c", app_wrk->app_index, |
| 141 | app_wrk->wrk_index, smm->seg_name_counter++, 0); |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 142 | |
| 143 | fs->ssvm.ssvm_size = segment_size; |
| 144 | fs->ssvm.name = seg_name; |
Florin Coras | 9a45bd8 | 2020-12-28 16:28:07 -0800 | [diff] [blame] | 145 | fs->ssvm.requested_va = 0; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 146 | |
Florin Coras | 5220a26 | 2020-09-29 18:11:24 -0700 | [diff] [blame] | 147 | if ((rv = ssvm_server_init (&fs->ssvm, props->segment_type))) |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 148 | { |
| 149 | clib_warning ("svm_master_init ('%v', %u) failed", seg_name, |
| 150 | segment_size); |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 151 | pool_put (sm->segments, fs); |
| 152 | goto done; |
| 153 | } |
| 154 | |
Florin Coras | f9d4ab4 | 2019-05-11 16:55:53 -0700 | [diff] [blame] | 155 | /* |
| 156 | * Initialize fifo segment |
| 157 | */ |
Florin Coras | 62ddc03 | 2019-12-08 18:30:42 -0800 | [diff] [blame] | 158 | fs->n_slices = props->n_slices; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 159 | fifo_segment_init (fs); |
| 160 | |
| 161 | /* |
| 162 | * Save segment index before dropping lock, if any held |
| 163 | */ |
| 164 | fs_index = fs - sm->segments; |
Florin Coras | cdfe8ab | 2021-12-22 12:54:17 -0800 | [diff] [blame] | 165 | fs->fs_index = fs_index; |
| 166 | fs->sm_index = segment_manager_index (sm); |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 167 | |
Ryujiro Shibuya | 234fe89 | 2019-12-25 07:40:54 +0000 | [diff] [blame] | 168 | /* |
| 169 | * Set watermarks in segment |
| 170 | */ |
Florin Coras | afbb33a | 2021-08-10 16:56:34 -0700 | [diff] [blame] | 171 | fs->high_watermark = sm->high_watermark; |
| 172 | fs->low_watermark = sm->low_watermark; |
| 173 | fs->flags = flags; |
| 174 | fs->flags &= ~FIFO_SEGMENT_F_MEM_LIMIT; |
Florin Coras | 2de9c0f | 2020-02-02 19:30:39 +0000 | [diff] [blame] | 175 | fs->h->pct_first_alloc = props->pct_first_alloc; |
Ryujiro Shibuya | 234fe89 | 2019-12-25 07:40:54 +0000 | [diff] [blame] | 176 | |
liuyacan | 9f29903 | 2021-04-25 20:11:30 +0800 | [diff] [blame] | 177 | if (notify_app) |
| 178 | { |
| 179 | app_worker_t *app_wrk; |
| 180 | u64 fs_handle; |
| 181 | fs_handle = segment_manager_segment_handle (sm, fs); |
| 182 | app_wrk = app_worker_get (sm->app_wrk_index); |
| 183 | rv = app_worker_add_segment_notify (app_wrk, fs_handle); |
| 184 | if (rv) |
wanghanlin | 19f6e26 | 2021-11-29 15:05:23 +0800 | [diff] [blame] | 185 | { |
| 186 | fs_index = rv; |
| 187 | goto done; |
| 188 | } |
liuyacan | 9f29903 | 2021-04-25 20:11:30 +0800 | [diff] [blame] | 189 | } |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 190 | done: |
| 191 | |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 192 | if (need_lock) |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 193 | clib_rwlock_writer_unlock (&sm->segments_rwlock); |
| 194 | |
| 195 | return fs_index; |
| 196 | } |
| 197 | |
Florin Coras | 6973c73 | 2021-06-17 15:53:38 -0700 | [diff] [blame] | 198 | int |
| 199 | segment_manager_add_segment (segment_manager_t *sm, uword segment_size, |
| 200 | u8 notify_app) |
| 201 | { |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 202 | return segment_manager_add_segment_inline (sm, segment_size, notify_app, |
| 203 | 0 /* flags */, 0 /* need_lock */); |
Florin Coras | 6973c73 | 2021-06-17 15:53:38 -0700 | [diff] [blame] | 204 | } |
| 205 | |
| 206 | int |
| 207 | segment_manager_add_segment2 (segment_manager_t *sm, uword segment_size, |
| 208 | u8 flags) |
| 209 | { |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 210 | return segment_manager_add_segment_inline (sm, segment_size, 0, flags, |
| 211 | vlib_num_workers ()); |
Florin Coras | 6973c73 | 2021-06-17 15:53:38 -0700 | [diff] [blame] | 212 | } |
| 213 | |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 214 | /** |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 215 | * Remove segment without lock |
| 216 | */ |
Florin Coras | f8f516a | 2018-02-08 15:10:09 -0800 | [diff] [blame] | 217 | void |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 218 | segment_manager_del_segment (segment_manager_t * sm, fifo_segment_t * fs) |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 219 | { |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 220 | if (ssvm_type (&fs->ssvm) != SSVM_SEGMENT_PRIVATE) |
Florin Coras | a0dbf9e | 2019-03-01 17:12:02 -0800 | [diff] [blame] | 221 | { |
Florin Coras | ea7e708 | 2020-07-07 17:57:28 -0700 | [diff] [blame] | 222 | if (!segment_manager_app_detached (sm)) |
Florin Coras | a0dbf9e | 2019-03-01 17:12:02 -0800 | [diff] [blame] | 223 | { |
| 224 | app_worker_t *app_wrk; |
| 225 | u64 segment_handle; |
| 226 | app_wrk = app_worker_get (sm->app_wrk_index); |
| 227 | segment_handle = segment_manager_segment_handle (sm, fs); |
| 228 | app_worker_del_segment_notify (app_wrk, segment_handle); |
| 229 | } |
| 230 | } |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 231 | |
Florin Coras | c547e91 | 2020-12-08 17:50:45 -0800 | [diff] [blame] | 232 | fifo_segment_cleanup (fs); |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 233 | ssvm_delete (&fs->ssvm); |
| 234 | |
| 235 | if (CLIB_DEBUG) |
Dave Barach | b7b9299 | 2018-10-17 10:38:51 -0400 | [diff] [blame] | 236 | clib_memset (fs, 0xfb, sizeof (*fs)); |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 237 | pool_put (sm->segments, fs); |
| 238 | } |
| 239 | |
Florin Coras | 57660d9 | 2020-04-04 22:45:34 +0000 | [diff] [blame] | 240 | static fifo_segment_t * |
| 241 | segment_manager_get_segment_if_valid (segment_manager_t * sm, |
| 242 | u32 segment_index) |
| 243 | { |
| 244 | if (pool_is_free_index (sm->segments, segment_index)) |
| 245 | return 0; |
| 246 | return pool_elt_at_index (sm->segments, segment_index); |
| 247 | } |
| 248 | |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 249 | /** |
| 250 | * Removes segment after acquiring writer lock |
| 251 | */ |
Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 252 | static inline void |
Florin Coras | 02cc719 | 2021-11-27 14:44:59 -0800 | [diff] [blame] | 253 | sm_lock_and_del_segment_inline (segment_manager_t *sm, u32 fs_index, |
| 254 | u8 check_if_empty) |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 255 | { |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 256 | fifo_segment_t *fs; |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 257 | u8 is_prealloc; |
| 258 | |
| 259 | clib_rwlock_writer_lock (&sm->segments_rwlock); |
Florin Coras | 57660d9 | 2020-04-04 22:45:34 +0000 | [diff] [blame] | 260 | |
| 261 | fs = segment_manager_get_segment_if_valid (sm, fs_index); |
| 262 | if (!fs) |
| 263 | goto done; |
| 264 | |
Florin Coras | 02cc719 | 2021-11-27 14:44:59 -0800 | [diff] [blame] | 265 | if (check_if_empty && fifo_segment_has_fifos (fs)) |
| 266 | goto done; |
| 267 | |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 268 | is_prealloc = fifo_segment_flags (fs) & FIFO_SEGMENT_F_IS_PREALLOCATED; |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 269 | if (is_prealloc && !segment_manager_app_detached (sm)) |
Florin Coras | 57660d9 | 2020-04-04 22:45:34 +0000 | [diff] [blame] | 270 | goto done; |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 271 | |
| 272 | segment_manager_del_segment (sm, fs); |
Florin Coras | 57660d9 | 2020-04-04 22:45:34 +0000 | [diff] [blame] | 273 | |
| 274 | done: |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 275 | clib_rwlock_writer_unlock (&sm->segments_rwlock); |
| 276 | } |
| 277 | |
Florin Coras | 2d0e3de | 2020-10-23 16:31:40 -0700 | [diff] [blame] | 278 | void |
| 279 | segment_manager_lock_and_del_segment (segment_manager_t * sm, u32 fs_index) |
| 280 | { |
Florin Coras | 02cc719 | 2021-11-27 14:44:59 -0800 | [diff] [blame] | 281 | sm_lock_and_del_segment_inline (sm, fs_index, 0 /* check_if_empty */); |
Florin Coras | 2d0e3de | 2020-10-23 16:31:40 -0700 | [diff] [blame] | 282 | } |
| 283 | |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 284 | /** |
| 285 | * Reads a segment from the segment manager's pool without lock |
| 286 | */ |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 287 | fifo_segment_t * |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 288 | segment_manager_get_segment (segment_manager_t * sm, u32 segment_index) |
| 289 | { |
| 290 | return pool_elt_at_index (sm->segments, segment_index); |
| 291 | } |
| 292 | |
Florin Coras | fa76a76 | 2018-11-29 12:40:10 -0800 | [diff] [blame] | 293 | u64 |
| 294 | segment_manager_segment_handle (segment_manager_t * sm, |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 295 | fifo_segment_t * segment) |
Florin Coras | fa76a76 | 2018-11-29 12:40:10 -0800 | [diff] [blame] | 296 | { |
| 297 | u32 segment_index = segment_manager_segment_index (sm, segment); |
| 298 | return (((u64) segment_manager_index (sm) << 32) | segment_index); |
| 299 | } |
| 300 | |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 301 | u64 |
| 302 | segment_manager_make_segment_handle (u32 segment_manager_index, |
| 303 | u32 segment_index) |
| 304 | { |
| 305 | return (((u64) segment_manager_index << 32) | segment_index); |
| 306 | } |
| 307 | |
| 308 | fifo_segment_t * |
Florin Coras | fa76a76 | 2018-11-29 12:40:10 -0800 | [diff] [blame] | 309 | segment_manager_get_segment_w_handle (u64 segment_handle) |
| 310 | { |
| 311 | u32 sm_index, segment_index; |
| 312 | segment_manager_t *sm; |
| 313 | |
| 314 | segment_manager_parse_segment_handle (segment_handle, &sm_index, |
| 315 | &segment_index); |
| 316 | sm = segment_manager_get (sm_index); |
| 317 | if (!sm || pool_is_free_index (sm->segments, segment_index)) |
| 318 | return 0; |
| 319 | return pool_elt_at_index (sm->segments, segment_index); |
| 320 | } |
| 321 | |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 322 | /** |
| 323 | * Reads a segment from the segment manager's pool and acquires reader lock |
| 324 | * |
| 325 | * Caller must drop the reader's lock by calling |
| 326 | * @ref segment_manager_segment_reader_unlock once it finishes working with |
| 327 | * the segment. |
| 328 | */ |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 329 | fifo_segment_t * |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 330 | segment_manager_get_segment_w_lock (segment_manager_t * sm, u32 segment_index) |
| 331 | { |
| 332 | clib_rwlock_reader_lock (&sm->segments_rwlock); |
| 333 | return pool_elt_at_index (sm->segments, segment_index); |
| 334 | } |
| 335 | |
| 336 | void |
Florin Coras | 75ccf7b | 2020-03-05 19:44:02 +0000 | [diff] [blame] | 337 | segment_manager_segment_reader_lock (segment_manager_t * sm) |
| 338 | { |
| 339 | clib_rwlock_reader_lock (&sm->segments_rwlock); |
| 340 | } |
| 341 | |
| 342 | void |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 343 | segment_manager_segment_reader_unlock (segment_manager_t * sm) |
| 344 | { |
| 345 | clib_rwlock_reader_unlock (&sm->segments_rwlock); |
| 346 | } |
| 347 | |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 348 | segment_manager_t * |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 349 | segment_manager_alloc (void) |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 350 | { |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 351 | segment_manager_main_t *smm = &sm_main; |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 352 | segment_manager_t *sm; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 353 | |
| 354 | pool_get_zero (smm->segment_managers, sm); |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 355 | clib_rwlock_init (&sm->segments_rwlock); |
| 356 | return sm; |
| 357 | } |
| 358 | |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 359 | int |
Ryujiro Shibuya | 234fe89 | 2019-12-25 07:40:54 +0000 | [diff] [blame] | 360 | segment_manager_init (segment_manager_t * sm) |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 361 | { |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 362 | segment_manager_props_t *props; |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 363 | |
| 364 | props = segment_manager_properties_get (sm); |
Ryujiro Shibuya | 234fe89 | 2019-12-25 07:40:54 +0000 | [diff] [blame] | 365 | |
Ryujiro Shibuya | d8f48e2 | 2020-01-22 12:11:42 +0000 | [diff] [blame] | 366 | sm->max_fifo_size = props->max_fifo_size ? |
| 367 | props->max_fifo_size : sm_main.default_max_fifo_size; |
| 368 | sm->max_fifo_size = clib_max (sm->max_fifo_size, 4096); |
| 369 | |
Ryujiro Shibuya | 234fe89 | 2019-12-25 07:40:54 +0000 | [diff] [blame] | 370 | segment_manager_set_watermarks (sm, |
| 371 | props->high_watermark, |
| 372 | props->low_watermark); |
Florin Coras | a107f40 | 2020-09-29 19:18:46 -0700 | [diff] [blame] | 373 | return 0; |
| 374 | } |
| 375 | |
| 376 | /** |
| 377 | * Initializes segment manager based on options provided. |
| 378 | * Returns error if ssvm segment(s) allocation fails. |
| 379 | */ |
| 380 | int |
| 381 | segment_manager_init_first (segment_manager_t * sm) |
| 382 | { |
| 383 | segment_manager_props_t *props; |
| 384 | uword first_seg_size; |
| 385 | fifo_segment_t *fs; |
| 386 | int fs_index, i; |
| 387 | |
| 388 | segment_manager_init (sm); |
| 389 | props = segment_manager_properties_get (sm); |
| 390 | first_seg_size = clib_max (props->segment_size, |
| 391 | sm_main.default_segment_size); |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 392 | |
Florin Coras | 9845c20 | 2020-04-28 01:54:22 +0000 | [diff] [blame] | 393 | if (props->prealloc_fifos) |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 394 | { |
Florin Coras | 9845c20 | 2020-04-28 01:54:22 +0000 | [diff] [blame] | 395 | u64 approx_total_size, max_seg_size = ((u64) 1 << 32) - (128 << 10); |
| 396 | u32 rx_rounded_data_size, tx_rounded_data_size; |
| 397 | u32 prealloc_fifo_pairs = props->prealloc_fifos; |
| 398 | u32 rx_fifo_size, tx_fifo_size, pair_size; |
| 399 | u32 approx_segment_count; |
| 400 | |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 401 | /* Figure out how many segments should be preallocated */ |
| 402 | rx_rounded_data_size = (1 << (max_log2 (props->rx_fifo_size))); |
| 403 | tx_rounded_data_size = (1 << (max_log2 (props->tx_fifo_size))); |
| 404 | |
| 405 | rx_fifo_size = sizeof (svm_fifo_t) + rx_rounded_data_size; |
| 406 | tx_fifo_size = sizeof (svm_fifo_t) + tx_rounded_data_size; |
| 407 | pair_size = rx_fifo_size + tx_fifo_size; |
| 408 | |
| 409 | approx_total_size = (u64) prealloc_fifo_pairs *pair_size; |
| 410 | if (first_seg_size > approx_total_size) |
| 411 | max_seg_size = first_seg_size; |
| 412 | approx_segment_count = (approx_total_size + (max_seg_size - 1)) |
| 413 | / max_seg_size; |
| 414 | |
| 415 | /* Allocate the segments */ |
| 416 | for (i = 0; i < approx_segment_count + 1; i++) |
| 417 | { |
liuyacan | 9f29903 | 2021-04-25 20:11:30 +0800 | [diff] [blame] | 418 | fs_index = segment_manager_add_segment (sm, max_seg_size, 0); |
Florin Coras | 9845c20 | 2020-04-28 01:54:22 +0000 | [diff] [blame] | 419 | if (fs_index < 0) |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 420 | { |
Filip Tehlar | ac3c8dc | 2023-03-14 08:50:28 +0100 | [diff] [blame] | 421 | SESSION_DBG ("Failed to preallocate segment %d", i); |
Florin Coras | 9845c20 | 2020-04-28 01:54:22 +0000 | [diff] [blame] | 422 | return fs_index; |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 423 | } |
| 424 | |
Florin Coras | 9845c20 | 2020-04-28 01:54:22 +0000 | [diff] [blame] | 425 | fs = segment_manager_get_segment (sm, fs_index); |
Florin Coras | f8f516a | 2018-02-08 15:10:09 -0800 | [diff] [blame] | 426 | if (i == 0) |
Florin Coras | 9845c20 | 2020-04-28 01:54:22 +0000 | [diff] [blame] | 427 | sm->event_queue = segment_manager_alloc_queue (fs, props); |
Florin Coras | f8f516a | 2018-02-08 15:10:09 -0800 | [diff] [blame] | 428 | |
Florin Coras | 9845c20 | 2020-04-28 01:54:22 +0000 | [diff] [blame] | 429 | fifo_segment_preallocate_fifo_pairs (fs, |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 430 | props->rx_fifo_size, |
| 431 | props->tx_fifo_size, |
| 432 | &prealloc_fifo_pairs); |
Florin Coras | 9845c20 | 2020-04-28 01:54:22 +0000 | [diff] [blame] | 433 | fifo_segment_flags (fs) = FIFO_SEGMENT_F_IS_PREALLOCATED; |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 434 | if (prealloc_fifo_pairs == 0) |
| 435 | break; |
| 436 | } |
Florin Coras | 9845c20 | 2020-04-28 01:54:22 +0000 | [diff] [blame] | 437 | return 0; |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 438 | } |
Florin Coras | 9845c20 | 2020-04-28 01:54:22 +0000 | [diff] [blame] | 439 | |
liuyacan | 9f29903 | 2021-04-25 20:11:30 +0800 | [diff] [blame] | 440 | fs_index = segment_manager_add_segment (sm, first_seg_size, 0); |
Florin Coras | 9845c20 | 2020-04-28 01:54:22 +0000 | [diff] [blame] | 441 | if (fs_index < 0) |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 442 | { |
Filip Tehlar | ac3c8dc | 2023-03-14 08:50:28 +0100 | [diff] [blame] | 443 | SESSION_DBG ("Failed to allocate segment"); |
Florin Coras | 9845c20 | 2020-04-28 01:54:22 +0000 | [diff] [blame] | 444 | return fs_index; |
| 445 | } |
| 446 | |
| 447 | fs = segment_manager_get_segment (sm, fs_index); |
| 448 | sm->event_queue = segment_manager_alloc_queue (fs, props); |
| 449 | |
| 450 | if (props->prealloc_fifo_hdrs) |
| 451 | { |
| 452 | u32 hdrs_per_slice; |
| 453 | |
| 454 | /* Do not preallocate on slice associated to main thread */ |
| 455 | i = (vlib_num_workers ()? 1 : 0); |
| 456 | hdrs_per_slice = props->prealloc_fifo_hdrs / (fs->n_slices - i); |
| 457 | |
| 458 | for (; i < fs->n_slices; i++) |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 459 | { |
Florin Coras | 9845c20 | 2020-04-28 01:54:22 +0000 | [diff] [blame] | 460 | if (fifo_segment_prealloc_fifo_hdrs (fs, i, hdrs_per_slice)) |
Filip Tehlar | ac3c8dc | 2023-03-14 08:50:28 +0100 | [diff] [blame] | 461 | return SESSION_E_SEG_CREATE; |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 462 | } |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 463 | } |
| 464 | |
| 465 | return 0; |
| 466 | } |
| 467 | |
Florin Coras | c8e812f | 2020-05-14 05:32:18 +0000 | [diff] [blame] | 468 | void |
| 469 | segment_manager_cleanup_detached_listener (segment_manager_t * sm) |
| 470 | { |
| 471 | app_worker_t *app_wrk; |
| 472 | |
| 473 | app_wrk = app_worker_get_if_valid (sm->app_wrk_index); |
| 474 | if (!app_wrk) |
| 475 | return; |
| 476 | |
| 477 | app_worker_del_detached_sm (app_wrk, segment_manager_index (sm)); |
| 478 | } |
| 479 | |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 480 | /** |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 481 | * Cleanup segment manager. |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 482 | */ |
| 483 | void |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 484 | segment_manager_free (segment_manager_t * sm) |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 485 | { |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 486 | segment_manager_main_t *smm = &sm_main; |
| 487 | fifo_segment_t *fifo_segment; |
Dave Wallace | 7b749fe | 2017-07-05 14:30:46 -0400 | [diff] [blame] | 488 | |
Florin Coras | 506aa3c | 2020-12-13 21:09:59 -0800 | [diff] [blame] | 489 | ASSERT (vlib_get_thread_index () == 0 |
| 490 | && !segment_manager_has_fifos (sm) |
Florin Coras | 9d06304 | 2017-09-14 03:08:00 -0400 | [diff] [blame] | 491 | && segment_manager_app_detached (sm)); |
| 492 | |
Florin Coras | c8e812f | 2020-05-14 05:32:18 +0000 | [diff] [blame] | 493 | if (sm->flags & SEG_MANAGER_F_DETACHED_LISTENER) |
| 494 | segment_manager_cleanup_detached_listener (sm); |
| 495 | |
Florin Coras | 9d06304 | 2017-09-14 03:08:00 -0400 | [diff] [blame] | 496 | /* If we have empty preallocated segments that haven't been removed, remove |
| 497 | * them now. Apart from that, the first segment in the first segment manager |
| 498 | * is not removed when all fifos are removed. It can only be removed when |
| 499 | * the manager is explicitly deleted/detached by the app. */ |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 500 | clib_rwlock_writer_lock (&sm->segments_rwlock); |
| 501 | |
| 502 | /* *INDENT-OFF* */ |
Damjan Marion | b2c31b6 | 2020-12-13 21:47:40 +0100 | [diff] [blame] | 503 | pool_foreach (fifo_segment, sm->segments) { |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 504 | segment_manager_del_segment (sm, fifo_segment); |
Damjan Marion | b2c31b6 | 2020-12-13 21:47:40 +0100 | [diff] [blame] | 505 | } |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 506 | /* *INDENT-ON* */ |
| 507 | |
Florin Coras | f1af21c | 2021-02-26 19:19:11 -0800 | [diff] [blame] | 508 | pool_free (sm->segments); |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 509 | clib_rwlock_writer_unlock (&sm->segments_rwlock); |
| 510 | |
| 511 | clib_rwlock_free (&sm->segments_rwlock); |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 512 | if (CLIB_DEBUG) |
Dave Barach | b7b9299 | 2018-10-17 10:38:51 -0400 | [diff] [blame] | 513 | clib_memset (sm, 0xfe, sizeof (*sm)); |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 514 | pool_put (smm->segment_managers, sm); |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 515 | } |
| 516 | |
Florin Coras | 506aa3c | 2020-12-13 21:09:59 -0800 | [diff] [blame] | 517 | static void |
| 518 | sm_free_w_index_helper (void *arg) |
| 519 | { |
| 520 | u32 sm_index = *(u32 *) arg; |
| 521 | segment_manager_t *sm; |
| 522 | |
| 523 | ASSERT (vlib_get_thread_index () == 0); |
| 524 | |
| 525 | if ((sm = segment_manager_get_if_valid (sm_index))) |
| 526 | segment_manager_free (sm); |
| 527 | } |
| 528 | |
Florin Coras | da78c5a | 2021-06-09 14:55:24 -0700 | [diff] [blame] | 529 | void |
| 530 | segment_manager_free_safe (segment_manager_t *sm) |
Florin Coras | 506aa3c | 2020-12-13 21:09:59 -0800 | [diff] [blame] | 531 | { |
| 532 | if (!vlib_thread_is_main_w_barrier ()) |
| 533 | { |
| 534 | u32 sm_index = segment_manager_index (sm); |
| 535 | vlib_rpc_call_main_thread (sm_free_w_index_helper, (u8 *) & sm_index, |
| 536 | sizeof (sm_index)); |
| 537 | } |
| 538 | else |
| 539 | { |
| 540 | segment_manager_free (sm); |
| 541 | } |
| 542 | } |
| 543 | |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 544 | void |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 545 | segment_manager_init_free (segment_manager_t * sm) |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 546 | { |
Florin Coras | 506aa3c | 2020-12-13 21:09:59 -0800 | [diff] [blame] | 547 | ASSERT (vlib_get_thread_index () == 0); |
| 548 | |
Florin Coras | 4e4531e | 2017-11-06 23:27:56 -0800 | [diff] [blame] | 549 | segment_manager_app_detach (sm); |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 550 | if (segment_manager_has_fifos (sm)) |
| 551 | segment_manager_del_sessions (sm); |
| 552 | else |
| 553 | { |
Florin Coras | 9d06304 | 2017-09-14 03:08:00 -0400 | [diff] [blame] | 554 | ASSERT (!sm->first_is_protected || segment_manager_app_detached (sm)); |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 555 | segment_manager_free (sm); |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 556 | } |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 557 | } |
| 558 | |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 559 | segment_manager_t * |
| 560 | segment_manager_get (u32 index) |
| 561 | { |
| 562 | return pool_elt_at_index (sm_main.segment_managers, index); |
| 563 | } |
| 564 | |
| 565 | segment_manager_t * |
| 566 | segment_manager_get_if_valid (u32 index) |
| 567 | { |
| 568 | if (pool_is_free_index (sm_main.segment_managers, index)) |
| 569 | return 0; |
| 570 | return pool_elt_at_index (sm_main.segment_managers, index); |
| 571 | } |
| 572 | |
| 573 | u32 |
| 574 | segment_manager_index (segment_manager_t * sm) |
| 575 | { |
| 576 | return sm - sm_main.segment_managers; |
| 577 | } |
| 578 | |
| 579 | u8 |
| 580 | segment_manager_has_fifos (segment_manager_t * sm) |
| 581 | { |
| 582 | fifo_segment_t *seg; |
| 583 | u8 first = 1; |
| 584 | |
| 585 | /* *INDENT-OFF* */ |
| 586 | segment_manager_foreach_segment_w_lock (seg, sm, ({ |
| 587 | if (CLIB_DEBUG && !first && !fifo_segment_has_fifos (seg) |
| 588 | && !(fifo_segment_flags (seg) & FIFO_SEGMENT_F_IS_PREALLOCATED)) |
| 589 | { |
| 590 | clib_warning ("segment %d has no fifos!", |
| 591 | segment_manager_segment_index (sm, seg)); |
| 592 | first = 0; |
| 593 | } |
| 594 | if (fifo_segment_has_fifos (seg)) |
| 595 | { |
| 596 | segment_manager_segment_reader_unlock (sm); |
| 597 | return 1; |
| 598 | } |
| 599 | })); |
| 600 | /* *INDENT-ON* */ |
| 601 | |
| 602 | return 0; |
| 603 | } |
| 604 | |
| 605 | /** |
| 606 | * Initiate disconnects for all sessions 'owned' by a segment manager |
| 607 | */ |
| 608 | void |
| 609 | segment_manager_del_sessions (segment_manager_t * sm) |
| 610 | { |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 611 | session_handle_t *handles = 0, *handle; |
Florin Coras | 62ddc03 | 2019-12-08 18:30:42 -0800 | [diff] [blame] | 612 | fifo_segment_t *fs; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 613 | session_t *session; |
Florin Coras | 62ddc03 | 2019-12-08 18:30:42 -0800 | [diff] [blame] | 614 | int slice_index; |
| 615 | svm_fifo_t *f; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 616 | |
| 617 | ASSERT (pool_elts (sm->segments) != 0); |
| 618 | |
| 619 | /* Across all fifo segments used by the server */ |
| 620 | /* *INDENT-OFF* */ |
Florin Coras | 62ddc03 | 2019-12-08 18:30:42 -0800 | [diff] [blame] | 621 | segment_manager_foreach_segment_w_lock (fs, sm, ({ |
| 622 | for (slice_index = 0; slice_index < fs->n_slices; slice_index++) |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 623 | { |
Florin Coras | 62ddc03 | 2019-12-08 18:30:42 -0800 | [diff] [blame] | 624 | f = fifo_segment_get_slice_fifo_list (fs, slice_index); |
| 625 | |
| 626 | /* |
| 627 | * Remove any residual sessions from the session lookup table |
| 628 | * Don't bother deleting the individual fifos, we're going to |
| 629 | * throw away the fifo segment in a minute. |
| 630 | */ |
| 631 | while (f) |
| 632 | { |
Florin Coras | c547e91 | 2020-12-08 17:50:45 -0800 | [diff] [blame] | 633 | session = session_get_if_valid (f->shr->master_session_index, |
| 634 | f->master_thread_index); |
| 635 | if (session) |
| 636 | vec_add1 (handles, session_handle (session)); |
| 637 | f = f->next; |
| 638 | } |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 639 | } |
| 640 | |
| 641 | /* Instead of removing the segment, test when cleaning up disconnected |
| 642 | * sessions if the segment can be removed. |
| 643 | */ |
| 644 | })); |
| 645 | /* *INDENT-ON* */ |
| 646 | |
| 647 | vec_foreach (handle, handles) |
Florin Coras | 77ea42b | 2020-04-14 23:52:12 +0000 | [diff] [blame] | 648 | { |
| 649 | session = session_get_from_handle (*handle); |
| 650 | session_close (session); |
| 651 | /* Avoid propagating notifications back to the app */ |
| 652 | session->app_wrk_index = APP_INVALID_INDEX; |
| 653 | } |
Florin Coras | f1af21c | 2021-02-26 19:19:11 -0800 | [diff] [blame] | 654 | vec_free (handles); |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 655 | } |
| 656 | |
liuyacan | 87d48ad | 2021-04-28 11:34:03 +0000 | [diff] [blame] | 657 | /** |
| 658 | * Initiate disconnects for sessions in specified state 'owned' by a segment |
| 659 | * manager |
| 660 | */ |
| 661 | void |
| 662 | segment_manager_del_sessions_filter (segment_manager_t *sm, |
| 663 | session_state_t *states) |
| 664 | { |
| 665 | session_handle_t *handles = 0, *handle; |
| 666 | fifo_segment_t *fs; |
| 667 | session_t *session; |
| 668 | int slice_index; |
| 669 | svm_fifo_t *f; |
| 670 | |
| 671 | ASSERT (pool_elts (sm->segments) != 0); |
| 672 | |
| 673 | /* Across all fifo segments used by the server */ |
| 674 | segment_manager_foreach_segment_w_lock ( |
| 675 | fs, sm, ({ |
| 676 | for (slice_index = 0; slice_index < fs->n_slices; slice_index++) |
| 677 | { |
| 678 | f = fifo_segment_get_slice_fifo_list (fs, slice_index); |
| 679 | while (f) |
| 680 | { |
| 681 | session = session_get_if_valid (f->shr->master_session_index, |
| 682 | f->master_thread_index); |
| 683 | if (session) |
| 684 | { |
| 685 | session_state_t *state; |
| 686 | vec_foreach (state, states) |
| 687 | { |
| 688 | if (session->session_state == *state) |
| 689 | { |
| 690 | vec_add1 (handles, session_handle (session)); |
| 691 | break; |
| 692 | } |
| 693 | } |
| 694 | } |
| 695 | f = f->next; |
| 696 | } |
| 697 | } |
| 698 | })); |
| 699 | |
| 700 | vec_foreach (handle, handles) |
| 701 | { |
| 702 | session = session_get_from_handle (*handle); |
| 703 | session_close (session); |
| 704 | /* Avoid propagating notifications back to the app */ |
| 705 | session->app_wrk_index = APP_INVALID_INDEX; |
| 706 | } |
| 707 | vec_free (handles); |
| 708 | } |
| 709 | |
Florin Coras | f8f516a | 2018-02-08 15:10:09 -0800 | [diff] [blame] | 710 | int |
Florin Coras | cdfe8ab | 2021-12-22 12:54:17 -0800 | [diff] [blame] | 711 | segment_manager_try_alloc_fifos (fifo_segment_t *fs, u32 thread_index, |
Florin Coras | f8f516a | 2018-02-08 15:10:09 -0800 | [diff] [blame] | 712 | u32 rx_fifo_size, u32 tx_fifo_size, |
Florin Coras | cdfe8ab | 2021-12-22 12:54:17 -0800 | [diff] [blame] | 713 | svm_fifo_t **rx_fifo, svm_fifo_t **tx_fifo) |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 714 | { |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 715 | rx_fifo_size = clib_max (rx_fifo_size, sm_main.default_fifo_size); |
Florin Coras | cdfe8ab | 2021-12-22 12:54:17 -0800 | [diff] [blame] | 716 | *rx_fifo = fifo_segment_alloc_fifo_w_slice (fs, thread_index, rx_fifo_size, |
Florin Coras | 62ddc03 | 2019-12-08 18:30:42 -0800 | [diff] [blame] | 717 | FIFO_SEGMENT_RX_FIFO); |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 718 | |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 719 | tx_fifo_size = clib_max (tx_fifo_size, sm_main.default_fifo_size); |
Florin Coras | cdfe8ab | 2021-12-22 12:54:17 -0800 | [diff] [blame] | 720 | *tx_fifo = fifo_segment_alloc_fifo_w_slice (fs, thread_index, tx_fifo_size, |
Florin Coras | 62ddc03 | 2019-12-08 18:30:42 -0800 | [diff] [blame] | 721 | FIFO_SEGMENT_TX_FIFO); |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 722 | |
| 723 | if (*rx_fifo == 0) |
| 724 | { |
| 725 | /* This would be very odd, but handle it... */ |
| 726 | if (*tx_fifo != 0) |
| 727 | { |
Florin Coras | cdfe8ab | 2021-12-22 12:54:17 -0800 | [diff] [blame] | 728 | fifo_segment_free_fifo (fs, *tx_fifo); |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 729 | *tx_fifo = 0; |
| 730 | } |
Florin Coras | cdfe8ab | 2021-12-22 12:54:17 -0800 | [diff] [blame] | 731 | return SESSION_E_SEG_NO_SPACE; |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 732 | } |
| 733 | if (*tx_fifo == 0) |
| 734 | { |
| 735 | if (*rx_fifo != 0) |
| 736 | { |
Florin Coras | cdfe8ab | 2021-12-22 12:54:17 -0800 | [diff] [blame] | 737 | fifo_segment_free_fifo (fs, *rx_fifo); |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 738 | *rx_fifo = 0; |
| 739 | } |
Florin Coras | cdfe8ab | 2021-12-22 12:54:17 -0800 | [diff] [blame] | 740 | return SESSION_E_SEG_NO_SPACE; |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 741 | } |
| 742 | |
| 743 | return 0; |
| 744 | } |
| 745 | |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 746 | static inline int |
| 747 | sm_lookup_segment_and_alloc_fifos (segment_manager_t *sm, |
| 748 | segment_manager_props_t *props, |
| 749 | u32 thread_index, svm_fifo_t **rx_fifo, |
| 750 | svm_fifo_t **tx_fifo) |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 751 | { |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 752 | uword free_bytes, max_free_bytes; |
| 753 | fifo_segment_t *cur, *fs = 0; |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 754 | |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 755 | max_free_bytes = props->rx_fifo_size + props->tx_fifo_size - 1; |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 756 | |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 757 | pool_foreach (cur, sm->segments) |
| 758 | { |
Florin Coras | da78c5a | 2021-06-09 14:55:24 -0700 | [diff] [blame] | 759 | if (fifo_segment_flags (cur) & FIFO_SEGMENT_F_CUSTOM_USE) |
| 760 | continue; |
| 761 | free_bytes = fifo_segment_available_bytes (cur); |
| 762 | if (free_bytes > max_free_bytes) |
| 763 | { |
| 764 | max_free_bytes = free_bytes; |
| 765 | fs = cur; |
| 766 | } |
Florin Coras | 75ccf7b | 2020-03-05 19:44:02 +0000 | [diff] [blame] | 767 | } |
Ryujiro Shibuya | 234fe89 | 2019-12-25 07:40:54 +0000 | [diff] [blame] | 768 | |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 769 | if (PREDICT_FALSE (!fs)) |
| 770 | return SESSION_E_SEG_NO_SPACE; |
| 771 | |
Florin Coras | cdfe8ab | 2021-12-22 12:54:17 -0800 | [diff] [blame] | 772 | return segment_manager_try_alloc_fifos ( |
| 773 | fs, thread_index, props->rx_fifo_size, props->tx_fifo_size, rx_fifo, |
| 774 | tx_fifo); |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 775 | } |
| 776 | |
| 777 | static int |
| 778 | sm_lock_and_alloc_segment_and_fifos (segment_manager_t *sm, |
| 779 | segment_manager_props_t *props, |
| 780 | u32 thread_index, svm_fifo_t **rx_fifo, |
| 781 | svm_fifo_t **tx_fifo) |
| 782 | { |
| 783 | int new_fs_index, rv; |
| 784 | fifo_segment_t *fs; |
| 785 | |
| 786 | if (!props->add_segment) |
| 787 | return SESSION_E_SEG_NO_SPACE; |
| 788 | |
| 789 | clib_rwlock_writer_lock (&sm->segments_rwlock); |
| 790 | |
| 791 | /* Make sure there really is no free space. Another worker might've freed |
| 792 | * some fifos or allocated a segment */ |
| 793 | rv = sm_lookup_segment_and_alloc_fifos (sm, props, thread_index, rx_fifo, |
| 794 | tx_fifo); |
Florin Coras | cdfe8ab | 2021-12-22 12:54:17 -0800 | [diff] [blame] | 795 | if (!rv) |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 796 | goto done; |
| 797 | |
| 798 | new_fs_index = |
| 799 | segment_manager_add_segment (sm, 0 /* segment_size*/, 1 /* notify_app */); |
| 800 | if (new_fs_index < 0) |
| 801 | { |
| 802 | rv = SESSION_E_SEG_CREATE; |
| 803 | goto done; |
| 804 | } |
| 805 | fs = segment_manager_get_segment (sm, new_fs_index); |
| 806 | rv = segment_manager_try_alloc_fifos (fs, thread_index, props->rx_fifo_size, |
| 807 | props->tx_fifo_size, rx_fifo, tx_fifo); |
| 808 | if (rv) |
| 809 | { |
Filip Tehlar | ac3c8dc | 2023-03-14 08:50:28 +0100 | [diff] [blame] | 810 | SESSION_DBG ("Added a segment, still can't allocate a fifo"); |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 811 | rv = SESSION_E_SEG_NO_SPACE2; |
| 812 | goto done; |
| 813 | } |
| 814 | |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 815 | done: |
| 816 | |
| 817 | clib_rwlock_writer_unlock (&sm->segments_rwlock); |
| 818 | |
| 819 | return rv; |
| 820 | } |
| 821 | |
| 822 | int |
| 823 | segment_manager_alloc_session_fifos (segment_manager_t * sm, |
| 824 | u32 thread_index, |
| 825 | svm_fifo_t ** rx_fifo, |
| 826 | svm_fifo_t ** tx_fifo) |
| 827 | { |
| 828 | segment_manager_props_t *props; |
Florin Coras | cdfe8ab | 2021-12-22 12:54:17 -0800 | [diff] [blame] | 829 | int rv; |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 830 | |
| 831 | props = segment_manager_properties_get (sm); |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 832 | |
| 833 | /* |
| 834 | * Fast path: find the first segment with enough free space and |
| 835 | * try to allocate the fifos. Done with reader lock |
| 836 | */ |
| 837 | |
| 838 | segment_manager_segment_reader_lock (sm); |
| 839 | |
Florin Coras | cdfe8ab | 2021-12-22 12:54:17 -0800 | [diff] [blame] | 840 | rv = sm_lookup_segment_and_alloc_fifos (sm, props, thread_index, rx_fifo, |
| 841 | tx_fifo); |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 842 | |
Florin Coras | 75ccf7b | 2020-03-05 19:44:02 +0000 | [diff] [blame] | 843 | segment_manager_segment_reader_unlock (sm); |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 844 | |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 845 | /* |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 846 | * Slow path: if no fifo segment or alloc fail grab writer lock and try |
| 847 | * to allocate new segment |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 848 | */ |
Florin Coras | cdfe8ab | 2021-12-22 12:54:17 -0800 | [diff] [blame] | 849 | if (PREDICT_FALSE (rv < 0)) |
| 850 | return sm_lock_and_alloc_segment_and_fifos (sm, props, thread_index, |
| 851 | rx_fifo, tx_fifo); |
liuyacan | 9f29903 | 2021-04-25 20:11:30 +0800 | [diff] [blame] | 852 | |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 853 | return 0; |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 854 | } |
| 855 | |
| 856 | void |
Florin Coras | 19223e0 | 2019-03-03 14:56:05 -0800 | [diff] [blame] | 857 | segment_manager_dealloc_fifos (svm_fifo_t * rx_fifo, svm_fifo_t * tx_fifo) |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 858 | { |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 859 | segment_manager_t *sm; |
Florin Coras | b095a3c | 2019-04-25 12:58:46 -0700 | [diff] [blame] | 860 | fifo_segment_t *fs; |
Florin Coras | 19223e0 | 2019-03-03 14:56:05 -0800 | [diff] [blame] | 861 | u32 segment_index; |
Florin Coras | 02cc719 | 2021-11-27 14:44:59 -0800 | [diff] [blame] | 862 | u8 try_delete = 0; |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 863 | |
Florin Coras | 58a93e8 | 2019-01-14 23:33:46 -0800 | [diff] [blame] | 864 | if (!rx_fifo || !tx_fifo) |
| 865 | return; |
| 866 | |
Florin Coras | 5122234 | 2022-06-01 14:44:11 -0700 | [diff] [blame] | 867 | /* Thread that allocated the fifos must be the one to clean them up */ |
| 868 | ASSERT (rx_fifo->master_thread_index == vlib_get_thread_index () || |
| 869 | rx_fifo->refcnt > 1); |
| 870 | |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 871 | /* It's possible to have no segment manager if the session was removed |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 872 | * as result of a detach. */ |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 873 | if (!(sm = segment_manager_get_if_valid (rx_fifo->segment_manager))) |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 874 | return; |
| 875 | |
Florin Coras | 19223e0 | 2019-03-03 14:56:05 -0800 | [diff] [blame] | 876 | segment_index = rx_fifo->segment_index; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 877 | fs = segment_manager_get_segment_w_lock (sm, segment_index); |
| 878 | fifo_segment_free_fifo (fs, rx_fifo); |
| 879 | fifo_segment_free_fifo (fs, tx_fifo); |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 880 | |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 881 | /* |
Florin Coras | 02cc719 | 2021-11-27 14:44:59 -0800 | [diff] [blame] | 882 | * Try to remove fifo segment if it has no fifos. This can be done only if |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 883 | * the segment is not the first in the segment manager or if it is first |
| 884 | * and it is not protected. Moreover, if the segment is first and the app |
| 885 | * has detached from the segment manager, remove the segment manager. |
| 886 | */ |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 887 | if (!fifo_segment_has_fifos (fs)) |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 888 | { |
Florin Coras | 02cc719 | 2021-11-27 14:44:59 -0800 | [diff] [blame] | 889 | /* If first, remove only if not protected */ |
| 890 | try_delete = segment_index != 0 || !sm->first_is_protected; |
| 891 | } |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 892 | |
Florin Coras | 02cc719 | 2021-11-27 14:44:59 -0800 | [diff] [blame] | 893 | segment_manager_segment_reader_unlock (sm); |
| 894 | |
| 895 | if (PREDICT_FALSE (try_delete)) |
| 896 | { |
| 897 | /* Only remove if empty after writer lock acquired */ |
| 898 | sm_lock_and_del_segment_inline (sm, segment_index, |
| 899 | 1 /* check_if_empty */); |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 900 | |
| 901 | /* Remove segment manager if no sessions and detached from app */ |
Florin Coras | 9d06304 | 2017-09-14 03:08:00 -0400 | [diff] [blame] | 902 | if (segment_manager_app_detached (sm) |
| 903 | && !segment_manager_has_fifos (sm)) |
Florin Coras | 506aa3c | 2020-12-13 21:09:59 -0800 | [diff] [blame] | 904 | segment_manager_free_safe (sm); |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 905 | } |
| 906 | } |
| 907 | |
Florin Coras | 6d7552c | 2020-04-09 01:49:45 +0000 | [diff] [blame] | 908 | void |
Florin Coras | 0bc78d8 | 2021-01-09 14:34:01 -0800 | [diff] [blame] | 909 | segment_manager_detach_fifo (segment_manager_t *sm, svm_fifo_t **f) |
Florin Coras | 6d7552c | 2020-04-09 01:49:45 +0000 | [diff] [blame] | 910 | { |
| 911 | fifo_segment_t *fs; |
| 912 | |
Florin Coras | 0bc78d8 | 2021-01-09 14:34:01 -0800 | [diff] [blame] | 913 | fs = segment_manager_get_segment_w_lock (sm, (*f)->segment_index); |
Florin Coras | 6d7552c | 2020-04-09 01:49:45 +0000 | [diff] [blame] | 914 | fifo_segment_detach_fifo (fs, f); |
| 915 | segment_manager_segment_reader_unlock (sm); |
| 916 | } |
| 917 | |
| 918 | void |
Florin Coras | 0bc78d8 | 2021-01-09 14:34:01 -0800 | [diff] [blame] | 919 | segment_manager_attach_fifo (segment_manager_t *sm, svm_fifo_t **f, |
| 920 | session_t *s) |
Florin Coras | 6d7552c | 2020-04-09 01:49:45 +0000 | [diff] [blame] | 921 | { |
| 922 | fifo_segment_t *fs; |
| 923 | |
Florin Coras | 0bc78d8 | 2021-01-09 14:34:01 -0800 | [diff] [blame] | 924 | fs = segment_manager_get_segment_w_lock (sm, (*f)->segment_index); |
Florin Coras | 6d7552c | 2020-04-09 01:49:45 +0000 | [diff] [blame] | 925 | fifo_segment_attach_fifo (fs, f, s->thread_index); |
| 926 | segment_manager_segment_reader_unlock (sm); |
| 927 | |
Florin Coras | 0bc78d8 | 2021-01-09 14:34:01 -0800 | [diff] [blame] | 928 | (*f)->shr->master_session_index = s->session_index; |
| 929 | (*f)->master_thread_index = s->thread_index; |
Florin Coras | 6d7552c | 2020-04-09 01:49:45 +0000 | [diff] [blame] | 930 | } |
| 931 | |
Florin Coras | 3c2fed5 | 2018-07-04 04:15:05 -0700 | [diff] [blame] | 932 | u32 |
| 933 | segment_manager_evt_q_expected_size (u32 q_len) |
| 934 | { |
| 935 | u32 fifo_evt_size, notif_q_size, q_hdrs; |
| 936 | u32 msg_q_sz, fifo_evt_ring_sz, session_ntf_ring_sz; |
| 937 | |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 938 | fifo_evt_size = 1 << max_log2 (sizeof (session_event_t)); |
Florin Coras | 3c2fed5 | 2018-07-04 04:15:05 -0700 | [diff] [blame] | 939 | notif_q_size = clib_max (16, q_len >> 4); |
| 940 | |
| 941 | msg_q_sz = q_len * sizeof (svm_msg_q_msg_t); |
| 942 | fifo_evt_ring_sz = q_len * fifo_evt_size; |
| 943 | session_ntf_ring_sz = notif_q_size * 256; |
| 944 | q_hdrs = sizeof (svm_queue_t) + sizeof (svm_msg_q_t); |
| 945 | |
| 946 | return (msg_q_sz + fifo_evt_ring_sz + session_ntf_ring_sz + q_hdrs); |
| 947 | } |
| 948 | |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 949 | /** |
| 950 | * Allocates shm queue in the first segment |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 951 | * |
| 952 | * Must be called with lock held |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 953 | */ |
Florin Coras | 3c2fed5 | 2018-07-04 04:15:05 -0700 | [diff] [blame] | 954 | svm_msg_q_t * |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 955 | segment_manager_alloc_queue (fifo_segment_t * segment, |
| 956 | segment_manager_props_t * props) |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 957 | { |
Florin Coras | 3c2fed5 | 2018-07-04 04:15:05 -0700 | [diff] [blame] | 958 | u32 fifo_evt_size, session_evt_size = 256, notif_q_size; |
| 959 | svm_msg_q_cfg_t _cfg, *cfg = &_cfg; |
| 960 | svm_msg_q_t *q; |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 961 | |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 962 | fifo_evt_size = sizeof (session_event_t); |
Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 963 | notif_q_size = clib_max (16, props->evt_q_size >> 4); |
Florin Coras | 3c2fed5 | 2018-07-04 04:15:05 -0700 | [diff] [blame] | 964 | /* *INDENT-OFF* */ |
| 965 | svm_msg_q_ring_cfg_t rc[SESSION_MQ_N_RINGS] = { |
Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 966 | {props->evt_q_size, fifo_evt_size, 0}, |
Florin Coras | 3c2fed5 | 2018-07-04 04:15:05 -0700 | [diff] [blame] | 967 | {notif_q_size, session_evt_size, 0} |
| 968 | }; |
| 969 | /* *INDENT-ON* */ |
| 970 | cfg->consumer_pid = 0; |
| 971 | cfg->n_rings = 2; |
Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 972 | cfg->q_nitems = props->evt_q_size; |
Florin Coras | 3c2fed5 | 2018-07-04 04:15:05 -0700 | [diff] [blame] | 973 | cfg->ring_cfgs = rc; |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 974 | |
Florin Coras | b462418 | 2020-12-11 13:58:12 -0800 | [diff] [blame] | 975 | q = fifo_segment_msg_q_alloc (segment, 0, cfg); |
Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 976 | |
| 977 | if (props->use_mq_eventfd) |
| 978 | { |
Florin Coras | 86f1232 | 2021-01-22 15:05:14 -0800 | [diff] [blame] | 979 | if (svm_msg_q_alloc_eventfd (q)) |
Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 980 | clib_warning ("failed to alloc eventfd"); |
| 981 | } |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 982 | return q; |
| 983 | } |
| 984 | |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 985 | svm_msg_q_t * |
| 986 | segment_manager_event_queue (segment_manager_t * sm) |
| 987 | { |
| 988 | return sm->event_queue; |
| 989 | } |
| 990 | |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 991 | /** |
| 992 | * Frees shm queue allocated in the first segment |
| 993 | */ |
| 994 | void |
Florin Coras | e86a8ed | 2018-01-05 03:20:25 -0800 | [diff] [blame] | 995 | segment_manager_dealloc_queue (segment_manager_t * sm, svm_queue_t * q) |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 996 | { |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 997 | fifo_segment_t *segment; |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 998 | ssvm_shared_header_t *sh; |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 999 | void *oldheap; |
| 1000 | |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 1001 | ASSERT (!pool_is_free_index (sm->segments, 0)); |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 1002 | |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 1003 | segment = segment_manager_get_segment_w_lock (sm, 0); |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 1004 | sh = segment->ssvm.sh; |
| 1005 | |
| 1006 | oldheap = ssvm_push_heap (sh); |
Florin Coras | e86a8ed | 2018-01-05 03:20:25 -0800 | [diff] [blame] | 1007 | svm_queue_free (q); |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 1008 | ssvm_pop_heap (oldheap); |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 1009 | segment_manager_segment_reader_unlock (sm); |
| 1010 | } |
| 1011 | |
| 1012 | /* |
| 1013 | * Init segment vm address allocator |
| 1014 | */ |
| 1015 | void |
Florin Coras | 9a45bd8 | 2020-12-28 16:28:07 -0800 | [diff] [blame] | 1016 | segment_manager_main_init (void) |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 1017 | { |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1018 | segment_manager_main_t *sm = &sm_main; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1019 | |
| 1020 | sm->default_fifo_size = 1 << 12; |
| 1021 | sm->default_segment_size = 1 << 20; |
| 1022 | sm->default_app_mq_size = 128; |
Ryujiro Shibuya | d8f48e2 | 2020-01-22 12:11:42 +0000 | [diff] [blame] | 1023 | sm->default_max_fifo_size = 4 << 20; |
Ryujiro Shibuya | 234fe89 | 2019-12-25 07:40:54 +0000 | [diff] [blame] | 1024 | sm->default_high_watermark = 80; |
| 1025 | sm->default_low_watermark = 50; |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 1026 | } |
| 1027 | |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 1028 | static u8 * |
| 1029 | format_segment_manager (u8 *s, va_list *args) |
| 1030 | { |
| 1031 | segment_manager_t *sm = va_arg (*args, segment_manager_t *); |
| 1032 | int verbose = va_arg (*args, int); |
| 1033 | app_worker_t *app_wrk; |
| 1034 | uword max_fifo_size; |
| 1035 | fifo_segment_t *seg; |
| 1036 | application_t *app; |
| 1037 | u8 custom_logic; |
| 1038 | |
| 1039 | app_wrk = app_worker_get_if_valid (sm->app_wrk_index); |
| 1040 | app = app_wrk ? application_get (app_wrk->app_index) : 0; |
| 1041 | custom_logic = (app && (app->cb_fns.fifo_tuning_callback)) ? 1 : 0; |
| 1042 | max_fifo_size = sm->max_fifo_size; |
| 1043 | |
| 1044 | s = format (s, |
| 1045 | "[%u] %v app-wrk: %u segs: %u max-fifo-sz: %U " |
| 1046 | "wmarks: %u %u %s flags: 0x%x", |
Florin Coras | 36eaaf3 | 2022-01-10 13:44:45 -0800 | [diff] [blame] | 1047 | segment_manager_index (sm), app ? app->name : 0, |
| 1048 | sm->app_wrk_index, pool_elts (sm->segments), format_memory_size, |
| 1049 | max_fifo_size, sm->high_watermark, sm->low_watermark, |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 1050 | custom_logic ? "custom-tuning" : "no-tuning", sm->flags); |
| 1051 | |
| 1052 | if (!verbose || !pool_elts (sm->segments)) |
| 1053 | return s; |
| 1054 | |
| 1055 | s = format (s, "\n\n"); |
| 1056 | |
| 1057 | segment_manager_foreach_segment_w_lock ( |
| 1058 | seg, sm, ({ s = format (s, " *%U", format_fifo_segment, seg, verbose); })); |
| 1059 | |
| 1060 | return s; |
| 1061 | } |
| 1062 | |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 1063 | static clib_error_t * |
| 1064 | segment_manager_show_fn (vlib_main_t * vm, unformat_input_t * input, |
| 1065 | vlib_cli_command_t * cmd) |
| 1066 | { |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 1067 | unformat_input_t _line_input, *line_input = &_line_input; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1068 | segment_manager_main_t *smm = &sm_main; |
Florin Coras | b384b54 | 2018-01-15 01:08:33 -0800 | [diff] [blame] | 1069 | u8 show_segments = 0, verbose = 0; |
Florin Coras | 5368bb0 | 2019-06-09 09:24:33 -0700 | [diff] [blame] | 1070 | segment_manager_t *sm; |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 1071 | u32 sm_index = ~0; |
Dave Barach | 91f3e74 | 2017-09-01 19:12:11 -0400 | [diff] [blame] | 1072 | |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 1073 | if (!unformat_user (input, unformat_line_input, line_input)) |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 1074 | { |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 1075 | vlib_cli_output (vm, "%d segment managers allocated", |
| 1076 | pool_elts (smm->segment_managers)); |
| 1077 | return 0; |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 1078 | } |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 1079 | |
| 1080 | while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 1081 | { |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 1082 | if (unformat (line_input, "segments")) |
| 1083 | show_segments = 1; |
| 1084 | else if (unformat (line_input, "verbose")) |
| 1085 | verbose = 1; |
| 1086 | else if (unformat (line_input, "index %u", &sm_index)) |
| 1087 | ; |
| 1088 | else |
| 1089 | { |
| 1090 | vlib_cli_output (vm, "unknown input [%U]", format_unformat_error, |
| 1091 | line_input); |
| 1092 | goto done; |
| 1093 | } |
| 1094 | } |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 1095 | |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 1096 | if (!pool_elts (smm->segment_managers)) |
| 1097 | goto done; |
| 1098 | |
| 1099 | if (sm_index != ~0) |
| 1100 | { |
| 1101 | sm = segment_manager_get_if_valid (sm_index); |
| 1102 | if (!sm) |
| 1103 | { |
| 1104 | vlib_cli_output (vm, "segment manager %u not allocated", sm_index); |
| 1105 | goto done; |
| 1106 | } |
| 1107 | vlib_cli_output (vm, "%U", format_segment_manager, sm, 1 /* verbose */); |
| 1108 | goto done; |
| 1109 | } |
| 1110 | |
| 1111 | if (verbose || show_segments) |
| 1112 | { |
Damjan Marion | b2c31b6 | 2020-12-13 21:47:40 +0100 | [diff] [blame] | 1113 | pool_foreach (sm, smm->segment_managers) { |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 1114 | vlib_cli_output (vm, "%U", format_segment_manager, sm, |
| 1115 | show_segments); |
Damjan Marion | b2c31b6 | 2020-12-13 21:47:40 +0100 | [diff] [blame] | 1116 | } |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 1117 | |
Florin Coras | 2a7c0b6 | 2020-09-28 23:40:28 -0700 | [diff] [blame] | 1118 | vlib_cli_output (vm, "\n"); |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 1119 | } |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 1120 | |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 1121 | done: |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 1122 | |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 1123 | unformat_free (line_input); |
| 1124 | |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 1125 | return 0; |
| 1126 | } |
| 1127 | |
Florin Coras | ad0c77f | 2017-11-09 18:00:15 -0800 | [diff] [blame] | 1128 | /* *INDENT-OFF* */ |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 1129 | VLIB_CLI_COMMAND (segment_manager_show_command, static) = { |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 1130 | .path = "show segment-manager", |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 1131 | .short_help = "show segment-manager [segments][verbose][index <nn>]", |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 1132 | .function = segment_manager_show_fn, |
| 1133 | }; |
| 1134 | /* *INDENT-ON* */ |
| 1135 | |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1136 | void |
| 1137 | segment_manager_format_sessions (segment_manager_t * sm, int verbose) |
| 1138 | { |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1139 | vlib_main_t *vm = vlib_get_main (); |
| 1140 | app_worker_t *app_wrk; |
Florin Coras | 62ddc03 | 2019-12-08 18:30:42 -0800 | [diff] [blame] | 1141 | fifo_segment_t *fs; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1142 | const u8 *app_name; |
Florin Coras | 62ddc03 | 2019-12-08 18:30:42 -0800 | [diff] [blame] | 1143 | int slice_index; |
| 1144 | u8 *s = 0, *str; |
| 1145 | svm_fifo_t *f; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1146 | |
| 1147 | if (!sm) |
| 1148 | { |
| 1149 | if (verbose) |
Xiaoming Jiang | 806709f | 2021-06-23 09:07:57 +0000 | [diff] [blame] | 1150 | vlib_cli_output (vm, "%-" SESSION_CLI_ID_LEN "s%-20s%-15s%-10s", |
| 1151 | "Connection", "App", "API Client", "SegManager"); |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1152 | else |
Xiaoming Jiang | 806709f | 2021-06-23 09:07:57 +0000 | [diff] [blame] | 1153 | vlib_cli_output (vm, "%-" SESSION_CLI_ID_LEN "s%-20s", "Connection", |
| 1154 | "App"); |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1155 | return; |
| 1156 | } |
| 1157 | |
| 1158 | app_wrk = app_worker_get (sm->app_wrk_index); |
| 1159 | app_name = application_name_from_index (app_wrk->app_index); |
| 1160 | |
| 1161 | clib_rwlock_reader_lock (&sm->segments_rwlock); |
| 1162 | |
| 1163 | /* *INDENT-OFF* */ |
Damjan Marion | b2c31b6 | 2020-12-13 21:47:40 +0100 | [diff] [blame] | 1164 | pool_foreach (fs, sm->segments) { |
Florin Coras | 62ddc03 | 2019-12-08 18:30:42 -0800 | [diff] [blame] | 1165 | for (slice_index = 0; slice_index < fs->n_slices; slice_index++) |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1166 | { |
Florin Coras | 62ddc03 | 2019-12-08 18:30:42 -0800 | [diff] [blame] | 1167 | f = fifo_segment_get_slice_fifo_list (fs, slice_index); |
| 1168 | while (f) |
| 1169 | { |
| 1170 | u32 session_index, thread_index; |
| 1171 | session_t *session; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1172 | |
Florin Coras | c547e91 | 2020-12-08 17:50:45 -0800 | [diff] [blame] | 1173 | session_index = f->shr->master_session_index; |
| 1174 | thread_index = f->master_thread_index; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1175 | |
Florin Coras | c547e91 | 2020-12-08 17:50:45 -0800 | [diff] [blame] | 1176 | session = session_get (session_index, thread_index); |
| 1177 | str = format (0, "%U", format_session, session, verbose); |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1178 | |
Florin Coras | c547e91 | 2020-12-08 17:50:45 -0800 | [diff] [blame] | 1179 | if (verbose) |
Xiaoming Jiang | 806709f | 2021-06-23 09:07:57 +0000 | [diff] [blame] | 1180 | s = format (s, "%-" SESSION_CLI_ID_LEN "v%-20v%-15u%-10u", str, |
| 1181 | app_name, app_wrk->api_client_index, |
Florin Coras | c547e91 | 2020-12-08 17:50:45 -0800 | [diff] [blame] | 1182 | app_wrk->connects_seg_manager); |
| 1183 | else |
Xiaoming Jiang | 806709f | 2021-06-23 09:07:57 +0000 | [diff] [blame] | 1184 | s = format (s, "%-" SESSION_CLI_ID_LEN "v%-20v", str, app_name); |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1185 | |
Florin Coras | c547e91 | 2020-12-08 17:50:45 -0800 | [diff] [blame] | 1186 | vlib_cli_output (vm, "%v", s); |
| 1187 | vec_reset_length (s); |
| 1188 | vec_free (str); |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1189 | |
Florin Coras | c547e91 | 2020-12-08 17:50:45 -0800 | [diff] [blame] | 1190 | f = f->next; |
| 1191 | } |
| 1192 | vec_free (s); |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1193 | } |
Damjan Marion | b2c31b6 | 2020-12-13 21:47:40 +0100 | [diff] [blame] | 1194 | } |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1195 | /* *INDENT-ON* */ |
| 1196 | |
| 1197 | clib_rwlock_reader_unlock (&sm->segments_rwlock); |
| 1198 | } |
| 1199 | |
Ryujiro Shibuya | 234fe89 | 2019-12-25 07:40:54 +0000 | [diff] [blame] | 1200 | void |
| 1201 | segment_manager_set_watermarks (segment_manager_t * sm, |
| 1202 | u8 high_watermark, u8 low_watermark) |
| 1203 | { |
Florin Coras | 8c79a4e | 2020-02-25 22:28:27 +0000 | [diff] [blame] | 1204 | ASSERT (high_watermark <= 100 && low_watermark <= 100 && |
Ryujiro Shibuya | 234fe89 | 2019-12-25 07:40:54 +0000 | [diff] [blame] | 1205 | low_watermark <= high_watermark); |
| 1206 | |
| 1207 | sm->high_watermark = high_watermark; |
| 1208 | sm->low_watermark = low_watermark; |
| 1209 | } |
| 1210 | |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 1211 | /* |
| 1212 | * fd.io coding-style-patch-verification: ON |
| 1213 | * |
| 1214 | * Local Variables: |
| 1215 | * eval: (c-set-style "gnu") |
| 1216 | * End: |
| 1217 | */ |