Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 1 | /* |
Florin Coras | 288eaab | 2019-02-03 15:26:14 -0800 | [diff] [blame] | 2 | * Copyright (c) 2017-2019 Cisco and/or its affiliates. |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 3 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | * you may not use this file except in compliance with the License. |
| 5 | * You may obtain a copy of the License at: |
| 6 | * |
| 7 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | * |
| 9 | * Unless required by applicable law or agreed to in writing, software |
| 10 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | * See the License for the specific language governing permissions and |
| 13 | * limitations under the License. |
| 14 | */ |
| 15 | |
| 16 | #include <vnet/session/segment_manager.h> |
| 17 | #include <vnet/session/session.h> |
| 18 | #include <vnet/session/application.h> |
| 19 | |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 20 | typedef struct segment_manager_main_ |
| 21 | { |
| 22 | segment_manager_t *segment_managers; /**< Pool of segment managers */ |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 23 | u32 seg_name_counter; /**< Counter for segment names */ |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 24 | |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 25 | /* |
| 26 | * Configuration |
| 27 | */ |
Ryujiro Shibuya | d8f48e2 | 2020-01-22 12:11:42 +0000 | [diff] [blame] | 28 | u32 default_fifo_size; /**< default rx/tx fifo size */ |
| 29 | u32 default_segment_size; /**< default fifo segment size */ |
| 30 | u32 default_app_mq_size; /**< default app msg q size */ |
| 31 | u32 default_max_fifo_size; /**< default max fifo size */ |
| 32 | u8 default_high_watermark; /**< default high watermark % */ |
| 33 | u8 default_low_watermark; /**< default low watermark % */ |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 34 | } segment_manager_main_t; |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 35 | |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 36 | static segment_manager_main_t sm_main; |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 37 | |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 38 | #define segment_manager_foreach_segment_w_lock(VAR, SM, BODY) \ |
| 39 | do { \ |
| 40 | clib_rwlock_reader_lock (&(SM)->segments_rwlock); \ |
Damjan Marion | b2c31b6 | 2020-12-13 21:47:40 +0100 | [diff] [blame] | 41 | pool_foreach((VAR), ((SM)->segments)) (BODY); \ |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 42 | clib_rwlock_reader_unlock (&(SM)->segments_rwlock); \ |
| 43 | } while (0) |
| 44 | |
| 45 | static segment_manager_props_t * |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 46 | segment_manager_properties_get (segment_manager_t * sm) |
Florin Coras | ad0c77f | 2017-11-09 18:00:15 -0800 | [diff] [blame] | 47 | { |
Florin Coras | ab2f6db | 2018-08-31 14:31:41 -0700 | [diff] [blame] | 48 | app_worker_t *app_wrk = app_worker_get (sm->app_wrk_index); |
| 49 | return application_get_segment_manager_properties (app_wrk->app_index); |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 50 | } |
| 51 | |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 52 | segment_manager_props_t * |
| 53 | segment_manager_props_init (segment_manager_props_t * props) |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 54 | { |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 55 | props->add_segment_size = sm_main.default_segment_size; |
| 56 | props->rx_fifo_size = sm_main.default_fifo_size; |
| 57 | props->tx_fifo_size = sm_main.default_fifo_size; |
| 58 | props->evt_q_size = sm_main.default_app_mq_size; |
Ryujiro Shibuya | d8f48e2 | 2020-01-22 12:11:42 +0000 | [diff] [blame] | 59 | props->max_fifo_size = sm_main.default_max_fifo_size; |
Ryujiro Shibuya | 234fe89 | 2019-12-25 07:40:54 +0000 | [diff] [blame] | 60 | props->high_watermark = sm_main.default_high_watermark; |
| 61 | props->low_watermark = sm_main.default_low_watermark; |
Florin Coras | 62ddc03 | 2019-12-08 18:30:42 -0800 | [diff] [blame] | 62 | props->n_slices = vlib_num_workers () + 1; |
Florin Coras | ad0c77f | 2017-11-09 18:00:15 -0800 | [diff] [blame] | 63 | return props; |
| 64 | } |
| 65 | |
Florin Coras | bf39597 | 2020-04-30 15:05:24 +0000 | [diff] [blame] | 66 | u8 |
Florin Coras | 9d06304 | 2017-09-14 03:08:00 -0400 | [diff] [blame] | 67 | segment_manager_app_detached (segment_manager_t * sm) |
| 68 | { |
Florin Coras | c8e812f | 2020-05-14 05:32:18 +0000 | [diff] [blame] | 69 | return (sm->flags & SEG_MANAGER_F_DETACHED); |
Dave Wallace | 7b749fe | 2017-07-05 14:30:46 -0400 | [diff] [blame] | 70 | } |
| 71 | |
Florin Coras | 4e4531e | 2017-11-06 23:27:56 -0800 | [diff] [blame] | 72 | void |
| 73 | segment_manager_app_detach (segment_manager_t * sm) |
| 74 | { |
Florin Coras | c8e812f | 2020-05-14 05:32:18 +0000 | [diff] [blame] | 75 | sm->flags |= SEG_MANAGER_F_DETACHED; |
Florin Coras | 4e4531e | 2017-11-06 23:27:56 -0800 | [diff] [blame] | 76 | } |
| 77 | |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 78 | always_inline u32 |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 79 | segment_manager_segment_index (segment_manager_t * sm, fifo_segment_t * seg) |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 80 | { |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 81 | return (seg - sm->segments); |
| 82 | } |
| 83 | |
| 84 | /** |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 85 | * Adds segment to segment manager's pool |
| 86 | * |
| 87 | * If needed a writer's lock is acquired before allocating a new segment |
| 88 | * to avoid affecting any of the segments pool readers. |
| 89 | */ |
Florin Coras | 6973c73 | 2021-06-17 15:53:38 -0700 | [diff] [blame] | 90 | static inline int |
| 91 | segment_manager_add_segment_inline (segment_manager_t *sm, uword segment_size, |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 92 | u8 notify_app, u8 flags, u8 need_lock) |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 93 | { |
| 94 | segment_manager_main_t *smm = &sm_main; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 95 | segment_manager_props_t *props; |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 96 | app_worker_t *app_wrk; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 97 | fifo_segment_t *fs; |
Florin Coras | 9a45bd8 | 2020-12-28 16:28:07 -0800 | [diff] [blame] | 98 | u32 fs_index = ~0; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 99 | u8 *seg_name; |
| 100 | int rv; |
| 101 | |
| 102 | props = segment_manager_properties_get (sm); |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 103 | app_wrk = app_worker_get (sm->app_wrk_index); |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 104 | |
| 105 | /* Not configured for addition of new segments and not first */ |
| 106 | if (!props->add_segment && !segment_size) |
| 107 | { |
Filip Tehlar | ac3c8dc | 2023-03-14 08:50:28 +0100 | [diff] [blame] | 108 | SESSION_DBG ("cannot allocate new segment"); |
| 109 | return SESSION_E_INVALID; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 110 | } |
| 111 | |
| 112 | /* |
Florin Coras | f9d4ab4 | 2019-05-11 16:55:53 -0700 | [diff] [blame] | 113 | * Allocate fifo segment and grab lock if needed |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 114 | */ |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 115 | if (need_lock) |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 116 | clib_rwlock_writer_lock (&sm->segments_rwlock); |
| 117 | |
| 118 | pool_get_zero (sm->segments, fs); |
| 119 | |
| 120 | /* |
Florin Coras | f9d4ab4 | 2019-05-11 16:55:53 -0700 | [diff] [blame] | 121 | * Allocate ssvm segment |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 122 | */ |
| 123 | segment_size = segment_size ? segment_size : props->add_segment_size; |
Filip Tehlar | 2711ca7 | 2021-11-24 10:30:59 +0000 | [diff] [blame] | 124 | /* add overhead to ensure the result segment size is at least |
| 125 | * of that requested */ |
| 126 | segment_size += |
| 127 | sizeof (fifo_segment_header_t) + |
| 128 | vlib_thread_main.n_vlib_mains * sizeof (fifo_segment_slice_t) + |
| 129 | FIFO_SEGMENT_ALLOC_OVERHEAD; |
Junfeng Wang | c795b88 | 2022-08-12 16:24:46 +0800 | [diff] [blame] | 130 | |
| 131 | if (props->huge_page) |
| 132 | { |
| 133 | uword hugepage_size = clib_mem_get_default_hugepage_size (); |
| 134 | segment_size = round_pow2 (segment_size, hugepage_size); |
| 135 | fs->ssvm.huge_page = 1; |
| 136 | } |
| 137 | else |
| 138 | segment_size = round_pow2 (segment_size, clib_mem_get_page_size ()); |
Florin Coras | f9d4ab4 | 2019-05-11 16:55:53 -0700 | [diff] [blame] | 139 | |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 140 | seg_name = format (0, "seg-%u-%u-%u%c", app_wrk->app_index, |
| 141 | app_wrk->wrk_index, smm->seg_name_counter++, 0); |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 142 | |
| 143 | fs->ssvm.ssvm_size = segment_size; |
| 144 | fs->ssvm.name = seg_name; |
Florin Coras | 9a45bd8 | 2020-12-28 16:28:07 -0800 | [diff] [blame] | 145 | fs->ssvm.requested_va = 0; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 146 | |
Florin Coras | 5220a26 | 2020-09-29 18:11:24 -0700 | [diff] [blame] | 147 | if ((rv = ssvm_server_init (&fs->ssvm, props->segment_type))) |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 148 | { |
| 149 | clib_warning ("svm_master_init ('%v', %u) failed", seg_name, |
| 150 | segment_size); |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 151 | pool_put (sm->segments, fs); |
| 152 | goto done; |
| 153 | } |
| 154 | |
Florin Coras | f9d4ab4 | 2019-05-11 16:55:53 -0700 | [diff] [blame] | 155 | /* |
| 156 | * Initialize fifo segment |
| 157 | */ |
Florin Coras | 62ddc03 | 2019-12-08 18:30:42 -0800 | [diff] [blame] | 158 | fs->n_slices = props->n_slices; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 159 | fifo_segment_init (fs); |
| 160 | |
| 161 | /* |
| 162 | * Save segment index before dropping lock, if any held |
| 163 | */ |
| 164 | fs_index = fs - sm->segments; |
Florin Coras | cdfe8ab | 2021-12-22 12:54:17 -0800 | [diff] [blame] | 165 | fs->fs_index = fs_index; |
| 166 | fs->sm_index = segment_manager_index (sm); |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 167 | |
Ryujiro Shibuya | 234fe89 | 2019-12-25 07:40:54 +0000 | [diff] [blame] | 168 | /* |
| 169 | * Set watermarks in segment |
| 170 | */ |
Florin Coras | afbb33a | 2021-08-10 16:56:34 -0700 | [diff] [blame] | 171 | fs->high_watermark = sm->high_watermark; |
| 172 | fs->low_watermark = sm->low_watermark; |
| 173 | fs->flags = flags; |
| 174 | fs->flags &= ~FIFO_SEGMENT_F_MEM_LIMIT; |
Florin Coras | 2de9c0f | 2020-02-02 19:30:39 +0000 | [diff] [blame] | 175 | fs->h->pct_first_alloc = props->pct_first_alloc; |
Ryujiro Shibuya | 234fe89 | 2019-12-25 07:40:54 +0000 | [diff] [blame] | 176 | |
liuyacan | 9f29903 | 2021-04-25 20:11:30 +0800 | [diff] [blame] | 177 | if (notify_app) |
| 178 | { |
| 179 | app_worker_t *app_wrk; |
| 180 | u64 fs_handle; |
| 181 | fs_handle = segment_manager_segment_handle (sm, fs); |
| 182 | app_wrk = app_worker_get (sm->app_wrk_index); |
| 183 | rv = app_worker_add_segment_notify (app_wrk, fs_handle); |
| 184 | if (rv) |
wanghanlin | 19f6e26 | 2021-11-29 15:05:23 +0800 | [diff] [blame] | 185 | { |
| 186 | fs_index = rv; |
| 187 | goto done; |
| 188 | } |
liuyacan | 9f29903 | 2021-04-25 20:11:30 +0800 | [diff] [blame] | 189 | } |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 190 | done: |
| 191 | |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 192 | if (need_lock) |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 193 | clib_rwlock_writer_unlock (&sm->segments_rwlock); |
| 194 | |
| 195 | return fs_index; |
| 196 | } |
| 197 | |
Florin Coras | 6973c73 | 2021-06-17 15:53:38 -0700 | [diff] [blame] | 198 | int |
| 199 | segment_manager_add_segment (segment_manager_t *sm, uword segment_size, |
| 200 | u8 notify_app) |
| 201 | { |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 202 | return segment_manager_add_segment_inline (sm, segment_size, notify_app, |
| 203 | 0 /* flags */, 0 /* need_lock */); |
Florin Coras | 6973c73 | 2021-06-17 15:53:38 -0700 | [diff] [blame] | 204 | } |
| 205 | |
| 206 | int |
| 207 | segment_manager_add_segment2 (segment_manager_t *sm, uword segment_size, |
| 208 | u8 flags) |
| 209 | { |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 210 | return segment_manager_add_segment_inline (sm, segment_size, 0, flags, |
| 211 | vlib_num_workers ()); |
Florin Coras | 6973c73 | 2021-06-17 15:53:38 -0700 | [diff] [blame] | 212 | } |
| 213 | |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 214 | /** |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 215 | * Remove segment without lock |
| 216 | */ |
Florin Coras | f8f516a | 2018-02-08 15:10:09 -0800 | [diff] [blame] | 217 | void |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 218 | segment_manager_del_segment (segment_manager_t * sm, fifo_segment_t * fs) |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 219 | { |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 220 | if (ssvm_type (&fs->ssvm) != SSVM_SEGMENT_PRIVATE) |
Florin Coras | a0dbf9e | 2019-03-01 17:12:02 -0800 | [diff] [blame] | 221 | { |
Florin Coras | ea7e708 | 2020-07-07 17:57:28 -0700 | [diff] [blame] | 222 | if (!segment_manager_app_detached (sm)) |
Florin Coras | a0dbf9e | 2019-03-01 17:12:02 -0800 | [diff] [blame] | 223 | { |
| 224 | app_worker_t *app_wrk; |
| 225 | u64 segment_handle; |
| 226 | app_wrk = app_worker_get (sm->app_wrk_index); |
| 227 | segment_handle = segment_manager_segment_handle (sm, fs); |
| 228 | app_worker_del_segment_notify (app_wrk, segment_handle); |
| 229 | } |
| 230 | } |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 231 | |
Florin Coras | c547e91 | 2020-12-08 17:50:45 -0800 | [diff] [blame] | 232 | fifo_segment_cleanup (fs); |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 233 | ssvm_delete (&fs->ssvm); |
| 234 | |
| 235 | if (CLIB_DEBUG) |
Dave Barach | b7b9299 | 2018-10-17 10:38:51 -0400 | [diff] [blame] | 236 | clib_memset (fs, 0xfb, sizeof (*fs)); |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 237 | pool_put (sm->segments, fs); |
| 238 | } |
| 239 | |
Florin Coras | 57660d9 | 2020-04-04 22:45:34 +0000 | [diff] [blame] | 240 | static fifo_segment_t * |
| 241 | segment_manager_get_segment_if_valid (segment_manager_t * sm, |
| 242 | u32 segment_index) |
| 243 | { |
| 244 | if (pool_is_free_index (sm->segments, segment_index)) |
| 245 | return 0; |
| 246 | return pool_elt_at_index (sm->segments, segment_index); |
| 247 | } |
| 248 | |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 249 | /** |
| 250 | * Removes segment after acquiring writer lock |
| 251 | */ |
Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 252 | static inline void |
Florin Coras | 02cc719 | 2021-11-27 14:44:59 -0800 | [diff] [blame] | 253 | sm_lock_and_del_segment_inline (segment_manager_t *sm, u32 fs_index, |
| 254 | u8 check_if_empty) |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 255 | { |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 256 | fifo_segment_t *fs; |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 257 | u8 is_prealloc; |
| 258 | |
| 259 | clib_rwlock_writer_lock (&sm->segments_rwlock); |
Florin Coras | 57660d9 | 2020-04-04 22:45:34 +0000 | [diff] [blame] | 260 | |
| 261 | fs = segment_manager_get_segment_if_valid (sm, fs_index); |
| 262 | if (!fs) |
| 263 | goto done; |
| 264 | |
Florin Coras | 02cc719 | 2021-11-27 14:44:59 -0800 | [diff] [blame] | 265 | if (check_if_empty && fifo_segment_has_fifos (fs)) |
| 266 | goto done; |
| 267 | |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 268 | is_prealloc = fifo_segment_flags (fs) & FIFO_SEGMENT_F_IS_PREALLOCATED; |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 269 | if (is_prealloc && !segment_manager_app_detached (sm)) |
Florin Coras | 57660d9 | 2020-04-04 22:45:34 +0000 | [diff] [blame] | 270 | goto done; |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 271 | |
| 272 | segment_manager_del_segment (sm, fs); |
Florin Coras | 57660d9 | 2020-04-04 22:45:34 +0000 | [diff] [blame] | 273 | |
| 274 | done: |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 275 | clib_rwlock_writer_unlock (&sm->segments_rwlock); |
| 276 | } |
| 277 | |
Florin Coras | 2d0e3de | 2020-10-23 16:31:40 -0700 | [diff] [blame] | 278 | void |
| 279 | segment_manager_lock_and_del_segment (segment_manager_t * sm, u32 fs_index) |
| 280 | { |
Florin Coras | 02cc719 | 2021-11-27 14:44:59 -0800 | [diff] [blame] | 281 | sm_lock_and_del_segment_inline (sm, fs_index, 0 /* check_if_empty */); |
Florin Coras | 2d0e3de | 2020-10-23 16:31:40 -0700 | [diff] [blame] | 282 | } |
| 283 | |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 284 | /** |
| 285 | * Reads a segment from the segment manager's pool without lock |
| 286 | */ |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 287 | fifo_segment_t * |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 288 | segment_manager_get_segment (segment_manager_t * sm, u32 segment_index) |
| 289 | { |
| 290 | return pool_elt_at_index (sm->segments, segment_index); |
| 291 | } |
| 292 | |
Florin Coras | fa76a76 | 2018-11-29 12:40:10 -0800 | [diff] [blame] | 293 | u64 |
| 294 | segment_manager_segment_handle (segment_manager_t * sm, |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 295 | fifo_segment_t * segment) |
Florin Coras | fa76a76 | 2018-11-29 12:40:10 -0800 | [diff] [blame] | 296 | { |
| 297 | u32 segment_index = segment_manager_segment_index (sm, segment); |
| 298 | return (((u64) segment_manager_index (sm) << 32) | segment_index); |
| 299 | } |
| 300 | |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 301 | u64 |
| 302 | segment_manager_make_segment_handle (u32 segment_manager_index, |
| 303 | u32 segment_index) |
| 304 | { |
| 305 | return (((u64) segment_manager_index << 32) | segment_index); |
| 306 | } |
| 307 | |
| 308 | fifo_segment_t * |
Florin Coras | fa76a76 | 2018-11-29 12:40:10 -0800 | [diff] [blame] | 309 | segment_manager_get_segment_w_handle (u64 segment_handle) |
| 310 | { |
| 311 | u32 sm_index, segment_index; |
| 312 | segment_manager_t *sm; |
| 313 | |
| 314 | segment_manager_parse_segment_handle (segment_handle, &sm_index, |
| 315 | &segment_index); |
| 316 | sm = segment_manager_get (sm_index); |
| 317 | if (!sm || pool_is_free_index (sm->segments, segment_index)) |
| 318 | return 0; |
| 319 | return pool_elt_at_index (sm->segments, segment_index); |
| 320 | } |
| 321 | |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 322 | /** |
| 323 | * Reads a segment from the segment manager's pool and acquires reader lock |
| 324 | * |
| 325 | * Caller must drop the reader's lock by calling |
| 326 | * @ref segment_manager_segment_reader_unlock once it finishes working with |
| 327 | * the segment. |
| 328 | */ |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 329 | fifo_segment_t * |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 330 | segment_manager_get_segment_w_lock (segment_manager_t * sm, u32 segment_index) |
| 331 | { |
| 332 | clib_rwlock_reader_lock (&sm->segments_rwlock); |
| 333 | return pool_elt_at_index (sm->segments, segment_index); |
| 334 | } |
| 335 | |
| 336 | void |
Florin Coras | 75ccf7b | 2020-03-05 19:44:02 +0000 | [diff] [blame] | 337 | segment_manager_segment_reader_lock (segment_manager_t * sm) |
| 338 | { |
| 339 | clib_rwlock_reader_lock (&sm->segments_rwlock); |
| 340 | } |
| 341 | |
| 342 | void |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 343 | segment_manager_segment_reader_unlock (segment_manager_t * sm) |
| 344 | { |
| 345 | clib_rwlock_reader_unlock (&sm->segments_rwlock); |
| 346 | } |
| 347 | |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 348 | segment_manager_t * |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 349 | segment_manager_alloc (void) |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 350 | { |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 351 | segment_manager_main_t *smm = &sm_main; |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 352 | segment_manager_t *sm; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 353 | |
| 354 | pool_get_zero (smm->segment_managers, sm); |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 355 | clib_rwlock_init (&sm->segments_rwlock); |
| 356 | return sm; |
| 357 | } |
| 358 | |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 359 | int |
Ryujiro Shibuya | 234fe89 | 2019-12-25 07:40:54 +0000 | [diff] [blame] | 360 | segment_manager_init (segment_manager_t * sm) |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 361 | { |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 362 | segment_manager_props_t *props; |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 363 | |
| 364 | props = segment_manager_properties_get (sm); |
Ryujiro Shibuya | 234fe89 | 2019-12-25 07:40:54 +0000 | [diff] [blame] | 365 | |
Ryujiro Shibuya | d8f48e2 | 2020-01-22 12:11:42 +0000 | [diff] [blame] | 366 | sm->max_fifo_size = props->max_fifo_size ? |
| 367 | props->max_fifo_size : sm_main.default_max_fifo_size; |
| 368 | sm->max_fifo_size = clib_max (sm->max_fifo_size, 4096); |
| 369 | |
Ryujiro Shibuya | 234fe89 | 2019-12-25 07:40:54 +0000 | [diff] [blame] | 370 | segment_manager_set_watermarks (sm, |
| 371 | props->high_watermark, |
| 372 | props->low_watermark); |
Florin Coras | a107f40 | 2020-09-29 19:18:46 -0700 | [diff] [blame] | 373 | return 0; |
| 374 | } |
| 375 | |
| 376 | /** |
| 377 | * Initializes segment manager based on options provided. |
| 378 | * Returns error if ssvm segment(s) allocation fails. |
| 379 | */ |
| 380 | int |
| 381 | segment_manager_init_first (segment_manager_t * sm) |
| 382 | { |
| 383 | segment_manager_props_t *props; |
| 384 | uword first_seg_size; |
| 385 | fifo_segment_t *fs; |
| 386 | int fs_index, i; |
| 387 | |
| 388 | segment_manager_init (sm); |
| 389 | props = segment_manager_properties_get (sm); |
| 390 | first_seg_size = clib_max (props->segment_size, |
| 391 | sm_main.default_segment_size); |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 392 | |
Florin Coras | 9845c20 | 2020-04-28 01:54:22 +0000 | [diff] [blame] | 393 | if (props->prealloc_fifos) |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 394 | { |
Florin Coras | 9845c20 | 2020-04-28 01:54:22 +0000 | [diff] [blame] | 395 | u64 approx_total_size, max_seg_size = ((u64) 1 << 32) - (128 << 10); |
| 396 | u32 rx_rounded_data_size, tx_rounded_data_size; |
| 397 | u32 prealloc_fifo_pairs = props->prealloc_fifos; |
| 398 | u32 rx_fifo_size, tx_fifo_size, pair_size; |
| 399 | u32 approx_segment_count; |
| 400 | |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 401 | /* Figure out how many segments should be preallocated */ |
| 402 | rx_rounded_data_size = (1 << (max_log2 (props->rx_fifo_size))); |
| 403 | tx_rounded_data_size = (1 << (max_log2 (props->tx_fifo_size))); |
| 404 | |
| 405 | rx_fifo_size = sizeof (svm_fifo_t) + rx_rounded_data_size; |
| 406 | tx_fifo_size = sizeof (svm_fifo_t) + tx_rounded_data_size; |
| 407 | pair_size = rx_fifo_size + tx_fifo_size; |
| 408 | |
| 409 | approx_total_size = (u64) prealloc_fifo_pairs *pair_size; |
| 410 | if (first_seg_size > approx_total_size) |
| 411 | max_seg_size = first_seg_size; |
| 412 | approx_segment_count = (approx_total_size + (max_seg_size - 1)) |
| 413 | / max_seg_size; |
| 414 | |
| 415 | /* Allocate the segments */ |
| 416 | for (i = 0; i < approx_segment_count + 1; i++) |
| 417 | { |
liuyacan | 9f29903 | 2021-04-25 20:11:30 +0800 | [diff] [blame] | 418 | fs_index = segment_manager_add_segment (sm, max_seg_size, 0); |
Florin Coras | 9845c20 | 2020-04-28 01:54:22 +0000 | [diff] [blame] | 419 | if (fs_index < 0) |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 420 | { |
Filip Tehlar | ac3c8dc | 2023-03-14 08:50:28 +0100 | [diff] [blame] | 421 | SESSION_DBG ("Failed to preallocate segment %d", i); |
Florin Coras | 9845c20 | 2020-04-28 01:54:22 +0000 | [diff] [blame] | 422 | return fs_index; |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 423 | } |
| 424 | |
Florin Coras | 9845c20 | 2020-04-28 01:54:22 +0000 | [diff] [blame] | 425 | fs = segment_manager_get_segment (sm, fs_index); |
Florin Coras | f8f516a | 2018-02-08 15:10:09 -0800 | [diff] [blame] | 426 | if (i == 0) |
Florin Coras | 9845c20 | 2020-04-28 01:54:22 +0000 | [diff] [blame] | 427 | sm->event_queue = segment_manager_alloc_queue (fs, props); |
Florin Coras | f8f516a | 2018-02-08 15:10:09 -0800 | [diff] [blame] | 428 | |
Florin Coras | 9845c20 | 2020-04-28 01:54:22 +0000 | [diff] [blame] | 429 | fifo_segment_preallocate_fifo_pairs (fs, |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 430 | props->rx_fifo_size, |
| 431 | props->tx_fifo_size, |
| 432 | &prealloc_fifo_pairs); |
Florin Coras | 9845c20 | 2020-04-28 01:54:22 +0000 | [diff] [blame] | 433 | fifo_segment_flags (fs) = FIFO_SEGMENT_F_IS_PREALLOCATED; |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 434 | if (prealloc_fifo_pairs == 0) |
| 435 | break; |
| 436 | } |
Florin Coras | 9845c20 | 2020-04-28 01:54:22 +0000 | [diff] [blame] | 437 | return 0; |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 438 | } |
Florin Coras | 9845c20 | 2020-04-28 01:54:22 +0000 | [diff] [blame] | 439 | |
liuyacan | 9f29903 | 2021-04-25 20:11:30 +0800 | [diff] [blame] | 440 | fs_index = segment_manager_add_segment (sm, first_seg_size, 0); |
Florin Coras | 9845c20 | 2020-04-28 01:54:22 +0000 | [diff] [blame] | 441 | if (fs_index < 0) |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 442 | { |
Filip Tehlar | ac3c8dc | 2023-03-14 08:50:28 +0100 | [diff] [blame] | 443 | SESSION_DBG ("Failed to allocate segment"); |
Florin Coras | 9845c20 | 2020-04-28 01:54:22 +0000 | [diff] [blame] | 444 | return fs_index; |
| 445 | } |
| 446 | |
| 447 | fs = segment_manager_get_segment (sm, fs_index); |
| 448 | sm->event_queue = segment_manager_alloc_queue (fs, props); |
| 449 | |
| 450 | if (props->prealloc_fifo_hdrs) |
| 451 | { |
| 452 | u32 hdrs_per_slice; |
| 453 | |
| 454 | /* Do not preallocate on slice associated to main thread */ |
| 455 | i = (vlib_num_workers ()? 1 : 0); |
| 456 | hdrs_per_slice = props->prealloc_fifo_hdrs / (fs->n_slices - i); |
| 457 | |
| 458 | for (; i < fs->n_slices; i++) |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 459 | { |
Florin Coras | 9845c20 | 2020-04-28 01:54:22 +0000 | [diff] [blame] | 460 | if (fifo_segment_prealloc_fifo_hdrs (fs, i, hdrs_per_slice)) |
Filip Tehlar | ac3c8dc | 2023-03-14 08:50:28 +0100 | [diff] [blame] | 461 | return SESSION_E_SEG_CREATE; |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 462 | } |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 463 | } |
| 464 | |
| 465 | return 0; |
| 466 | } |
| 467 | |
Florin Coras | c8e812f | 2020-05-14 05:32:18 +0000 | [diff] [blame] | 468 | void |
| 469 | segment_manager_cleanup_detached_listener (segment_manager_t * sm) |
| 470 | { |
| 471 | app_worker_t *app_wrk; |
| 472 | |
| 473 | app_wrk = app_worker_get_if_valid (sm->app_wrk_index); |
| 474 | if (!app_wrk) |
| 475 | return; |
| 476 | |
| 477 | app_worker_del_detached_sm (app_wrk, segment_manager_index (sm)); |
| 478 | } |
| 479 | |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 480 | /** |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 481 | * Cleanup segment manager. |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 482 | */ |
| 483 | void |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 484 | segment_manager_free (segment_manager_t * sm) |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 485 | { |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 486 | segment_manager_main_t *smm = &sm_main; |
| 487 | fifo_segment_t *fifo_segment; |
Dave Wallace | 7b749fe | 2017-07-05 14:30:46 -0400 | [diff] [blame] | 488 | |
Florin Coras | 506aa3c | 2020-12-13 21:09:59 -0800 | [diff] [blame] | 489 | ASSERT (vlib_get_thread_index () == 0 |
| 490 | && !segment_manager_has_fifos (sm) |
Florin Coras | 9d06304 | 2017-09-14 03:08:00 -0400 | [diff] [blame] | 491 | && segment_manager_app_detached (sm)); |
| 492 | |
Florin Coras | c8e812f | 2020-05-14 05:32:18 +0000 | [diff] [blame] | 493 | if (sm->flags & SEG_MANAGER_F_DETACHED_LISTENER) |
| 494 | segment_manager_cleanup_detached_listener (sm); |
| 495 | |
Florin Coras | 9d06304 | 2017-09-14 03:08:00 -0400 | [diff] [blame] | 496 | /* If we have empty preallocated segments that haven't been removed, remove |
| 497 | * them now. Apart from that, the first segment in the first segment manager |
| 498 | * is not removed when all fifos are removed. It can only be removed when |
| 499 | * the manager is explicitly deleted/detached by the app. */ |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 500 | clib_rwlock_writer_lock (&sm->segments_rwlock); |
| 501 | |
Damjan Marion | b2c31b6 | 2020-12-13 21:47:40 +0100 | [diff] [blame] | 502 | pool_foreach (fifo_segment, sm->segments) { |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 503 | segment_manager_del_segment (sm, fifo_segment); |
Damjan Marion | b2c31b6 | 2020-12-13 21:47:40 +0100 | [diff] [blame] | 504 | } |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 505 | |
Florin Coras | f1af21c | 2021-02-26 19:19:11 -0800 | [diff] [blame] | 506 | pool_free (sm->segments); |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 507 | clib_rwlock_writer_unlock (&sm->segments_rwlock); |
| 508 | |
| 509 | clib_rwlock_free (&sm->segments_rwlock); |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 510 | if (CLIB_DEBUG) |
Dave Barach | b7b9299 | 2018-10-17 10:38:51 -0400 | [diff] [blame] | 511 | clib_memset (sm, 0xfe, sizeof (*sm)); |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 512 | pool_put (smm->segment_managers, sm); |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 513 | } |
| 514 | |
Florin Coras | 506aa3c | 2020-12-13 21:09:59 -0800 | [diff] [blame] | 515 | static void |
| 516 | sm_free_w_index_helper (void *arg) |
| 517 | { |
| 518 | u32 sm_index = *(u32 *) arg; |
| 519 | segment_manager_t *sm; |
| 520 | |
| 521 | ASSERT (vlib_get_thread_index () == 0); |
| 522 | |
| 523 | if ((sm = segment_manager_get_if_valid (sm_index))) |
| 524 | segment_manager_free (sm); |
| 525 | } |
| 526 | |
Florin Coras | da78c5a | 2021-06-09 14:55:24 -0700 | [diff] [blame] | 527 | void |
| 528 | segment_manager_free_safe (segment_manager_t *sm) |
Florin Coras | 506aa3c | 2020-12-13 21:09:59 -0800 | [diff] [blame] | 529 | { |
| 530 | if (!vlib_thread_is_main_w_barrier ()) |
| 531 | { |
| 532 | u32 sm_index = segment_manager_index (sm); |
| 533 | vlib_rpc_call_main_thread (sm_free_w_index_helper, (u8 *) & sm_index, |
| 534 | sizeof (sm_index)); |
| 535 | } |
| 536 | else |
| 537 | { |
| 538 | segment_manager_free (sm); |
| 539 | } |
| 540 | } |
| 541 | |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 542 | void |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 543 | segment_manager_init_free (segment_manager_t * sm) |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 544 | { |
Florin Coras | 506aa3c | 2020-12-13 21:09:59 -0800 | [diff] [blame] | 545 | ASSERT (vlib_get_thread_index () == 0); |
| 546 | |
Florin Coras | 4e4531e | 2017-11-06 23:27:56 -0800 | [diff] [blame] | 547 | segment_manager_app_detach (sm); |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 548 | if (segment_manager_has_fifos (sm)) |
| 549 | segment_manager_del_sessions (sm); |
| 550 | else |
| 551 | { |
Florin Coras | 9d06304 | 2017-09-14 03:08:00 -0400 | [diff] [blame] | 552 | ASSERT (!sm->first_is_protected || segment_manager_app_detached (sm)); |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 553 | segment_manager_free (sm); |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 554 | } |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 555 | } |
| 556 | |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 557 | segment_manager_t * |
| 558 | segment_manager_get (u32 index) |
| 559 | { |
| 560 | return pool_elt_at_index (sm_main.segment_managers, index); |
| 561 | } |
| 562 | |
| 563 | segment_manager_t * |
| 564 | segment_manager_get_if_valid (u32 index) |
| 565 | { |
| 566 | if (pool_is_free_index (sm_main.segment_managers, index)) |
| 567 | return 0; |
| 568 | return pool_elt_at_index (sm_main.segment_managers, index); |
| 569 | } |
| 570 | |
| 571 | u32 |
| 572 | segment_manager_index (segment_manager_t * sm) |
| 573 | { |
| 574 | return sm - sm_main.segment_managers; |
| 575 | } |
| 576 | |
| 577 | u8 |
| 578 | segment_manager_has_fifos (segment_manager_t * sm) |
| 579 | { |
| 580 | fifo_segment_t *seg; |
| 581 | u8 first = 1; |
| 582 | |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 583 | segment_manager_foreach_segment_w_lock (seg, sm, ({ |
| 584 | if (CLIB_DEBUG && !first && !fifo_segment_has_fifos (seg) |
| 585 | && !(fifo_segment_flags (seg) & FIFO_SEGMENT_F_IS_PREALLOCATED)) |
| 586 | { |
| 587 | clib_warning ("segment %d has no fifos!", |
| 588 | segment_manager_segment_index (sm, seg)); |
| 589 | first = 0; |
| 590 | } |
| 591 | if (fifo_segment_has_fifos (seg)) |
| 592 | { |
| 593 | segment_manager_segment_reader_unlock (sm); |
| 594 | return 1; |
| 595 | } |
| 596 | })); |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 597 | |
| 598 | return 0; |
| 599 | } |
| 600 | |
| 601 | /** |
| 602 | * Initiate disconnects for all sessions 'owned' by a segment manager |
| 603 | */ |
| 604 | void |
| 605 | segment_manager_del_sessions (segment_manager_t * sm) |
| 606 | { |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 607 | session_handle_t *handles = 0, *handle; |
Florin Coras | 62ddc03 | 2019-12-08 18:30:42 -0800 | [diff] [blame] | 608 | fifo_segment_t *fs; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 609 | session_t *session; |
Florin Coras | 62ddc03 | 2019-12-08 18:30:42 -0800 | [diff] [blame] | 610 | int slice_index; |
| 611 | svm_fifo_t *f; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 612 | |
| 613 | ASSERT (pool_elts (sm->segments) != 0); |
| 614 | |
| 615 | /* Across all fifo segments used by the server */ |
Florin Coras | 62ddc03 | 2019-12-08 18:30:42 -0800 | [diff] [blame] | 616 | segment_manager_foreach_segment_w_lock (fs, sm, ({ |
| 617 | for (slice_index = 0; slice_index < fs->n_slices; slice_index++) |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 618 | { |
Florin Coras | 62ddc03 | 2019-12-08 18:30:42 -0800 | [diff] [blame] | 619 | f = fifo_segment_get_slice_fifo_list (fs, slice_index); |
| 620 | |
| 621 | /* |
| 622 | * Remove any residual sessions from the session lookup table |
| 623 | * Don't bother deleting the individual fifos, we're going to |
| 624 | * throw away the fifo segment in a minute. |
| 625 | */ |
| 626 | while (f) |
| 627 | { |
Florin Coras | c547e91 | 2020-12-08 17:50:45 -0800 | [diff] [blame] | 628 | session = session_get_if_valid (f->shr->master_session_index, |
| 629 | f->master_thread_index); |
| 630 | if (session) |
| 631 | vec_add1 (handles, session_handle (session)); |
| 632 | f = f->next; |
| 633 | } |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 634 | } |
| 635 | |
| 636 | /* Instead of removing the segment, test when cleaning up disconnected |
| 637 | * sessions if the segment can be removed. |
| 638 | */ |
| 639 | })); |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 640 | |
| 641 | vec_foreach (handle, handles) |
Florin Coras | 77ea42b | 2020-04-14 23:52:12 +0000 | [diff] [blame] | 642 | { |
| 643 | session = session_get_from_handle (*handle); |
| 644 | session_close (session); |
| 645 | /* Avoid propagating notifications back to the app */ |
| 646 | session->app_wrk_index = APP_INVALID_INDEX; |
| 647 | } |
Florin Coras | f1af21c | 2021-02-26 19:19:11 -0800 | [diff] [blame] | 648 | vec_free (handles); |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 649 | } |
| 650 | |
liuyacan | 87d48ad | 2021-04-28 11:34:03 +0000 | [diff] [blame] | 651 | /** |
| 652 | * Initiate disconnects for sessions in specified state 'owned' by a segment |
| 653 | * manager |
| 654 | */ |
| 655 | void |
| 656 | segment_manager_del_sessions_filter (segment_manager_t *sm, |
| 657 | session_state_t *states) |
| 658 | { |
| 659 | session_handle_t *handles = 0, *handle; |
| 660 | fifo_segment_t *fs; |
| 661 | session_t *session; |
| 662 | int slice_index; |
| 663 | svm_fifo_t *f; |
| 664 | |
| 665 | ASSERT (pool_elts (sm->segments) != 0); |
| 666 | |
| 667 | /* Across all fifo segments used by the server */ |
| 668 | segment_manager_foreach_segment_w_lock ( |
| 669 | fs, sm, ({ |
| 670 | for (slice_index = 0; slice_index < fs->n_slices; slice_index++) |
| 671 | { |
| 672 | f = fifo_segment_get_slice_fifo_list (fs, slice_index); |
| 673 | while (f) |
| 674 | { |
| 675 | session = session_get_if_valid (f->shr->master_session_index, |
| 676 | f->master_thread_index); |
| 677 | if (session) |
| 678 | { |
| 679 | session_state_t *state; |
| 680 | vec_foreach (state, states) |
| 681 | { |
| 682 | if (session->session_state == *state) |
| 683 | { |
| 684 | vec_add1 (handles, session_handle (session)); |
| 685 | break; |
| 686 | } |
| 687 | } |
| 688 | } |
| 689 | f = f->next; |
| 690 | } |
| 691 | } |
| 692 | })); |
| 693 | |
| 694 | vec_foreach (handle, handles) |
| 695 | { |
| 696 | session = session_get_from_handle (*handle); |
| 697 | session_close (session); |
| 698 | /* Avoid propagating notifications back to the app */ |
| 699 | session->app_wrk_index = APP_INVALID_INDEX; |
| 700 | } |
| 701 | vec_free (handles); |
| 702 | } |
| 703 | |
Florin Coras | f8f516a | 2018-02-08 15:10:09 -0800 | [diff] [blame] | 704 | int |
Florin Coras | cdfe8ab | 2021-12-22 12:54:17 -0800 | [diff] [blame] | 705 | segment_manager_try_alloc_fifos (fifo_segment_t *fs, u32 thread_index, |
Florin Coras | f8f516a | 2018-02-08 15:10:09 -0800 | [diff] [blame] | 706 | u32 rx_fifo_size, u32 tx_fifo_size, |
Florin Coras | cdfe8ab | 2021-12-22 12:54:17 -0800 | [diff] [blame] | 707 | svm_fifo_t **rx_fifo, svm_fifo_t **tx_fifo) |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 708 | { |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 709 | rx_fifo_size = clib_max (rx_fifo_size, sm_main.default_fifo_size); |
Florin Coras | cdfe8ab | 2021-12-22 12:54:17 -0800 | [diff] [blame] | 710 | *rx_fifo = fifo_segment_alloc_fifo_w_slice (fs, thread_index, rx_fifo_size, |
Florin Coras | 62ddc03 | 2019-12-08 18:30:42 -0800 | [diff] [blame] | 711 | FIFO_SEGMENT_RX_FIFO); |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 712 | |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 713 | tx_fifo_size = clib_max (tx_fifo_size, sm_main.default_fifo_size); |
Florin Coras | cdfe8ab | 2021-12-22 12:54:17 -0800 | [diff] [blame] | 714 | *tx_fifo = fifo_segment_alloc_fifo_w_slice (fs, thread_index, tx_fifo_size, |
Florin Coras | 62ddc03 | 2019-12-08 18:30:42 -0800 | [diff] [blame] | 715 | FIFO_SEGMENT_TX_FIFO); |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 716 | |
| 717 | if (*rx_fifo == 0) |
| 718 | { |
| 719 | /* This would be very odd, but handle it... */ |
| 720 | if (*tx_fifo != 0) |
| 721 | { |
Florin Coras | cdfe8ab | 2021-12-22 12:54:17 -0800 | [diff] [blame] | 722 | fifo_segment_free_fifo (fs, *tx_fifo); |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 723 | *tx_fifo = 0; |
| 724 | } |
Florin Coras | cdfe8ab | 2021-12-22 12:54:17 -0800 | [diff] [blame] | 725 | return SESSION_E_SEG_NO_SPACE; |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 726 | } |
| 727 | if (*tx_fifo == 0) |
| 728 | { |
| 729 | if (*rx_fifo != 0) |
| 730 | { |
Florin Coras | cdfe8ab | 2021-12-22 12:54:17 -0800 | [diff] [blame] | 731 | fifo_segment_free_fifo (fs, *rx_fifo); |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 732 | *rx_fifo = 0; |
| 733 | } |
Florin Coras | cdfe8ab | 2021-12-22 12:54:17 -0800 | [diff] [blame] | 734 | return SESSION_E_SEG_NO_SPACE; |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 735 | } |
| 736 | |
| 737 | return 0; |
| 738 | } |
| 739 | |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 740 | static inline int |
| 741 | sm_lookup_segment_and_alloc_fifos (segment_manager_t *sm, |
| 742 | segment_manager_props_t *props, |
| 743 | u32 thread_index, svm_fifo_t **rx_fifo, |
| 744 | svm_fifo_t **tx_fifo) |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 745 | { |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 746 | uword free_bytes, max_free_bytes; |
| 747 | fifo_segment_t *cur, *fs = 0; |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 748 | |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 749 | max_free_bytes = props->rx_fifo_size + props->tx_fifo_size - 1; |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 750 | |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 751 | pool_foreach (cur, sm->segments) |
| 752 | { |
Florin Coras | da78c5a | 2021-06-09 14:55:24 -0700 | [diff] [blame] | 753 | if (fifo_segment_flags (cur) & FIFO_SEGMENT_F_CUSTOM_USE) |
| 754 | continue; |
| 755 | free_bytes = fifo_segment_available_bytes (cur); |
| 756 | if (free_bytes > max_free_bytes) |
| 757 | { |
| 758 | max_free_bytes = free_bytes; |
| 759 | fs = cur; |
| 760 | } |
Florin Coras | 75ccf7b | 2020-03-05 19:44:02 +0000 | [diff] [blame] | 761 | } |
Ryujiro Shibuya | 234fe89 | 2019-12-25 07:40:54 +0000 | [diff] [blame] | 762 | |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 763 | if (PREDICT_FALSE (!fs)) |
| 764 | return SESSION_E_SEG_NO_SPACE; |
| 765 | |
Florin Coras | cdfe8ab | 2021-12-22 12:54:17 -0800 | [diff] [blame] | 766 | return segment_manager_try_alloc_fifos ( |
| 767 | fs, thread_index, props->rx_fifo_size, props->tx_fifo_size, rx_fifo, |
| 768 | tx_fifo); |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 769 | } |
| 770 | |
| 771 | static int |
| 772 | sm_lock_and_alloc_segment_and_fifos (segment_manager_t *sm, |
| 773 | segment_manager_props_t *props, |
| 774 | u32 thread_index, svm_fifo_t **rx_fifo, |
| 775 | svm_fifo_t **tx_fifo) |
| 776 | { |
| 777 | int new_fs_index, rv; |
| 778 | fifo_segment_t *fs; |
| 779 | |
| 780 | if (!props->add_segment) |
| 781 | return SESSION_E_SEG_NO_SPACE; |
| 782 | |
| 783 | clib_rwlock_writer_lock (&sm->segments_rwlock); |
| 784 | |
| 785 | /* Make sure there really is no free space. Another worker might've freed |
| 786 | * some fifos or allocated a segment */ |
| 787 | rv = sm_lookup_segment_and_alloc_fifos (sm, props, thread_index, rx_fifo, |
| 788 | tx_fifo); |
Florin Coras | cdfe8ab | 2021-12-22 12:54:17 -0800 | [diff] [blame] | 789 | if (!rv) |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 790 | goto done; |
| 791 | |
| 792 | new_fs_index = |
| 793 | segment_manager_add_segment (sm, 0 /* segment_size*/, 1 /* notify_app */); |
| 794 | if (new_fs_index < 0) |
| 795 | { |
| 796 | rv = SESSION_E_SEG_CREATE; |
| 797 | goto done; |
| 798 | } |
| 799 | fs = segment_manager_get_segment (sm, new_fs_index); |
| 800 | rv = segment_manager_try_alloc_fifos (fs, thread_index, props->rx_fifo_size, |
| 801 | props->tx_fifo_size, rx_fifo, tx_fifo); |
| 802 | if (rv) |
| 803 | { |
Filip Tehlar | ac3c8dc | 2023-03-14 08:50:28 +0100 | [diff] [blame] | 804 | SESSION_DBG ("Added a segment, still can't allocate a fifo"); |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 805 | rv = SESSION_E_SEG_NO_SPACE2; |
| 806 | goto done; |
| 807 | } |
| 808 | |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 809 | done: |
| 810 | |
| 811 | clib_rwlock_writer_unlock (&sm->segments_rwlock); |
| 812 | |
| 813 | return rv; |
| 814 | } |
| 815 | |
| 816 | int |
| 817 | segment_manager_alloc_session_fifos (segment_manager_t * sm, |
| 818 | u32 thread_index, |
| 819 | svm_fifo_t ** rx_fifo, |
| 820 | svm_fifo_t ** tx_fifo) |
| 821 | { |
| 822 | segment_manager_props_t *props; |
Florin Coras | cdfe8ab | 2021-12-22 12:54:17 -0800 | [diff] [blame] | 823 | int rv; |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 824 | |
| 825 | props = segment_manager_properties_get (sm); |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 826 | |
| 827 | /* |
| 828 | * Fast path: find the first segment with enough free space and |
| 829 | * try to allocate the fifos. Done with reader lock |
| 830 | */ |
| 831 | |
| 832 | segment_manager_segment_reader_lock (sm); |
| 833 | |
Florin Coras | cdfe8ab | 2021-12-22 12:54:17 -0800 | [diff] [blame] | 834 | rv = sm_lookup_segment_and_alloc_fifos (sm, props, thread_index, rx_fifo, |
| 835 | tx_fifo); |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 836 | |
Florin Coras | 75ccf7b | 2020-03-05 19:44:02 +0000 | [diff] [blame] | 837 | segment_manager_segment_reader_unlock (sm); |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 838 | |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 839 | /* |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 840 | * Slow path: if no fifo segment or alloc fail grab writer lock and try |
| 841 | * to allocate new segment |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 842 | */ |
Florin Coras | cdfe8ab | 2021-12-22 12:54:17 -0800 | [diff] [blame] | 843 | if (PREDICT_FALSE (rv < 0)) |
| 844 | return sm_lock_and_alloc_segment_and_fifos (sm, props, thread_index, |
| 845 | rx_fifo, tx_fifo); |
liuyacan | 9f29903 | 2021-04-25 20:11:30 +0800 | [diff] [blame] | 846 | |
Florin Coras | c824977 | 2021-11-30 20:21:57 -0800 | [diff] [blame] | 847 | return 0; |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 848 | } |
| 849 | |
| 850 | void |
Florin Coras | 19223e0 | 2019-03-03 14:56:05 -0800 | [diff] [blame] | 851 | segment_manager_dealloc_fifos (svm_fifo_t * rx_fifo, svm_fifo_t * tx_fifo) |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 852 | { |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 853 | segment_manager_t *sm; |
Florin Coras | b095a3c | 2019-04-25 12:58:46 -0700 | [diff] [blame] | 854 | fifo_segment_t *fs; |
Florin Coras | 19223e0 | 2019-03-03 14:56:05 -0800 | [diff] [blame] | 855 | u32 segment_index; |
Florin Coras | 02cc719 | 2021-11-27 14:44:59 -0800 | [diff] [blame] | 856 | u8 try_delete = 0; |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 857 | |
Florin Coras | 58a93e8 | 2019-01-14 23:33:46 -0800 | [diff] [blame] | 858 | if (!rx_fifo || !tx_fifo) |
| 859 | return; |
| 860 | |
Florin Coras | 5122234 | 2022-06-01 14:44:11 -0700 | [diff] [blame] | 861 | /* Thread that allocated the fifos must be the one to clean them up */ |
| 862 | ASSERT (rx_fifo->master_thread_index == vlib_get_thread_index () || |
Florin Coras | 0242d30 | 2022-12-22 15:03:44 -0800 | [diff] [blame] | 863 | rx_fifo->refcnt > 1 || vlib_thread_is_main_w_barrier ()); |
Florin Coras | 5122234 | 2022-06-01 14:44:11 -0700 | [diff] [blame] | 864 | |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 865 | /* It's possible to have no segment manager if the session was removed |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 866 | * as result of a detach. */ |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 867 | if (!(sm = segment_manager_get_if_valid (rx_fifo->segment_manager))) |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 868 | return; |
| 869 | |
Florin Coras | 19223e0 | 2019-03-03 14:56:05 -0800 | [diff] [blame] | 870 | segment_index = rx_fifo->segment_index; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 871 | fs = segment_manager_get_segment_w_lock (sm, segment_index); |
| 872 | fifo_segment_free_fifo (fs, rx_fifo); |
| 873 | fifo_segment_free_fifo (fs, tx_fifo); |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 874 | |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 875 | /* |
Florin Coras | 02cc719 | 2021-11-27 14:44:59 -0800 | [diff] [blame] | 876 | * Try to remove fifo segment if it has no fifos. This can be done only if |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 877 | * the segment is not the first in the segment manager or if it is first |
| 878 | * and it is not protected. Moreover, if the segment is first and the app |
| 879 | * has detached from the segment manager, remove the segment manager. |
| 880 | */ |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 881 | if (!fifo_segment_has_fifos (fs)) |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 882 | { |
Florin Coras | 02cc719 | 2021-11-27 14:44:59 -0800 | [diff] [blame] | 883 | /* If first, remove only if not protected */ |
| 884 | try_delete = segment_index != 0 || !sm->first_is_protected; |
| 885 | } |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 886 | |
Florin Coras | 02cc719 | 2021-11-27 14:44:59 -0800 | [diff] [blame] | 887 | segment_manager_segment_reader_unlock (sm); |
| 888 | |
| 889 | if (PREDICT_FALSE (try_delete)) |
| 890 | { |
| 891 | /* Only remove if empty after writer lock acquired */ |
| 892 | sm_lock_and_del_segment_inline (sm, segment_index, |
| 893 | 1 /* check_if_empty */); |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 894 | |
| 895 | /* Remove segment manager if no sessions and detached from app */ |
Florin Coras | 9d06304 | 2017-09-14 03:08:00 -0400 | [diff] [blame] | 896 | if (segment_manager_app_detached (sm) |
| 897 | && !segment_manager_has_fifos (sm)) |
Florin Coras | 506aa3c | 2020-12-13 21:09:59 -0800 | [diff] [blame] | 898 | segment_manager_free_safe (sm); |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 899 | } |
| 900 | } |
| 901 | |
Florin Coras | 6d7552c | 2020-04-09 01:49:45 +0000 | [diff] [blame] | 902 | void |
Florin Coras | 0bc78d8 | 2021-01-09 14:34:01 -0800 | [diff] [blame] | 903 | segment_manager_detach_fifo (segment_manager_t *sm, svm_fifo_t **f) |
Florin Coras | 6d7552c | 2020-04-09 01:49:45 +0000 | [diff] [blame] | 904 | { |
| 905 | fifo_segment_t *fs; |
| 906 | |
Florin Coras | 0bc78d8 | 2021-01-09 14:34:01 -0800 | [diff] [blame] | 907 | fs = segment_manager_get_segment_w_lock (sm, (*f)->segment_index); |
Florin Coras | 6d7552c | 2020-04-09 01:49:45 +0000 | [diff] [blame] | 908 | fifo_segment_detach_fifo (fs, f); |
| 909 | segment_manager_segment_reader_unlock (sm); |
| 910 | } |
| 911 | |
| 912 | void |
Florin Coras | 0bc78d8 | 2021-01-09 14:34:01 -0800 | [diff] [blame] | 913 | segment_manager_attach_fifo (segment_manager_t *sm, svm_fifo_t **f, |
| 914 | session_t *s) |
Florin Coras | 6d7552c | 2020-04-09 01:49:45 +0000 | [diff] [blame] | 915 | { |
| 916 | fifo_segment_t *fs; |
| 917 | |
Florin Coras | 0bc78d8 | 2021-01-09 14:34:01 -0800 | [diff] [blame] | 918 | fs = segment_manager_get_segment_w_lock (sm, (*f)->segment_index); |
Florin Coras | 6d7552c | 2020-04-09 01:49:45 +0000 | [diff] [blame] | 919 | fifo_segment_attach_fifo (fs, f, s->thread_index); |
| 920 | segment_manager_segment_reader_unlock (sm); |
| 921 | |
Florin Coras | 0bc78d8 | 2021-01-09 14:34:01 -0800 | [diff] [blame] | 922 | (*f)->shr->master_session_index = s->session_index; |
| 923 | (*f)->master_thread_index = s->thread_index; |
Florin Coras | 6d7552c | 2020-04-09 01:49:45 +0000 | [diff] [blame] | 924 | } |
| 925 | |
Florin Coras | 3c2fed5 | 2018-07-04 04:15:05 -0700 | [diff] [blame] | 926 | u32 |
| 927 | segment_manager_evt_q_expected_size (u32 q_len) |
| 928 | { |
| 929 | u32 fifo_evt_size, notif_q_size, q_hdrs; |
| 930 | u32 msg_q_sz, fifo_evt_ring_sz, session_ntf_ring_sz; |
| 931 | |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 932 | fifo_evt_size = 1 << max_log2 (sizeof (session_event_t)); |
Florin Coras | 3c2fed5 | 2018-07-04 04:15:05 -0700 | [diff] [blame] | 933 | notif_q_size = clib_max (16, q_len >> 4); |
| 934 | |
| 935 | msg_q_sz = q_len * sizeof (svm_msg_q_msg_t); |
| 936 | fifo_evt_ring_sz = q_len * fifo_evt_size; |
| 937 | session_ntf_ring_sz = notif_q_size * 256; |
| 938 | q_hdrs = sizeof (svm_queue_t) + sizeof (svm_msg_q_t); |
| 939 | |
| 940 | return (msg_q_sz + fifo_evt_ring_sz + session_ntf_ring_sz + q_hdrs); |
| 941 | } |
| 942 | |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 943 | /** |
| 944 | * Allocates shm queue in the first segment |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 945 | * |
| 946 | * Must be called with lock held |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 947 | */ |
Florin Coras | 3c2fed5 | 2018-07-04 04:15:05 -0700 | [diff] [blame] | 948 | svm_msg_q_t * |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 949 | segment_manager_alloc_queue (fifo_segment_t * segment, |
| 950 | segment_manager_props_t * props) |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 951 | { |
Florin Coras | 3c2fed5 | 2018-07-04 04:15:05 -0700 | [diff] [blame] | 952 | u32 fifo_evt_size, session_evt_size = 256, notif_q_size; |
| 953 | svm_msg_q_cfg_t _cfg, *cfg = &_cfg; |
| 954 | svm_msg_q_t *q; |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 955 | |
Florin Coras | 52207f1 | 2018-07-12 14:48:06 -0700 | [diff] [blame] | 956 | fifo_evt_size = sizeof (session_event_t); |
Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 957 | notif_q_size = clib_max (16, props->evt_q_size >> 4); |
Florin Coras | 3c2fed5 | 2018-07-04 04:15:05 -0700 | [diff] [blame] | 958 | svm_msg_q_ring_cfg_t rc[SESSION_MQ_N_RINGS] = { |
Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 959 | {props->evt_q_size, fifo_evt_size, 0}, |
Florin Coras | 3c2fed5 | 2018-07-04 04:15:05 -0700 | [diff] [blame] | 960 | {notif_q_size, session_evt_size, 0} |
| 961 | }; |
Florin Coras | 3c2fed5 | 2018-07-04 04:15:05 -0700 | [diff] [blame] | 962 | cfg->consumer_pid = 0; |
| 963 | cfg->n_rings = 2; |
Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 964 | cfg->q_nitems = props->evt_q_size; |
Florin Coras | 3c2fed5 | 2018-07-04 04:15:05 -0700 | [diff] [blame] | 965 | cfg->ring_cfgs = rc; |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 966 | |
Florin Coras | b462418 | 2020-12-11 13:58:12 -0800 | [diff] [blame] | 967 | q = fifo_segment_msg_q_alloc (segment, 0, cfg); |
Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 968 | |
| 969 | if (props->use_mq_eventfd) |
| 970 | { |
Florin Coras | 86f1232 | 2021-01-22 15:05:14 -0800 | [diff] [blame] | 971 | if (svm_msg_q_alloc_eventfd (q)) |
Florin Coras | 9936831 | 2018-08-02 10:45:44 -0700 | [diff] [blame] | 972 | clib_warning ("failed to alloc eventfd"); |
| 973 | } |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 974 | return q; |
| 975 | } |
| 976 | |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 977 | svm_msg_q_t * |
| 978 | segment_manager_event_queue (segment_manager_t * sm) |
| 979 | { |
| 980 | return sm->event_queue; |
| 981 | } |
| 982 | |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 983 | /** |
| 984 | * Frees shm queue allocated in the first segment |
| 985 | */ |
| 986 | void |
Florin Coras | e86a8ed | 2018-01-05 03:20:25 -0800 | [diff] [blame] | 987 | segment_manager_dealloc_queue (segment_manager_t * sm, svm_queue_t * q) |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 988 | { |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 989 | fifo_segment_t *segment; |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 990 | ssvm_shared_header_t *sh; |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 991 | void *oldheap; |
| 992 | |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 993 | ASSERT (!pool_is_free_index (sm->segments, 0)); |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 994 | |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 995 | segment = segment_manager_get_segment_w_lock (sm, 0); |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 996 | sh = segment->ssvm.sh; |
| 997 | |
| 998 | oldheap = ssvm_push_heap (sh); |
Florin Coras | e86a8ed | 2018-01-05 03:20:25 -0800 | [diff] [blame] | 999 | svm_queue_free (q); |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 1000 | ssvm_pop_heap (oldheap); |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 1001 | segment_manager_segment_reader_unlock (sm); |
| 1002 | } |
| 1003 | |
| 1004 | /* |
| 1005 | * Init segment vm address allocator |
| 1006 | */ |
| 1007 | void |
Florin Coras | 9a45bd8 | 2020-12-28 16:28:07 -0800 | [diff] [blame] | 1008 | segment_manager_main_init (void) |
Florin Coras | a332c46 | 2018-01-31 06:52:17 -0800 | [diff] [blame] | 1009 | { |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1010 | segment_manager_main_t *sm = &sm_main; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1011 | |
| 1012 | sm->default_fifo_size = 1 << 12; |
| 1013 | sm->default_segment_size = 1 << 20; |
| 1014 | sm->default_app_mq_size = 128; |
Ryujiro Shibuya | d8f48e2 | 2020-01-22 12:11:42 +0000 | [diff] [blame] | 1015 | sm->default_max_fifo_size = 4 << 20; |
Ryujiro Shibuya | 234fe89 | 2019-12-25 07:40:54 +0000 | [diff] [blame] | 1016 | sm->default_high_watermark = 80; |
| 1017 | sm->default_low_watermark = 50; |
Florin Coras | a546481 | 2017-04-19 13:00:05 -0700 | [diff] [blame] | 1018 | } |
| 1019 | |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 1020 | static u8 * |
| 1021 | format_segment_manager (u8 *s, va_list *args) |
| 1022 | { |
| 1023 | segment_manager_t *sm = va_arg (*args, segment_manager_t *); |
| 1024 | int verbose = va_arg (*args, int); |
| 1025 | app_worker_t *app_wrk; |
| 1026 | uword max_fifo_size; |
| 1027 | fifo_segment_t *seg; |
| 1028 | application_t *app; |
| 1029 | u8 custom_logic; |
| 1030 | |
| 1031 | app_wrk = app_worker_get_if_valid (sm->app_wrk_index); |
| 1032 | app = app_wrk ? application_get (app_wrk->app_index) : 0; |
| 1033 | custom_logic = (app && (app->cb_fns.fifo_tuning_callback)) ? 1 : 0; |
| 1034 | max_fifo_size = sm->max_fifo_size; |
| 1035 | |
| 1036 | s = format (s, |
| 1037 | "[%u] %v app-wrk: %u segs: %u max-fifo-sz: %U " |
| 1038 | "wmarks: %u %u %s flags: 0x%x", |
Florin Coras | 36eaaf3 | 2022-01-10 13:44:45 -0800 | [diff] [blame] | 1039 | segment_manager_index (sm), app ? app->name : 0, |
| 1040 | sm->app_wrk_index, pool_elts (sm->segments), format_memory_size, |
| 1041 | max_fifo_size, sm->high_watermark, sm->low_watermark, |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 1042 | custom_logic ? "custom-tuning" : "no-tuning", sm->flags); |
| 1043 | |
| 1044 | if (!verbose || !pool_elts (sm->segments)) |
| 1045 | return s; |
| 1046 | |
| 1047 | s = format (s, "\n\n"); |
| 1048 | |
| 1049 | segment_manager_foreach_segment_w_lock ( |
| 1050 | seg, sm, ({ s = format (s, " *%U", format_fifo_segment, seg, verbose); })); |
| 1051 | |
| 1052 | return s; |
| 1053 | } |
| 1054 | |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 1055 | static clib_error_t * |
| 1056 | segment_manager_show_fn (vlib_main_t * vm, unformat_input_t * input, |
| 1057 | vlib_cli_command_t * cmd) |
| 1058 | { |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 1059 | unformat_input_t _line_input, *line_input = &_line_input; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1060 | segment_manager_main_t *smm = &sm_main; |
Florin Coras | b384b54 | 2018-01-15 01:08:33 -0800 | [diff] [blame] | 1061 | u8 show_segments = 0, verbose = 0; |
Florin Coras | 5368bb0 | 2019-06-09 09:24:33 -0700 | [diff] [blame] | 1062 | segment_manager_t *sm; |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 1063 | u32 sm_index = ~0; |
Dave Barach | 91f3e74 | 2017-09-01 19:12:11 -0400 | [diff] [blame] | 1064 | |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 1065 | if (!unformat_user (input, unformat_line_input, line_input)) |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 1066 | { |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 1067 | vlib_cli_output (vm, "%d segment managers allocated", |
| 1068 | pool_elts (smm->segment_managers)); |
| 1069 | return 0; |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 1070 | } |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 1071 | |
| 1072 | while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 1073 | { |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 1074 | if (unformat (line_input, "segments")) |
| 1075 | show_segments = 1; |
| 1076 | else if (unformat (line_input, "verbose")) |
| 1077 | verbose = 1; |
| 1078 | else if (unformat (line_input, "index %u", &sm_index)) |
| 1079 | ; |
| 1080 | else |
| 1081 | { |
| 1082 | vlib_cli_output (vm, "unknown input [%U]", format_unformat_error, |
| 1083 | line_input); |
| 1084 | goto done; |
| 1085 | } |
| 1086 | } |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 1087 | |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 1088 | if (!pool_elts (smm->segment_managers)) |
| 1089 | goto done; |
| 1090 | |
| 1091 | if (sm_index != ~0) |
| 1092 | { |
| 1093 | sm = segment_manager_get_if_valid (sm_index); |
| 1094 | if (!sm) |
| 1095 | { |
| 1096 | vlib_cli_output (vm, "segment manager %u not allocated", sm_index); |
| 1097 | goto done; |
| 1098 | } |
| 1099 | vlib_cli_output (vm, "%U", format_segment_manager, sm, 1 /* verbose */); |
| 1100 | goto done; |
| 1101 | } |
| 1102 | |
| 1103 | if (verbose || show_segments) |
| 1104 | { |
Damjan Marion | b2c31b6 | 2020-12-13 21:47:40 +0100 | [diff] [blame] | 1105 | pool_foreach (sm, smm->segment_managers) { |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 1106 | vlib_cli_output (vm, "%U", format_segment_manager, sm, |
| 1107 | show_segments); |
Damjan Marion | b2c31b6 | 2020-12-13 21:47:40 +0100 | [diff] [blame] | 1108 | } |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 1109 | |
Florin Coras | 2a7c0b6 | 2020-09-28 23:40:28 -0700 | [diff] [blame] | 1110 | vlib_cli_output (vm, "\n"); |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 1111 | } |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 1112 | |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 1113 | done: |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 1114 | |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 1115 | unformat_free (line_input); |
| 1116 | |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 1117 | return 0; |
| 1118 | } |
| 1119 | |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 1120 | VLIB_CLI_COMMAND (segment_manager_show_command, static) = { |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 1121 | .path = "show segment-manager", |
Florin Coras | a0024a6 | 2021-12-16 10:09:17 -0800 | [diff] [blame] | 1122 | .short_help = "show segment-manager [segments][verbose][index <nn>]", |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 1123 | .function = segment_manager_show_fn, |
| 1124 | }; |
Florin Coras | c87c91d | 2017-08-16 19:55:49 -0700 | [diff] [blame] | 1125 | |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1126 | void |
| 1127 | segment_manager_format_sessions (segment_manager_t * sm, int verbose) |
| 1128 | { |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1129 | vlib_main_t *vm = vlib_get_main (); |
| 1130 | app_worker_t *app_wrk; |
Florin Coras | 62ddc03 | 2019-12-08 18:30:42 -0800 | [diff] [blame] | 1131 | fifo_segment_t *fs; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1132 | const u8 *app_name; |
Florin Coras | 62ddc03 | 2019-12-08 18:30:42 -0800 | [diff] [blame] | 1133 | int slice_index; |
| 1134 | u8 *s = 0, *str; |
| 1135 | svm_fifo_t *f; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1136 | |
| 1137 | if (!sm) |
| 1138 | { |
| 1139 | if (verbose) |
Xiaoming Jiang | 806709f | 2021-06-23 09:07:57 +0000 | [diff] [blame] | 1140 | vlib_cli_output (vm, "%-" SESSION_CLI_ID_LEN "s%-20s%-15s%-10s", |
| 1141 | "Connection", "App", "API Client", "SegManager"); |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1142 | else |
Xiaoming Jiang | 806709f | 2021-06-23 09:07:57 +0000 | [diff] [blame] | 1143 | vlib_cli_output (vm, "%-" SESSION_CLI_ID_LEN "s%-20s", "Connection", |
| 1144 | "App"); |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1145 | return; |
| 1146 | } |
| 1147 | |
| 1148 | app_wrk = app_worker_get (sm->app_wrk_index); |
| 1149 | app_name = application_name_from_index (app_wrk->app_index); |
| 1150 | |
| 1151 | clib_rwlock_reader_lock (&sm->segments_rwlock); |
| 1152 | |
Damjan Marion | b2c31b6 | 2020-12-13 21:47:40 +0100 | [diff] [blame] | 1153 | pool_foreach (fs, sm->segments) { |
Florin Coras | 62ddc03 | 2019-12-08 18:30:42 -0800 | [diff] [blame] | 1154 | for (slice_index = 0; slice_index < fs->n_slices; slice_index++) |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1155 | { |
Florin Coras | 62ddc03 | 2019-12-08 18:30:42 -0800 | [diff] [blame] | 1156 | f = fifo_segment_get_slice_fifo_list (fs, slice_index); |
| 1157 | while (f) |
| 1158 | { |
| 1159 | u32 session_index, thread_index; |
| 1160 | session_t *session; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1161 | |
Florin Coras | c547e91 | 2020-12-08 17:50:45 -0800 | [diff] [blame] | 1162 | session_index = f->shr->master_session_index; |
| 1163 | thread_index = f->master_thread_index; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1164 | |
Florin Coras | c547e91 | 2020-12-08 17:50:45 -0800 | [diff] [blame] | 1165 | session = session_get (session_index, thread_index); |
| 1166 | str = format (0, "%U", format_session, session, verbose); |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1167 | |
Florin Coras | c547e91 | 2020-12-08 17:50:45 -0800 | [diff] [blame] | 1168 | if (verbose) |
Xiaoming Jiang | 806709f | 2021-06-23 09:07:57 +0000 | [diff] [blame] | 1169 | s = format (s, "%-" SESSION_CLI_ID_LEN "v%-20v%-15u%-10u", str, |
| 1170 | app_name, app_wrk->api_client_index, |
Florin Coras | c547e91 | 2020-12-08 17:50:45 -0800 | [diff] [blame] | 1171 | app_wrk->connects_seg_manager); |
| 1172 | else |
Xiaoming Jiang | 806709f | 2021-06-23 09:07:57 +0000 | [diff] [blame] | 1173 | s = format (s, "%-" SESSION_CLI_ID_LEN "v%-20v", str, app_name); |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1174 | |
Florin Coras | c547e91 | 2020-12-08 17:50:45 -0800 | [diff] [blame] | 1175 | vlib_cli_output (vm, "%v", s); |
| 1176 | vec_reset_length (s); |
| 1177 | vec_free (str); |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1178 | |
Florin Coras | c547e91 | 2020-12-08 17:50:45 -0800 | [diff] [blame] | 1179 | f = f->next; |
| 1180 | } |
| 1181 | vec_free (s); |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1182 | } |
Damjan Marion | b2c31b6 | 2020-12-13 21:47:40 +0100 | [diff] [blame] | 1183 | } |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 1184 | |
| 1185 | clib_rwlock_reader_unlock (&sm->segments_rwlock); |
| 1186 | } |
| 1187 | |
Ryujiro Shibuya | 234fe89 | 2019-12-25 07:40:54 +0000 | [diff] [blame] | 1188 | void |
| 1189 | segment_manager_set_watermarks (segment_manager_t * sm, |
| 1190 | u8 high_watermark, u8 low_watermark) |
| 1191 | { |
Florin Coras | 8c79a4e | 2020-02-25 22:28:27 +0000 | [diff] [blame] | 1192 | ASSERT (high_watermark <= 100 && low_watermark <= 100 && |
Ryujiro Shibuya | 234fe89 | 2019-12-25 07:40:54 +0000 | [diff] [blame] | 1193 | low_watermark <= high_watermark); |
| 1194 | |
| 1195 | sm->high_watermark = high_watermark; |
| 1196 | sm->low_watermark = low_watermark; |
| 1197 | } |
| 1198 | |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 1199 | /* |
| 1200 | * fd.io coding-style-patch-verification: ON |
| 1201 | * |
| 1202 | * Local Variables: |
| 1203 | * eval: (c-set-style "gnu") |
| 1204 | * End: |
| 1205 | */ |