Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 1 | /* |
Florin Coras | c5df8c7 | 2019-04-08 07:42:30 -0700 | [diff] [blame] | 2 | * Copyright (c) 2016-2019 Cisco and/or its affiliates. |
Sirshak Das | 28aa539 | 2019-02-05 01:33:33 -0600 | [diff] [blame] | 3 | * Copyright (c) 2019 Arm Limited |
| 4 | * Copyright (c) 2010-2017 Intel Corporation and/or its affiliates. |
| 5 | * Copyright (c) 2007-2009 Kip Macy kmacy@freebsd.org |
| 6 | * Inspired from DPDK rte_ring.h (SPSC only) (derived from freebsd bufring.h). |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 7 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 8 | * you may not use this file except in compliance with the License. |
| 9 | * You may obtain a copy of the License at: |
| 10 | * |
| 11 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 12 | * |
| 13 | * Unless required by applicable law or agreed to in writing, software |
| 14 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 16 | * See the License for the specific language governing permissions and |
| 17 | * limitations under the License. |
| 18 | */ |
| 19 | #ifndef __included_ssvm_fifo_h__ |
| 20 | #define __included_ssvm_fifo_h__ |
| 21 | |
| 22 | #include <vppinfra/clib.h> |
| 23 | #include <vppinfra/vec.h> |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 24 | #include <vppinfra/pool.h> |
| 25 | #include <vppinfra/format.h> |
Florin Coras | f22f4e5 | 2019-12-19 16:10:58 -0800 | [diff] [blame] | 26 | #include <svm/fifo_types.h> |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 27 | |
Florin Coras | 326b81e | 2018-10-04 19:03:05 -0700 | [diff] [blame] | 28 | #define OOO_SEGMENT_INVALID_INDEX ((u32)~0) |
| 29 | #define SVM_FIFO_INVALID_SESSION_INDEX ((u32)~0) |
Florin Coras | fa76a76 | 2018-11-29 12:40:10 -0800 | [diff] [blame] | 30 | #define SVM_FIFO_INVALID_INDEX ((u32)~0) |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 31 | |
Florin Coras | 2d379d8 | 2019-06-28 12:45:12 -0700 | [diff] [blame] | 32 | typedef enum svm_fifo_deq_ntf_ |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 33 | { |
Florin Coras | 2d379d8 | 2019-06-28 12:45:12 -0700 | [diff] [blame] | 34 | SVM_FIFO_NO_DEQ_NOTIF = 0, /**< No notification requested */ |
| 35 | SVM_FIFO_WANT_DEQ_NOTIF = 1, /**< Notify on dequeue */ |
| 36 | SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL = 2, /**< Notify on transition from full */ |
| 37 | SVM_FIFO_WANT_DEQ_NOTIF_IF_EMPTY = 4, /**< Notify on transition to empty */ |
| 38 | } svm_fifo_deq_ntf_t; |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 39 | |
Florin Coras | 2309e7a | 2019-04-18 18:58:10 -0700 | [diff] [blame] | 40 | typedef enum svm_fifo_flag_ |
| 41 | { |
Florin Coras | f22f4e5 | 2019-12-19 16:10:58 -0800 | [diff] [blame] | 42 | SVM_FIFO_F_LL_TRACKED = 1 << 0, |
Florin Coras | 2309e7a | 2019-04-18 18:58:10 -0700 | [diff] [blame] | 43 | } svm_fifo_flag_t; |
| 44 | |
Florin Coras | 7fb0fe1 | 2018-04-09 09:24:52 -0700 | [diff] [blame] | 45 | typedef enum |
| 46 | { |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 47 | SVM_FIFO_EFULL = -2, |
| 48 | SVM_FIFO_EEMPTY = -3, |
Florin Coras | f22f4e5 | 2019-12-19 16:10:58 -0800 | [diff] [blame] | 49 | SVM_FIFO_EGROW = -4, |
Florin Coras | 7fb0fe1 | 2018-04-09 09:24:52 -0700 | [diff] [blame] | 50 | } svm_fifo_err_t; |
| 51 | |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 52 | typedef struct svm_fifo_seg_ |
Florin Coras | 2cba853 | 2018-09-11 16:33:36 -0700 | [diff] [blame] | 53 | { |
| 54 | u8 *data; |
| 55 | u32 len; |
Florin Coras | 88001c6 | 2019-04-24 14:44:46 -0700 | [diff] [blame] | 56 | } svm_fifo_seg_t; |
Florin Coras | 2cba853 | 2018-09-11 16:33:36 -0700 | [diff] [blame] | 57 | |
Florin Coras | 3eb5062 | 2017-07-13 01:24:57 -0400 | [diff] [blame] | 58 | #if SVM_FIFO_TRACE |
| 59 | #define svm_fifo_trace_add(_f, _s, _l, _t) \ |
| 60 | { \ |
| 61 | svm_fifo_trace_elem_t *trace_elt; \ |
| 62 | vec_add2(_f->trace, trace_elt, 1); \ |
| 63 | trace_elt->offset = _s; \ |
| 64 | trace_elt->len = _l; \ |
| 65 | trace_elt->action = _t; \ |
| 66 | } |
| 67 | #else |
| 68 | #define svm_fifo_trace_add(_f, _s, _l, _t) |
| 69 | #endif |
| 70 | |
| 71 | u8 *svm_fifo_dump_trace (u8 * s, svm_fifo_t * f); |
| 72 | u8 *svm_fifo_replay (u8 * s, svm_fifo_t * f, u8 no_read, u8 verbose); |
| 73 | |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 74 | /** |
| 75 | * Load head and tail optimized for consumer |
| 76 | * |
| 77 | * Internal function. |
| 78 | */ |
Sirshak Das | 28aa539 | 2019-02-05 01:33:33 -0600 | [diff] [blame] | 79 | static inline void |
| 80 | f_load_head_tail_cons (svm_fifo_t * f, u32 * head, u32 * tail) |
| 81 | { |
| 82 | /* load-relaxed: consumer owned index */ |
| 83 | *head = f->head; |
| 84 | /* load-acq: consumer foreign index (paired with store-rel in producer) */ |
| 85 | *tail = clib_atomic_load_acq_n (&f->tail); |
| 86 | } |
| 87 | |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 88 | /** Load head and tail optimized for producer |
| 89 | * |
| 90 | * Internal function |
| 91 | */ |
Sirshak Das | 28aa539 | 2019-02-05 01:33:33 -0600 | [diff] [blame] | 92 | static inline void |
| 93 | f_load_head_tail_prod (svm_fifo_t * f, u32 * head, u32 * tail) |
| 94 | { |
| 95 | /* load relaxed: producer owned index */ |
| 96 | *tail = f->tail; |
| 97 | /* load-acq: producer foreign index (paired with store-rel in consumer) */ |
| 98 | *head = clib_atomic_load_acq_n (&f->head); |
| 99 | } |
| 100 | |
Florin Coras | a7570b0 | 2019-05-02 12:52:19 -0700 | [diff] [blame] | 101 | /** |
| 102 | * Load head and tail independent of producer/consumer role |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 103 | * |
| 104 | * Internal function. |
| 105 | */ |
Sirshak Das | 28aa539 | 2019-02-05 01:33:33 -0600 | [diff] [blame] | 106 | static inline void |
| 107 | f_load_head_tail_all_acq (svm_fifo_t * f, u32 * head, u32 * tail) |
| 108 | { |
| 109 | /* load-acq : consumer foreign index (paired with store-rel) */ |
| 110 | *tail = clib_atomic_load_acq_n (&f->tail); |
| 111 | /* load-acq : producer foriegn index (paired with store-rel) */ |
| 112 | *head = clib_atomic_load_acq_n (&f->head); |
| 113 | } |
| 114 | |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 115 | /** |
Florin Coras | 29a59c3 | 2019-05-02 12:52:19 -0700 | [diff] [blame] | 116 | * Fifo current size, i.e., number of bytes enqueued |
| 117 | * |
| 118 | * Internal function. |
| 119 | */ |
| 120 | static inline u32 |
| 121 | f_cursize (svm_fifo_t * f, u32 head, u32 tail) |
| 122 | { |
Florin Coras | f22f4e5 | 2019-12-19 16:10:58 -0800 | [diff] [blame] | 123 | return tail - head; |
Florin Coras | 29a59c3 | 2019-05-02 12:52:19 -0700 | [diff] [blame] | 124 | } |
| 125 | |
| 126 | /** |
| 127 | * Fifo free bytes, i.e., number of free bytes |
| 128 | * |
| 129 | * Internal function |
| 130 | */ |
| 131 | static inline u32 |
| 132 | f_free_count (svm_fifo_t * f, u32 head, u32 tail) |
| 133 | { |
Florin Coras | f22f4e5 | 2019-12-19 16:10:58 -0800 | [diff] [blame] | 134 | return (f->size - f_cursize (f, head, tail)); |
Florin Coras | 29a59c3 | 2019-05-02 12:52:19 -0700 | [diff] [blame] | 135 | } |
| 136 | |
Florin Coras | f22f4e5 | 2019-12-19 16:10:58 -0800 | [diff] [blame] | 137 | always_inline u32 |
| 138 | f_chunk_end (svm_fifo_chunk_t * c) |
| 139 | { |
| 140 | return c->start_byte + c->length; |
| 141 | } |
| 142 | |
| 143 | always_inline int |
| 144 | f_pos_lt (u32 a, u32 b) |
| 145 | { |
| 146 | return ((i32) (a - b) < 0); |
| 147 | } |
| 148 | |
| 149 | always_inline int |
| 150 | f_pos_leq (u32 a, u32 b) |
| 151 | { |
| 152 | return ((i32) (a - b) <= 0); |
| 153 | } |
| 154 | |
| 155 | always_inline int |
| 156 | f_pos_gt (u32 a, u32 b) |
| 157 | { |
| 158 | return ((i32) (a - b) > 0); |
| 159 | } |
| 160 | |
| 161 | always_inline int |
| 162 | f_pos_geq (u32 a, u32 b) |
| 163 | { |
| 164 | return ((i32) (a - b) >= 0); |
| 165 | } |
| 166 | |
| 167 | always_inline u8 |
| 168 | f_chunk_includes_pos (svm_fifo_chunk_t * c, u32 pos) |
| 169 | { |
| 170 | return (f_pos_geq (pos, c->start_byte) |
| 171 | && f_pos_lt (pos, c->start_byte + c->length)); |
| 172 | } |
Florin Coras | a7570b0 | 2019-05-02 12:52:19 -0700 | [diff] [blame] | 173 | |
| 174 | /** |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 175 | * Create fifo of requested size |
Florin Coras | 41c9e04 | 2018-09-11 00:10:41 -0700 | [diff] [blame] | 176 | * |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 177 | * Allocates fifo on current heap. |
| 178 | * |
| 179 | * @param size data size in bytes for fifo to be allocated. Will be |
| 180 | * rounded to the next highest power-of-two value. |
| 181 | * @return pointer to new fifo |
Florin Coras | 6792ec0 | 2017-03-13 03:49:51 -0700 | [diff] [blame] | 182 | */ |
Florin Coras | f22f4e5 | 2019-12-19 16:10:58 -0800 | [diff] [blame] | 183 | svm_fifo_t *svm_fifo_alloc (u32 size); |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 184 | /** |
| 185 | * Initialize fifo |
| 186 | * |
Florin Coras | eaacce4 | 2019-07-02 13:07:37 -0700 | [diff] [blame] | 187 | * @param f fifo |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 188 | * @param size size for fifo |
| 189 | */ |
Florin Coras | 0a84680 | 2019-04-09 18:29:14 -0700 | [diff] [blame] | 190 | void svm_fifo_init (svm_fifo_t * f, u32 size); |
Florin Coras | b095a3c | 2019-04-25 12:58:46 -0700 | [diff] [blame] | 191 | /** |
| 192 | * Allocate a fifo chunk on heap |
| 193 | * |
| 194 | * If the chunk is allocated on a fifo segment, this should be called |
| 195 | * with the segment's heap pushed. |
| 196 | * |
| 197 | * @param size chunk size in bytes. Will be rounded to the next highest |
| 198 | * power-of-two |
| 199 | * @return new chunk or 0 if alloc failed |
| 200 | */ |
| 201 | svm_fifo_chunk_t *svm_fifo_chunk_alloc (u32 size); |
Florin Coras | 4375fa3 | 2019-04-19 15:56:00 -0700 | [diff] [blame] | 202 | /** |
Florin Coras | 9e61d9a | 2020-02-05 21:13:18 +0000 | [diff] [blame] | 203 | * Ensure the whole fifo size is writeable |
Florin Coras | 4375fa3 | 2019-04-19 15:56:00 -0700 | [diff] [blame] | 204 | * |
Florin Coras | 9e61d9a | 2020-02-05 21:13:18 +0000 | [diff] [blame] | 205 | * Allocates enough chunks to cover the whole fifo size. |
Florin Coras | 4375fa3 | 2019-04-19 15:56:00 -0700 | [diff] [blame] | 206 | * |
Florin Coras | 9e61d9a | 2020-02-05 21:13:18 +0000 | [diff] [blame] | 207 | * @param f fifo |
Florin Coras | 4375fa3 | 2019-04-19 15:56:00 -0700 | [diff] [blame] | 208 | */ |
Florin Coras | f22f4e5 | 2019-12-19 16:10:58 -0800 | [diff] [blame] | 209 | int svm_fifo_fill_chunk_list (svm_fifo_t * f); |
Florin Coras | 9e61d9a | 2020-02-05 21:13:18 +0000 | [diff] [blame] | 210 | /** |
Florin Coras | 40a5da8 | 2020-12-18 09:19:18 -0800 | [diff] [blame^] | 211 | * Provision and return chunks for number of bytes requested |
| 212 | * |
| 213 | * Allocates enough chunks to cover the bytes requested and returns them |
| 214 | * in the fifo segment array. The number of bytes provisioned may be less |
| 215 | * than requested if not enough segments were provided. |
| 216 | * |
| 217 | * @param f fifo |
| 218 | * @param fs array of fifo segments |
| 219 | * @param n_segs length of fifo segments array |
| 220 | * @param len number of bytes to preallocate |
| 221 | * @return number of fifo segments provisioned or error |
| 222 | */ |
| 223 | int svm_fifo_provision_chunks (svm_fifo_t *f, svm_fifo_seg_t *fs, u32 n_segs, |
| 224 | u32 len); |
| 225 | /** |
Florin Coras | 9e61d9a | 2020-02-05 21:13:18 +0000 | [diff] [blame] | 226 | * Initialize rbtrees used for ooo lookups |
| 227 | * |
| 228 | * @param f fifo |
| 229 | * @param ooo_type type of ooo operation (0 enqueue, 1 dequeue) |
| 230 | */ |
Florin Coras | f22f4e5 | 2019-12-19 16:10:58 -0800 | [diff] [blame] | 231 | void svm_fifo_init_ooo_lookup (svm_fifo_t * f, u8 ooo_type); |
Florin Coras | a7570b0 | 2019-05-02 12:52:19 -0700 | [diff] [blame] | 232 | /** |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 233 | * Free fifo and associated state |
| 234 | * |
| 235 | * @param f fifo |
| 236 | */ |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 237 | void svm_fifo_free (svm_fifo_t * f); |
Florin Coras | b095a3c | 2019-04-25 12:58:46 -0700 | [diff] [blame] | 238 | /** |
| 239 | * Cleanup fifo chunk lookup rb tree |
| 240 | * |
| 241 | * The rb tree is allocated in segment heap so this should be called |
| 242 | * with it pushed. |
| 243 | * |
| 244 | * @param f fifo to cleanup |
| 245 | */ |
| 246 | void svm_fifo_free_chunk_lookup (svm_fifo_t * f); |
| 247 | /** |
| 248 | * Cleanup fifo ooo data |
| 249 | * |
| 250 | * The ooo data is allocated in producer process memory. The fifo |
| 251 | * segment heap should not be pushed. |
| 252 | * |
| 253 | * @param f fifo to cleanup |
| 254 | */ |
| 255 | void svm_fifo_free_ooo_data (svm_fifo_t * f); |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 256 | /** |
| 257 | * Init fifo head and tail |
| 258 | * |
| 259 | * @param f fifo |
| 260 | * @param head head value that will be matched to a chunk |
| 261 | * @param tail tail value that will be matched to a chunk |
| 262 | */ |
| 263 | void svm_fifo_init_pointers (svm_fifo_t * f, u32 head, u32 tail); |
| 264 | /** |
| 265 | * Clone fifo |
| 266 | * |
| 267 | * Clones single/default chunk fifo. It does not work for fifos with |
| 268 | * multiple chunks. |
| 269 | */ |
| 270 | void svm_fifo_clone (svm_fifo_t * df, svm_fifo_t * sf); |
| 271 | /** |
| 272 | * Enqueue data to fifo |
| 273 | * |
| 274 | * Data is enqueued and tail pointer is updated atomically. If the new data |
| 275 | * enqueued partly overlaps or "touches" an out-of-order segment, said segment |
| 276 | * is "consumed" and the number of bytes returned is appropriately updated. |
| 277 | * |
| 278 | * @param f fifo |
| 279 | * @param len length of data to copy |
| 280 | * @param src buffer from where to copy the data |
| 281 | * @return number of contiguous bytes that can be consumed or error |
| 282 | */ |
| 283 | int svm_fifo_enqueue (svm_fifo_t * f, u32 len, const u8 * src); |
| 284 | /** |
| 285 | * Enqueue data to fifo with offset |
| 286 | * |
| 287 | * Data is enqueued without updating tail pointer. Instead, an out-of-order |
| 288 | * list of segments is generated and maintained. Fifo takes care of coalescing |
| 289 | * contiguous or overlapping segments. |
| 290 | * |
| 291 | * @param f fifo |
| 292 | * @param offset offset at which to copy the data |
| 293 | * @param len len of data to copy |
| 294 | * @param src buffer from where to copy the data |
| 295 | * @return 0 if enqueue was successful, error otherwise |
| 296 | */ |
| 297 | int svm_fifo_enqueue_with_offset (svm_fifo_t * f, u32 offset, u32 len, |
| 298 | u8 * src); |
Florin Coras | a7570b0 | 2019-05-02 12:52:19 -0700 | [diff] [blame] | 299 | |
| 300 | /** |
| 301 | * Advance tail pointer |
| 302 | * |
| 303 | * Useful for moving tail pointer after external enqueue. |
| 304 | * |
| 305 | * @param f fifo |
| 306 | * @param len number of bytes to add to tail |
| 307 | */ |
| 308 | void svm_fifo_enqueue_nocopy (svm_fifo_t * f, u32 len); |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 309 | /** |
Florin Coras | c95cfa2 | 2020-11-24 08:41:17 -0800 | [diff] [blame] | 310 | * Enqueue array of @ref svm_fifo_seg_t in order |
| 311 | * |
| 312 | * @param f fifo |
| 313 | * @param segs array of segments to enqueue |
| 314 | * @param n_segs number of segments |
| 315 | * @param allow_partial if set partial enqueues are allowed |
| 316 | * @return len if enqueue was successful, error otherwise |
| 317 | */ |
| 318 | int svm_fifo_enqueue_segments (svm_fifo_t * f, const svm_fifo_seg_t segs[], |
| 319 | u32 n_segs, u8 allow_partial); |
| 320 | /** |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 321 | * Overwrite fifo head with new data |
| 322 | * |
| 323 | * This should be typically used by dgram transport protocols that need |
Florin Coras | 97d39e3 | 2020-09-04 08:57:27 -0700 | [diff] [blame] | 324 | * to update the dgram header after dequeuing a chunk of data. It assumes |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 325 | * that the dgram header is at most spread over two chunks. |
| 326 | * |
| 327 | * @param f fifo |
| 328 | * @param src src of new data |
| 329 | * @param len length of new data |
| 330 | */ |
| 331 | void svm_fifo_overwrite_head (svm_fifo_t * f, u8 * src, u32 len); |
| 332 | /** |
| 333 | * Dequeue data from fifo |
| 334 | * |
| 335 | * Data is dequeued to consumer provided buffer and head is atomically |
Florin Coras | 97d39e3 | 2020-09-04 08:57:27 -0700 | [diff] [blame] | 336 | * updated. This should not be used in combination with ooo lookups. If |
| 337 | * ooo peeking of data is needed in combination with dequeuing use @ref |
| 338 | * svm_fifo_dequeue_drop. |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 339 | * |
| 340 | * @param f fifo |
| 341 | * @param len length of data to dequeue |
| 342 | * @param dst buffer to where to dequeue the data |
| 343 | * @return number of bytes dequeued or error |
| 344 | */ |
| 345 | int svm_fifo_dequeue (svm_fifo_t * f, u32 len, u8 * dst); |
| 346 | /** |
| 347 | * Peek data from fifo |
| 348 | * |
| 349 | * Data is copied from requested offset into provided dst buffer. Head is |
| 350 | * not updated. |
| 351 | * |
| 352 | * @param f fifo |
| 353 | * @param offset offset from which to copy the data |
| 354 | * @param len length of data to copy |
| 355 | * @param dst buffer to where to dequeue the data |
| 356 | * @return number of bytes peeked |
| 357 | */ |
| 358 | int svm_fifo_peek (svm_fifo_t * f, u32 offset, u32 len, u8 * dst); |
| 359 | /** |
| 360 | * Dequeue and drop bytes from fifo |
| 361 | * |
| 362 | * Advances fifo head by requested amount of bytes. |
| 363 | * |
| 364 | * @param f fifo |
| 365 | * @param len number of bytes to drop |
| 366 | * @return number of bytes dropped |
| 367 | */ |
| 368 | int svm_fifo_dequeue_drop (svm_fifo_t * f, u32 len); |
| 369 | /** |
| 370 | * Dequeue and drop all bytes from fifo |
| 371 | * |
| 372 | * Advances head to tail position. |
| 373 | * |
| 374 | * @param f fifo |
| 375 | */ |
Florin Coras | 25579b4 | 2018-06-06 17:55:02 -0700 | [diff] [blame] | 376 | void svm_fifo_dequeue_drop_all (svm_fifo_t * f); |
Florin Coras | d68faf8 | 2020-09-25 15:18:13 -0700 | [diff] [blame] | 377 | /** |
| 378 | * Get pointers to fifo chunks data in @ref svm_fifo_seg_t array |
| 379 | * |
| 380 | * Populates fifo segment array with pointers to fifo chunk data and lengths. |
| 381 | * Because this returns pointers to data, it must be paired with |
| 382 | * @ref svm_fifo_dequeue_drop to actually release the fifo chunks after the |
| 383 | * data is consumed. |
| 384 | * |
| 385 | * @param f fifo |
Florin Coras | d1cc38d | 2020-10-11 11:05:04 -0700 | [diff] [blame] | 386 | * @param offset offset from where to retrieve segments |
Florin Coras | d68faf8 | 2020-09-25 15:18:13 -0700 | [diff] [blame] | 387 | * @param fs array of fifo segments allocated by caller |
| 388 | * @param n_segs number of fifo segments in array |
| 389 | * @param max_bytes max bytes to be mapped to fifo segments |
| 390 | * @return number of bytes in fifo segments or SVM_FIFO_EEMPTY |
| 391 | */ |
Florin Coras | d1cc38d | 2020-10-11 11:05:04 -0700 | [diff] [blame] | 392 | int svm_fifo_segments (svm_fifo_t * f, u32 offset, svm_fifo_seg_t * fs, |
| 393 | u32 n_segs, u32 max_bytes); |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 394 | /** |
| 395 | * Add io events subscriber to list |
| 396 | * |
| 397 | * @param f fifo |
| 398 | * @param sub subscriber opaque index (typically app worker index) |
| 399 | */ |
| 400 | void svm_fifo_add_subscriber (svm_fifo_t * f, u8 sub); |
| 401 | /** |
| 402 | * Remove io events subscriber form list |
| 403 | * |
| 404 | * @param f fifo |
| 405 | * @param sub subscriber index to be removed |
| 406 | */ |
Florin Coras | 72b0428 | 2019-01-14 17:23:11 -0800 | [diff] [blame] | 407 | void svm_fifo_del_subscriber (svm_fifo_t * f, u8 subscriber); |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 408 | /** |
| 409 | * Number of out-of-order segments for fifo |
| 410 | * |
| 411 | * @param f fifo |
| 412 | * @return number of out of order segments |
| 413 | */ |
| 414 | u32 svm_fifo_n_ooo_segments (svm_fifo_t * f); |
Florin Coras | eaacce4 | 2019-07-02 13:07:37 -0700 | [diff] [blame] | 415 | /** |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 416 | * First out-of-order segment for fifo |
| 417 | * |
| 418 | * @param f fifo |
| 419 | * @return first out-of-order segment for fifo |
| 420 | */ |
| 421 | ooo_segment_t *svm_fifo_first_ooo_segment (svm_fifo_t * f); |
Florin Coras | eaacce4 | 2019-07-02 13:07:37 -0700 | [diff] [blame] | 422 | /** |
| 423 | * Check if fifo is sane. Debug only. |
| 424 | * |
| 425 | * @param f fifo |
| 426 | * @return 1 if sane, 0 otherwise |
| 427 | */ |
| 428 | u8 svm_fifo_is_sane (svm_fifo_t * f); |
Florin Coras | 9e61d9a | 2020-02-05 21:13:18 +0000 | [diff] [blame] | 429 | /** |
| 430 | * Number of chunks linked into the fifo |
| 431 | * |
| 432 | * @param f fifo |
| 433 | * @return number of chunks in fifo linked list |
| 434 | */ |
Florin Coras | f22f4e5 | 2019-12-19 16:10:58 -0800 | [diff] [blame] | 435 | u32 svm_fifo_n_chunks (svm_fifo_t * f); |
Florin Coras | 6cf30ad | 2017-04-04 23:08:23 -0700 | [diff] [blame] | 436 | format_function_t format_svm_fifo; |
| 437 | |
Florin Coras | 58d36f0 | 2018-03-09 13:05:53 -0800 | [diff] [blame] | 438 | /** |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 439 | * Fifo max bytes to dequeue optimized for consumer |
| 440 | * |
| 441 | * @param f fifo |
| 442 | * @return max number of bytes that can be dequeued |
| 443 | */ |
| 444 | static inline u32 |
| 445 | svm_fifo_max_dequeue_cons (svm_fifo_t * f) |
| 446 | { |
| 447 | u32 tail, head; |
| 448 | f_load_head_tail_cons (f, &head, &tail); |
| 449 | return f_cursize (f, head, tail); |
| 450 | } |
| 451 | |
| 452 | /** |
| 453 | * Fifo max bytes to dequeue optimized for producer |
| 454 | * |
| 455 | * @param f fifo |
| 456 | * @return max number of bytes that can be dequeued |
| 457 | */ |
| 458 | static inline u32 |
| 459 | svm_fifo_max_dequeue_prod (svm_fifo_t * f) |
| 460 | { |
| 461 | u32 tail, head; |
| 462 | f_load_head_tail_prod (f, &head, &tail); |
| 463 | return f_cursize (f, head, tail); |
| 464 | } |
| 465 | |
| 466 | /** |
| 467 | * Fifo max bytes to dequeue |
| 468 | * |
| 469 | * Note: use producer or consumer specific functions for performance: |
| 470 | * @ref svm_fifo_max_dequeue_cons (svm_fifo_t *f) |
| 471 | * @ref svm_fifo_max_dequeue_prod (svm_fifo_t *f) |
| 472 | */ |
| 473 | static inline u32 |
| 474 | svm_fifo_max_dequeue (svm_fifo_t * f) |
| 475 | { |
| 476 | u32 tail, head; |
| 477 | f_load_head_tail_all_acq (f, &head, &tail); |
| 478 | return f_cursize (f, head, tail); |
| 479 | } |
| 480 | |
| 481 | /** |
| 482 | * Check if fifo is full optimized for producer |
| 483 | * |
| 484 | * @param f fifo |
| 485 | * @return 1 if fifo is full 0 otherwise |
| 486 | */ |
| 487 | static inline int |
| 488 | svm_fifo_is_full_prod (svm_fifo_t * f) |
| 489 | { |
Florin Coras | f22f4e5 | 2019-12-19 16:10:58 -0800 | [diff] [blame] | 490 | return (svm_fifo_max_dequeue_prod (f) == f->size); |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 491 | } |
| 492 | |
| 493 | /* Check if fifo is full. |
| 494 | * |
| 495 | * Note: use producer or consumer specific functions for performance. |
| 496 | * @ref svm_fifo_is_full_prod (svm_fifo_t * f) |
| 497 | * add cons version if needed |
| 498 | */ |
| 499 | static inline int |
| 500 | svm_fifo_is_full (svm_fifo_t * f) |
| 501 | { |
Florin Coras | f22f4e5 | 2019-12-19 16:10:58 -0800 | [diff] [blame] | 502 | return (svm_fifo_max_dequeue (f) == f->size); |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 503 | } |
| 504 | |
| 505 | /** |
| 506 | * Check if fifo is empty optimized for consumer |
| 507 | * |
| 508 | * @param f fifo |
| 509 | * @return 1 if fifo is empty 0 otherwise |
| 510 | */ |
| 511 | static inline int |
| 512 | svm_fifo_is_empty_cons (svm_fifo_t * f) |
| 513 | { |
| 514 | return (svm_fifo_max_dequeue_cons (f) == 0); |
| 515 | } |
| 516 | |
| 517 | /** |
| 518 | * Check if fifo is empty optimized for producer |
| 519 | * |
| 520 | * @param f fifo |
| 521 | * @return 1 if fifo is empty 0 otherwise |
| 522 | */ |
| 523 | static inline int |
| 524 | svm_fifo_is_empty_prod (svm_fifo_t * f) |
| 525 | { |
| 526 | return (svm_fifo_max_dequeue_prod (f) == 0); |
| 527 | } |
| 528 | |
| 529 | /** |
| 530 | * Check if fifo is empty |
| 531 | * |
| 532 | * Note: use producer or consumer specific functions for perfomance. |
| 533 | * @ref svm_fifo_is_empty_cons (svm_fifo_t * f) |
| 534 | * @ref svm_fifo_is_empty_prod (svm_fifo_t * f) |
| 535 | */ |
| 536 | static inline int |
| 537 | svm_fifo_is_empty (svm_fifo_t * f) |
| 538 | { |
| 539 | return (svm_fifo_max_dequeue (f) == 0); |
| 540 | } |
| 541 | |
| 542 | /** |
| 543 | * Check if fifo is wrapped |
| 544 | * |
| 545 | * @param f fifo |
| 546 | * @return 1 if 'normalized' head is ahead of tail |
| 547 | */ |
| 548 | static inline u8 |
| 549 | svm_fifo_is_wrapped (svm_fifo_t * f) |
| 550 | { |
| 551 | u32 head, tail; |
| 552 | f_load_head_tail_all_acq (f, &head, &tail); |
Florin Coras | 29a59c3 | 2019-05-02 12:52:19 -0700 | [diff] [blame] | 553 | return head > tail; |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 554 | } |
| 555 | |
| 556 | /** |
| 557 | * Maximum number of bytes that can be enqueued into fifo |
| 558 | * |
| 559 | * Optimized for producer |
| 560 | * |
| 561 | * @param f fifo |
| 562 | * @return max number of bytes that can be enqueued into fifo |
| 563 | */ |
| 564 | static inline u32 |
| 565 | svm_fifo_max_enqueue_prod (svm_fifo_t * f) |
| 566 | { |
| 567 | u32 head, tail; |
| 568 | f_load_head_tail_prod (f, &head, &tail); |
| 569 | return f_free_count (f, head, tail); |
| 570 | } |
| 571 | |
| 572 | /* Maximum number of bytes that can be enqueued into fifo |
| 573 | * |
| 574 | * Note: use producer or consumer specific functions for performance. |
| 575 | * @ref svm_fifo_max_enqueue_prod (svm_fifo_t *f) |
| 576 | * add consumer specific version if needed. |
| 577 | */ |
| 578 | static inline u32 |
| 579 | svm_fifo_max_enqueue (svm_fifo_t * f) |
| 580 | { |
| 581 | u32 head, tail; |
| 582 | f_load_head_tail_all_acq (f, &head, &tail); |
| 583 | return f_free_count (f, head, tail); |
| 584 | } |
| 585 | |
| 586 | /** |
Florin Coras | f22f4e5 | 2019-12-19 16:10:58 -0800 | [diff] [blame] | 587 | * Max contiguous chunk of data that can be read. |
| 588 | * |
| 589 | * Should only be called by consumers. |
Florin Coras | 58d36f0 | 2018-03-09 13:05:53 -0800 | [diff] [blame] | 590 | */ |
Florin Coras | f22f4e5 | 2019-12-19 16:10:58 -0800 | [diff] [blame] | 591 | u32 svm_fifo_max_read_chunk (svm_fifo_t * f); |
Florin Coras | 58d36f0 | 2018-03-09 13:05:53 -0800 | [diff] [blame] | 592 | |
| 593 | /** |
| 594 | * Max contiguous chunk of data that can be written |
Florin Coras | f22f4e5 | 2019-12-19 16:10:58 -0800 | [diff] [blame] | 595 | * |
| 596 | * Should only be called by producers |
Florin Coras | 58d36f0 | 2018-03-09 13:05:53 -0800 | [diff] [blame] | 597 | */ |
Florin Coras | f22f4e5 | 2019-12-19 16:10:58 -0800 | [diff] [blame] | 598 | u32 svm_fifo_max_write_chunk (svm_fifo_t * f); |
| 599 | |
Florin Coras | 9e61d9a | 2020-02-05 21:13:18 +0000 | [diff] [blame] | 600 | /** |
| 601 | * Fifo head chunk getter |
| 602 | * |
| 603 | * @param f fifo |
| 604 | * @return head chunk pointer |
| 605 | */ |
Florin Coras | f22f4e5 | 2019-12-19 16:10:58 -0800 | [diff] [blame] | 606 | static inline svm_fifo_chunk_t * |
| 607 | svm_fifo_head_chunk (svm_fifo_t * f) |
Florin Coras | 58d36f0 | 2018-03-09 13:05:53 -0800 | [diff] [blame] | 608 | { |
Florin Coras | f22f4e5 | 2019-12-19 16:10:58 -0800 | [diff] [blame] | 609 | return f->head_chunk; |
Florin Coras | 58d36f0 | 2018-03-09 13:05:53 -0800 | [diff] [blame] | 610 | } |
| 611 | |
Florin Coras | 9e61d9a | 2020-02-05 21:13:18 +0000 | [diff] [blame] | 612 | /** |
| 613 | * Fifo head pointer getter |
| 614 | * |
| 615 | * @param f fifo |
| 616 | * @return head pointer |
| 617 | */ |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 618 | static inline u8 * |
Florin Coras | 58d36f0 | 2018-03-09 13:05:53 -0800 | [diff] [blame] | 619 | svm_fifo_head (svm_fifo_t * f) |
| 620 | { |
Florin Coras | f22f4e5 | 2019-12-19 16:10:58 -0800 | [diff] [blame] | 621 | if (!f->head_chunk) |
| 622 | return 0; |
Sirshak Das | 28aa539 | 2019-02-05 01:33:33 -0600 | [diff] [blame] | 623 | /* load-relaxed: consumer owned index */ |
Florin Coras | 29a59c3 | 2019-05-02 12:52:19 -0700 | [diff] [blame] | 624 | return (f->head_chunk->data + (f->head - f->head_chunk->start_byte)); |
Florin Coras | 58d36f0 | 2018-03-09 13:05:53 -0800 | [diff] [blame] | 625 | } |
| 626 | |
Florin Coras | 9e61d9a | 2020-02-05 21:13:18 +0000 | [diff] [blame] | 627 | /** |
| 628 | * Fifo tail chunk getter |
| 629 | * |
| 630 | * @param f fifo |
| 631 | * @return tail chunk pointer |
| 632 | */ |
Florin Coras | f22f4e5 | 2019-12-19 16:10:58 -0800 | [diff] [blame] | 633 | static inline svm_fifo_chunk_t * |
| 634 | svm_fifo_tail_chunk (svm_fifo_t * f) |
| 635 | { |
| 636 | return f->tail_chunk; |
| 637 | } |
| 638 | |
Florin Coras | 9e61d9a | 2020-02-05 21:13:18 +0000 | [diff] [blame] | 639 | /** |
| 640 | * Fifo tail pointer getter |
| 641 | * |
| 642 | * @param f fifo |
| 643 | * @return tail pointer |
| 644 | */ |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 645 | static inline u8 * |
Florin Coras | 58d36f0 | 2018-03-09 13:05:53 -0800 | [diff] [blame] | 646 | svm_fifo_tail (svm_fifo_t * f) |
| 647 | { |
Sirshak Das | 28aa539 | 2019-02-05 01:33:33 -0600 | [diff] [blame] | 648 | /* load-relaxed: producer owned index */ |
Florin Coras | 29a59c3 | 2019-05-02 12:52:19 -0700 | [diff] [blame] | 649 | return (f->tail_chunk->data + (f->tail - f->tail_chunk->start_byte)); |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 650 | } |
| 651 | |
Florin Coras | 9e61d9a | 2020-02-05 21:13:18 +0000 | [diff] [blame] | 652 | /** |
| 653 | * Fifo number of subscribers getter |
| 654 | * |
| 655 | * @param f fifo |
| 656 | * @return number of subscribers |
| 657 | */ |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 658 | static inline u8 |
| 659 | svm_fifo_n_subscribers (svm_fifo_t * f) |
| 660 | { |
| 661 | return f->n_subscribers; |
| 662 | } |
| 663 | |
| 664 | /** |
| 665 | * Check if fifo has out-of-order data |
| 666 | * |
| 667 | * @param f fifo |
| 668 | * @return 1 if fifo has ooo data, 0 otherwise |
| 669 | */ |
| 670 | static inline u8 |
| 671 | svm_fifo_has_ooo_data (svm_fifo_t * f) |
| 672 | { |
| 673 | return f->ooos_list_head != OOO_SEGMENT_INVALID_INDEX; |
| 674 | } |
| 675 | |
| 676 | static inline ooo_segment_t * |
| 677 | svm_fifo_newest_ooo_segment (svm_fifo_t * f) |
| 678 | { |
| 679 | if (f->ooos_newest == OOO_SEGMENT_INVALID_INDEX) |
| 680 | return 0; |
| 681 | return pool_elt_at_index (f->ooo_segments, f->ooos_newest); |
| 682 | } |
| 683 | |
| 684 | static inline void |
| 685 | svm_fifo_newest_ooo_segment_reset (svm_fifo_t * f) |
| 686 | { |
| 687 | f->ooos_newest = OOO_SEGMENT_INVALID_INDEX; |
| 688 | } |
| 689 | |
| 690 | static inline u32 |
| 691 | ooo_segment_offset_prod (svm_fifo_t * f, ooo_segment_t * s) |
| 692 | { |
| 693 | u32 tail; |
| 694 | /* load-relaxed: producer owned index */ |
| 695 | tail = f->tail; |
| 696 | |
Florin Coras | f22f4e5 | 2019-12-19 16:10:58 -0800 | [diff] [blame] | 697 | return (s->start - tail); |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 698 | } |
| 699 | |
| 700 | static inline u32 |
| 701 | ooo_segment_length (svm_fifo_t * f, ooo_segment_t * s) |
| 702 | { |
| 703 | return s->length; |
| 704 | } |
| 705 | |
Florin Coras | f22f4e5 | 2019-12-19 16:10:58 -0800 | [diff] [blame] | 706 | static inline u32 |
| 707 | svm_fifo_size (svm_fifo_t * f) |
| 708 | { |
| 709 | return f->size; |
| 710 | } |
| 711 | |
| 712 | static inline void |
| 713 | svm_fifo_set_size (svm_fifo_t * f, u32 size) |
| 714 | { |
Florin Coras | 0e6199d | 2020-04-17 20:15:22 +0000 | [diff] [blame] | 715 | if (size > (1 << f->fs_hdr->max_log2_chunk_size)) |
| 716 | return; |
Florin Coras | 06d4775 | 2020-03-06 02:23:58 +0000 | [diff] [blame] | 717 | fsh_virtual_mem_update (f->fs_hdr, f->slice_index, (int) f->size - size); |
Florin Coras | f22f4e5 | 2019-12-19 16:10:58 -0800 | [diff] [blame] | 718 | f->size = size; |
| 719 | } |
| 720 | |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 721 | /** |
| 722 | * Check if fifo has io event |
| 723 | * |
| 724 | * @param f fifo |
| 725 | * @return 1 if fifo has event, 0 otherwise |
| 726 | */ |
| 727 | static inline int |
| 728 | svm_fifo_has_event (svm_fifo_t * f) |
| 729 | { |
| 730 | return f->has_event; |
| 731 | } |
| 732 | |
| 733 | /** |
| 734 | * Set fifo event flag. |
| 735 | * |
| 736 | * Forces release semantics. |
| 737 | * |
| 738 | * @param f fifo |
| 739 | * @return 1 if flag was not set, 0 otherwise |
| 740 | */ |
| 741 | always_inline u8 |
| 742 | svm_fifo_set_event (svm_fifo_t * f) |
| 743 | { |
| 744 | return !clib_atomic_swap_rel_n (&f->has_event, 1); |
| 745 | } |
| 746 | |
| 747 | /** |
| 748 | * Unset fifo event flag. |
| 749 | * |
| 750 | * Forces acquire semantics |
| 751 | * |
| 752 | * @param f fifo |
| 753 | */ |
| 754 | always_inline void |
| 755 | svm_fifo_unset_event (svm_fifo_t * f) |
| 756 | { |
| 757 | clib_atomic_swap_acq_n (&f->has_event, 0); |
| 758 | } |
| 759 | |
| 760 | /** |
Florin Coras | 2d379d8 | 2019-06-28 12:45:12 -0700 | [diff] [blame] | 761 | * Set specific want notification flag |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 762 | * |
Florin Coras | 2d379d8 | 2019-06-28 12:45:12 -0700 | [diff] [blame] | 763 | * For list of flags see @ref svm_fifo_deq_ntf_t |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 764 | * |
| 765 | * @param f fifo |
| 766 | * @param ntf_type type of notification requested |
| 767 | */ |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 768 | static inline void |
Florin Coras | 2d379d8 | 2019-06-28 12:45:12 -0700 | [diff] [blame] | 769 | svm_fifo_add_want_deq_ntf (svm_fifo_t * f, u8 ntf_type) |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 770 | { |
Florin Coras | 2d379d8 | 2019-06-28 12:45:12 -0700 | [diff] [blame] | 771 | f->want_deq_ntf |= ntf_type; |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 772 | } |
| 773 | |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 774 | /** |
Florin Coras | 2d379d8 | 2019-06-28 12:45:12 -0700 | [diff] [blame] | 775 | * Clear specific want notification flag |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 776 | * |
Florin Coras | 2d379d8 | 2019-06-28 12:45:12 -0700 | [diff] [blame] | 777 | * For list of flags see @ref svm_fifo_ntf_t |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 778 | * |
| 779 | * @param f fifo |
| 780 | * @param ntf_type type of notification to be cleared |
| 781 | */ |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 782 | static inline void |
Florin Coras | 2d379d8 | 2019-06-28 12:45:12 -0700 | [diff] [blame] | 783 | svm_fifo_del_want_deq_ntf (svm_fifo_t * f, u8 ntf_type) |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 784 | { |
Florin Coras | 2d379d8 | 2019-06-28 12:45:12 -0700 | [diff] [blame] | 785 | f->want_deq_ntf &= ~ntf_type; |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 786 | } |
| 787 | |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 788 | /** |
Florin Coras | 2d379d8 | 2019-06-28 12:45:12 -0700 | [diff] [blame] | 789 | * Clear the want notification flag and set has notification |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 790 | * |
Florin Coras | 2d379d8 | 2019-06-28 12:45:12 -0700 | [diff] [blame] | 791 | * Should be used after enqueuing an event. This clears the |
| 792 | * SVM_FIFO_WANT_NOTIF flag but it does not clear |
| 793 | * SVM_FIFO_WANT_NOTIF_IF_FULL. If the latter was set, has_ntf is |
| 794 | * set to avoid enqueueing events for for all dequeue operations until |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 795 | * it is manually cleared. |
| 796 | * |
| 797 | * @param f fifo |
| 798 | */ |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 799 | static inline void |
Florin Coras | 2d379d8 | 2019-06-28 12:45:12 -0700 | [diff] [blame] | 800 | svm_fifo_clear_deq_ntf (svm_fifo_t * f) |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 801 | { |
Florin Coras | 2d379d8 | 2019-06-28 12:45:12 -0700 | [diff] [blame] | 802 | /* Set the flag if want_notif_if_full was the only ntf requested */ |
| 803 | f->has_deq_ntf = f->want_deq_ntf == SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL; |
| 804 | svm_fifo_del_want_deq_ntf (f, SVM_FIFO_WANT_DEQ_NOTIF); |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 805 | } |
| 806 | |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 807 | /** |
Florin Coras | 2d379d8 | 2019-06-28 12:45:12 -0700 | [diff] [blame] | 808 | * Clear has notification flag |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 809 | * |
Florin Coras | 2d379d8 | 2019-06-28 12:45:12 -0700 | [diff] [blame] | 810 | * The fifo generates only one event per SVM_FIFO_WANT_NOTIF_IF_FULL |
| 811 | * request and sets has_ntf. To received new events the flag must be |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 812 | * cleared using this function. |
| 813 | * |
| 814 | * @param f fifo |
| 815 | */ |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 816 | static inline void |
Florin Coras | 2d379d8 | 2019-06-28 12:45:12 -0700 | [diff] [blame] | 817 | svm_fifo_reset_has_deq_ntf (svm_fifo_t * f) |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 818 | { |
Florin Coras | 2d379d8 | 2019-06-28 12:45:12 -0700 | [diff] [blame] | 819 | f->has_deq_ntf = 0; |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 820 | } |
| 821 | |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 822 | /** |
Florin Coras | 2d379d8 | 2019-06-28 12:45:12 -0700 | [diff] [blame] | 823 | * Check if fifo needs dequeue notification |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 824 | * |
Florin Coras | 2d379d8 | 2019-06-28 12:45:12 -0700 | [diff] [blame] | 825 | * Determines based on notification request flags and state of the fifo if |
| 826 | * an event should be generated. |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 827 | * |
| 828 | * @param f fifo |
| 829 | * @param n_last_deq number of bytes last dequeued |
Florin Coras | 2d379d8 | 2019-06-28 12:45:12 -0700 | [diff] [blame] | 830 | * @return 1 if event should be generated, 0 otherwise |
Florin Coras | 87b15ce | 2019-04-28 21:16:30 -0700 | [diff] [blame] | 831 | */ |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 832 | static inline u8 |
Florin Coras | 2d379d8 | 2019-06-28 12:45:12 -0700 | [diff] [blame] | 833 | svm_fifo_needs_deq_ntf (svm_fifo_t * f, u32 n_last_deq) |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 834 | { |
Florin Coras | 2d379d8 | 2019-06-28 12:45:12 -0700 | [diff] [blame] | 835 | u8 want_ntf = f->want_deq_ntf; |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 836 | |
Florin Coras | 2d379d8 | 2019-06-28 12:45:12 -0700 | [diff] [blame] | 837 | if (PREDICT_TRUE (want_ntf == SVM_FIFO_NO_DEQ_NOTIF)) |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 838 | return 0; |
Florin Coras | 2d379d8 | 2019-06-28 12:45:12 -0700 | [diff] [blame] | 839 | else if (want_ntf & SVM_FIFO_WANT_DEQ_NOTIF) |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 840 | return 1; |
Florin Coras | 2d379d8 | 2019-06-28 12:45:12 -0700 | [diff] [blame] | 841 | if (want_ntf & SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL) |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 842 | { |
Sirshak Das | 28aa539 | 2019-02-05 01:33:33 -0600 | [diff] [blame] | 843 | u32 max_deq = svm_fifo_max_dequeue_cons (f); |
Florin Coras | f22f4e5 | 2019-12-19 16:10:58 -0800 | [diff] [blame] | 844 | u32 size = f->size; |
| 845 | if (!f->has_deq_ntf && max_deq < size && max_deq + n_last_deq >= size) |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 846 | return 1; |
Nathan Skrzypczak | e82a7ad | 2019-05-22 18:41:50 +0200 | [diff] [blame] | 847 | } |
Florin Coras | 2d379d8 | 2019-06-28 12:45:12 -0700 | [diff] [blame] | 848 | if (want_ntf & SVM_FIFO_WANT_DEQ_NOTIF_IF_EMPTY) |
Nathan Skrzypczak | e82a7ad | 2019-05-22 18:41:50 +0200 | [diff] [blame] | 849 | { |
Florin Coras | 2d379d8 | 2019-06-28 12:45:12 -0700 | [diff] [blame] | 850 | if (!f->has_deq_ntf && svm_fifo_is_empty (f)) |
Nathan Skrzypczak | e82a7ad | 2019-05-22 18:41:50 +0200 | [diff] [blame] | 851 | return 1; |
Florin Coras | 1bcad5c | 2019-01-09 20:04:38 -0800 | [diff] [blame] | 852 | } |
| 853 | return 0; |
| 854 | } |
| 855 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 856 | #endif /* __included_ssvm_fifo_h__ */ |
| 857 | |
| 858 | /* |
| 859 | * fd.io coding-style-patch-verification: ON |
| 860 | * |
| 861 | * Local Variables: |
| 862 | * eval: (c-set-style "gnu") |
| 863 | * End: |
| 864 | */ |