blob: e7a1188d59157956ff7840dc1f15b56fcbd59484 [file] [log] [blame]
Dave Barach68b0fb02017-02-28 15:15:56 -05001/*
2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
Florin Coras6792ec02017-03-13 03:49:51 -070016#include <svm/svm_fifo.h>
Florin Corasf6359c82017-06-19 12:26:09 -040017#include <vppinfra/cpu.h>
Dave Barach68b0fb02017-02-28 15:15:56 -050018
Florin Corasf03a59a2017-06-09 21:07:32 -070019static inline u8
20position_lt (svm_fifo_t * f, u32 a, u32 b)
21{
Dave Barach2c25a622017-06-26 11:35:07 -040022 return (ooo_segment_distance_from_tail (f, a)
23 < ooo_segment_distance_from_tail (f, b));
Florin Corasf03a59a2017-06-09 21:07:32 -070024}
25
26static inline u8
27position_leq (svm_fifo_t * f, u32 a, u32 b)
28{
Dave Barach2c25a622017-06-26 11:35:07 -040029 return (ooo_segment_distance_from_tail (f, a)
30 <= ooo_segment_distance_from_tail (f, b));
Florin Corasf03a59a2017-06-09 21:07:32 -070031}
32
33static inline u8
34position_gt (svm_fifo_t * f, u32 a, u32 b)
35{
Dave Barach2c25a622017-06-26 11:35:07 -040036 return (ooo_segment_distance_from_tail (f, a)
37 > ooo_segment_distance_from_tail (f, b));
Florin Corasf03a59a2017-06-09 21:07:32 -070038}
39
40static inline u32
41position_diff (svm_fifo_t * f, u32 posa, u32 posb)
42{
Dave Barach2c25a622017-06-26 11:35:07 -040043 return ooo_segment_distance_from_tail (f, posa)
44 - ooo_segment_distance_from_tail (f, posb);
Florin Corasf03a59a2017-06-09 21:07:32 -070045}
46
47static inline u32
48ooo_segment_end_pos (svm_fifo_t * f, ooo_segment_t * s)
49{
50 return (s->start + s->length) % f->nitems;
51}
Dave Barach1f75cfd2017-04-14 16:46:44 -040052
Florin Coras3aa7af32018-06-29 08:44:31 -070053#ifndef CLIB_MARCH_VARIANT
54
Dave Barach1f75cfd2017-04-14 16:46:44 -040055u8 *
56format_ooo_segment (u8 * s, va_list * args)
57{
Florin Coras1f152cd2017-08-18 19:28:03 -070058 svm_fifo_t *f = va_arg (*args, svm_fifo_t *);
Dave Barach1f75cfd2017-04-14 16:46:44 -040059 ooo_segment_t *seg = va_arg (*args, ooo_segment_t *);
Florin Coras1f152cd2017-08-18 19:28:03 -070060 u32 normalized_start = (seg->start + f->nitems - f->tail) % f->nitems;
61 s = format (s, "[%u, %u], len %u, next %d, prev %d", normalized_start,
62 (normalized_start + seg->length) % f->nitems, seg->length,
63 seg->next, seg->prev);
Dave Barach1f75cfd2017-04-14 16:46:44 -040064 return s;
65}
66
67u8 *
Florin Coras3eb50622017-07-13 01:24:57 -040068svm_fifo_dump_trace (u8 * s, svm_fifo_t * f)
69{
70#if SVM_FIFO_TRACE
71 svm_fifo_trace_elem_t *seg = 0;
72 int i = 0;
73
74 if (f->trace)
75 {
76 vec_foreach (seg, f->trace)
77 {
78 s = format (s, "{%u, %u, %u}, ", seg->offset, seg->len, seg->action);
79 i++;
80 if (i % 5 == 0)
81 s = format (s, "\n");
82 }
83 s = format (s, "\n");
84 }
85 return s;
86#else
87 return 0;
88#endif
89}
90
91u8 *
92svm_fifo_replay (u8 * s, svm_fifo_t * f, u8 no_read, u8 verbose)
93{
94 int i, trace_len;
95 u8 *data = 0;
96 svm_fifo_trace_elem_t *trace;
97 u32 offset;
98 svm_fifo_t *dummy_fifo;
99
100 if (!f)
101 return s;
102
103#if SVM_FIFO_TRACE
104 trace = f->trace;
105 trace_len = vec_len (trace);
106#else
107 trace = 0;
108 trace_len = 0;
109#endif
110
111 dummy_fifo = svm_fifo_create (f->nitems);
Dave Barachb7b92992018-10-17 10:38:51 -0400112 clib_memset (f->data, 0xFF, f->nitems);
Florin Coras3eb50622017-07-13 01:24:57 -0400113
114 vec_validate (data, f->nitems);
115 for (i = 0; i < vec_len (data); i++)
116 data[i] = i;
117
118 for (i = 0; i < trace_len; i++)
119 {
120 offset = trace[i].offset;
121 if (trace[i].action == 1)
122 {
123 if (verbose)
124 s = format (s, "adding [%u, %u]:", trace[i].offset,
125 (trace[i].offset +
126 trace[i].len) % dummy_fifo->nitems);
127 svm_fifo_enqueue_with_offset (dummy_fifo, trace[i].offset,
128 trace[i].len, &data[offset]);
129 }
130 else if (trace[i].action == 2)
131 {
132 if (verbose)
133 s = format (s, "adding [%u, %u]:", 0, trace[i].len);
134 svm_fifo_enqueue_nowait (dummy_fifo, trace[i].len, &data[offset]);
135 }
136 else if (!no_read)
137 {
138 if (verbose)
139 s = format (s, "read: %u", trace[i].len);
140 svm_fifo_dequeue_drop (dummy_fifo, trace[i].len);
141 }
142 if (verbose)
143 s = format (s, "%U", format_svm_fifo, dummy_fifo, 1);
144 }
145
146 s = format (s, "result: %U", format_svm_fifo, dummy_fifo, 1);
147
148 return s;
149}
150
151u8 *
Dave Barach1f75cfd2017-04-14 16:46:44 -0400152format_ooo_list (u8 * s, va_list * args)
153{
154 svm_fifo_t *f = va_arg (*args, svm_fifo_t *);
Florin Corasde9a8492018-10-24 22:18:58 -0700155 u32 indent = va_arg (*args, u32);
Dave Barach1f75cfd2017-04-14 16:46:44 -0400156 u32 ooo_segment_index = f->ooos_list_head;
157 ooo_segment_t *seg;
158
159 while (ooo_segment_index != OOO_SEGMENT_INVALID_INDEX)
160 {
161 seg = pool_elt_at_index (f->ooo_segments, ooo_segment_index);
Florin Corasde9a8492018-10-24 22:18:58 -0700162 s = format (s, "%U%U\n", format_white_space, indent, format_ooo_segment,
163 f, seg);
Dave Barach1f75cfd2017-04-14 16:46:44 -0400164 ooo_segment_index = seg->next;
165 }
Florin Coras3eb50622017-07-13 01:24:57 -0400166
Dave Barach1f75cfd2017-04-14 16:46:44 -0400167 return s;
168}
169
Florin Corasb59a7052017-04-18 22:07:29 -0700170u8 *
171format_svm_fifo (u8 * s, va_list * args)
172{
173 svm_fifo_t *f = va_arg (*args, svm_fifo_t *);
174 int verbose = va_arg (*args, int);
Florin Corasde9a8492018-10-24 22:18:58 -0700175 u32 indent;
Florin Corasb59a7052017-04-18 22:07:29 -0700176
Florin Coras7fb0fe12018-04-09 09:24:52 -0700177 if (!s)
178 return s;
179
Florin Corasde9a8492018-10-24 22:18:58 -0700180 indent = format_get_indent (s);
Florin Corasb59a7052017-04-18 22:07:29 -0700181 s = format (s, "cursize %u nitems %u has_event %d\n",
182 f->cursize, f->nitems, f->has_event);
Florin Corasde9a8492018-10-24 22:18:58 -0700183 s = format (s, "%Uhead %d tail %d segment manager %u\n", format_white_space,
184 indent, f->head, f->tail, f->segment_manager);
Florin Corasb59a7052017-04-18 22:07:29 -0700185
186 if (verbose > 1)
Florin Corasde9a8492018-10-24 22:18:58 -0700187 s = format (s, "%Uvpp session %d thread %d app session %d thread %d\n",
188 format_white_space, indent, f->master_session_index,
189 f->master_thread_index, f->client_session_index,
190 f->client_thread_index);
Florin Corasb59a7052017-04-18 22:07:29 -0700191
192 if (verbose)
193 {
Florin Corasde9a8492018-10-24 22:18:58 -0700194 s = format (s, "%Uooo pool %d active elts newest %u\n",
195 format_white_space, indent, pool_elts (f->ooo_segments),
196 f->ooos_newest);
Florin Corasbb292f42017-05-19 09:49:19 -0700197 if (svm_fifo_has_ooo_data (f))
Florin Corasde9a8492018-10-24 22:18:58 -0700198 s = format (s, " %U", format_ooo_list, f, indent, verbose);
Florin Corasb59a7052017-04-18 22:07:29 -0700199 }
200 return s;
201}
202
Dave Barach68b0fb02017-02-28 15:15:56 -0500203/** create an svm fifo, in the current heap. Fails vs blow up the process */
204svm_fifo_t *
205svm_fifo_create (u32 data_size_in_bytes)
206{
207 svm_fifo_t *f;
Dave Barach818eb542017-08-02 13:56:13 -0400208 u32 rounded_data_size;
Dave Barach68b0fb02017-02-28 15:15:56 -0500209
Dave Barach818eb542017-08-02 13:56:13 -0400210 /* always round fifo data size to the next highest power-of-two */
211 rounded_data_size = (1 << (max_log2 (data_size_in_bytes)));
212 f = clib_mem_alloc_aligned_or_null (sizeof (*f) + rounded_data_size,
Dave Barach68b0fb02017-02-28 15:15:56 -0500213 CLIB_CACHE_LINE_BYTES);
214 if (f == 0)
215 return 0;
216
Dave Barachb7b92992018-10-17 10:38:51 -0400217 clib_memset (f, 0, sizeof (*f));
Dave Barach68b0fb02017-02-28 15:15:56 -0500218 f->nitems = data_size_in_bytes;
219 f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX;
Florin Coras326b81e2018-10-04 19:03:05 -0700220 f->ct_session_index = SVM_FIFO_INVALID_SESSION_INDEX;
Florin Corasfa76a762018-11-29 12:40:10 -0800221 f->segment_index = SVM_FIFO_INVALID_INDEX;
Dave Barach52851e62017-08-07 09:35:25 -0400222 f->refcnt = 1;
Dave Barach68b0fb02017-02-28 15:15:56 -0500223 return (f);
224}
225
Florin Coras6cf30ad2017-04-04 23:08:23 -0700226void
227svm_fifo_free (svm_fifo_t * f)
228{
Dave Barach52851e62017-08-07 09:35:25 -0400229 ASSERT (f->refcnt > 0);
230
231 if (--f->refcnt == 0)
232 {
233 pool_free (f->ooo_segments);
234 clib_mem_free (f);
235 }
Florin Coras6cf30ad2017-04-04 23:08:23 -0700236}
Florin Coras3aa7af32018-06-29 08:44:31 -0700237#endif
Florin Coras6cf30ad2017-04-04 23:08:23 -0700238
Dave Barach68b0fb02017-02-28 15:15:56 -0500239always_inline ooo_segment_t *
240ooo_segment_new (svm_fifo_t * f, u32 start, u32 length)
241{
242 ooo_segment_t *s;
243
244 pool_get (f->ooo_segments, s);
245
Dave Barach1f75cfd2017-04-14 16:46:44 -0400246 s->start = start;
Dave Barach68b0fb02017-02-28 15:15:56 -0500247 s->length = length;
248
249 s->prev = s->next = OOO_SEGMENT_INVALID_INDEX;
250
251 return s;
252}
253
254always_inline void
255ooo_segment_del (svm_fifo_t * f, u32 index)
256{
257 ooo_segment_t *cur, *prev = 0, *next = 0;
258 cur = pool_elt_at_index (f->ooo_segments, index);
259
260 if (cur->next != OOO_SEGMENT_INVALID_INDEX)
261 {
262 next = pool_elt_at_index (f->ooo_segments, cur->next);
263 next->prev = cur->prev;
264 }
265
266 if (cur->prev != OOO_SEGMENT_INVALID_INDEX)
267 {
268 prev = pool_elt_at_index (f->ooo_segments, cur->prev);
269 prev->next = cur->next;
270 }
271 else
272 {
273 f->ooos_list_head = cur->next;
274 }
275
276 pool_put (f->ooo_segments, cur);
277}
278
279/**
280 * Add segment to fifo's out-of-order segment list. Takes care of merging
281 * adjacent segments and removing overlapping ones.
282 */
283static void
284ooo_segment_add (svm_fifo_t * f, u32 offset, u32 length)
285{
286 ooo_segment_t *s, *new_s, *prev, *next, *it;
Florin Corasf03a59a2017-06-09 21:07:32 -0700287 u32 new_index, s_end_pos, s_index;
288 u32 normalized_position, normalized_end_position;
Dave Barach68b0fb02017-02-28 15:15:56 -0500289
Florin Coras3eb50622017-07-13 01:24:57 -0400290 ASSERT (offset + length <= ooo_segment_distance_from_tail (f, f->head));
Florin Corasf03a59a2017-06-09 21:07:32 -0700291 normalized_position = (f->tail + offset) % f->nitems;
292 normalized_end_position = (f->tail + offset + length) % f->nitems;
293
294 f->ooos_newest = OOO_SEGMENT_INVALID_INDEX;
Dave Barach68b0fb02017-02-28 15:15:56 -0500295
296 if (f->ooos_list_head == OOO_SEGMENT_INVALID_INDEX)
297 {
Florin Corasf03a59a2017-06-09 21:07:32 -0700298 s = ooo_segment_new (f, normalized_position, length);
Dave Barach68b0fb02017-02-28 15:15:56 -0500299 f->ooos_list_head = s - f->ooo_segments;
300 f->ooos_newest = f->ooos_list_head;
301 return;
302 }
303
304 /* Find first segment that starts after new segment */
305 s = pool_elt_at_index (f->ooo_segments, f->ooos_list_head);
306 while (s->next != OOO_SEGMENT_INVALID_INDEX
Florin Corasf03a59a2017-06-09 21:07:32 -0700307 && position_lt (f, s->start, normalized_position))
Dave Barach68b0fb02017-02-28 15:15:56 -0500308 s = pool_elt_at_index (f->ooo_segments, s->next);
309
Florin Corasc28764f2017-04-26 00:08:42 -0700310 /* If we have a previous and we overlap it, use it as starting point */
311 prev = ooo_segment_get_prev (f, s);
Florin Corasf03a59a2017-06-09 21:07:32 -0700312 if (prev
313 && position_leq (f, normalized_position, ooo_segment_end_pos (f, prev)))
Florin Corasc28764f2017-04-26 00:08:42 -0700314 {
315 s = prev;
Florin Corasf03a59a2017-06-09 21:07:32 -0700316 s_end_pos = ooo_segment_end_pos (f, s);
Dave Barach2c25a622017-06-26 11:35:07 -0400317
Florin Coras3eb50622017-07-13 01:24:57 -0400318 /* Since we have previous, normalized start position cannot be smaller
319 * than prev->start. Check tail */
320 ASSERT (position_lt (f, s->start, normalized_position));
Dave Barach2c25a622017-06-26 11:35:07 -0400321 goto check_tail;
Florin Corasc28764f2017-04-26 00:08:42 -0700322 }
323
Dave Barach68b0fb02017-02-28 15:15:56 -0500324 s_index = s - f->ooo_segments;
Florin Corasf03a59a2017-06-09 21:07:32 -0700325 s_end_pos = ooo_segment_end_pos (f, s);
Dave Barach68b0fb02017-02-28 15:15:56 -0500326
327 /* No overlap, add before current segment */
Florin Corasf03a59a2017-06-09 21:07:32 -0700328 if (position_lt (f, normalized_end_position, s->start))
Dave Barach68b0fb02017-02-28 15:15:56 -0500329 {
Florin Corasf03a59a2017-06-09 21:07:32 -0700330 new_s = ooo_segment_new (f, normalized_position, length);
Dave Barach68b0fb02017-02-28 15:15:56 -0500331 new_index = new_s - f->ooo_segments;
332
333 /* Pool might've moved, get segment again */
334 s = pool_elt_at_index (f->ooo_segments, s_index);
Dave Barach68b0fb02017-02-28 15:15:56 -0500335 if (s->prev != OOO_SEGMENT_INVALID_INDEX)
336 {
337 new_s->prev = s->prev;
Dave Barach68b0fb02017-02-28 15:15:56 -0500338 prev = pool_elt_at_index (f->ooo_segments, new_s->prev);
339 prev->next = new_index;
340 }
341 else
342 {
343 /* New head */
344 f->ooos_list_head = new_index;
345 }
346
Florin Corasf03a59a2017-06-09 21:07:32 -0700347 new_s->next = s_index;
Dave Barach68b0fb02017-02-28 15:15:56 -0500348 s->prev = new_index;
349 f->ooos_newest = new_index;
350 return;
351 }
352 /* No overlap, add after current segment */
Florin Corasf03a59a2017-06-09 21:07:32 -0700353 else if (position_gt (f, normalized_position, s_end_pos))
Dave Barach68b0fb02017-02-28 15:15:56 -0500354 {
Florin Corasf03a59a2017-06-09 21:07:32 -0700355 new_s = ooo_segment_new (f, normalized_position, length);
Dave Barach68b0fb02017-02-28 15:15:56 -0500356 new_index = new_s - f->ooo_segments;
357
358 /* Pool might've moved, get segment again */
359 s = pool_elt_at_index (f->ooo_segments, s_index);
360
Florin Coras3eb50622017-07-13 01:24:57 -0400361 /* Needs to be last */
Florin Corasf03a59a2017-06-09 21:07:32 -0700362 ASSERT (s->next == OOO_SEGMENT_INVALID_INDEX);
Dave Barach68b0fb02017-02-28 15:15:56 -0500363
Florin Corasf03a59a2017-06-09 21:07:32 -0700364 new_s->prev = s_index;
Dave Barach68b0fb02017-02-28 15:15:56 -0500365 s->next = new_index;
366 f->ooos_newest = new_index;
367
368 return;
369 }
370
371 /*
372 * Merge needed
373 */
374
375 /* Merge at head */
Florin Corasf03a59a2017-06-09 21:07:32 -0700376 if (position_lt (f, normalized_position, s->start))
Dave Barach68b0fb02017-02-28 15:15:56 -0500377 {
Florin Corasf03a59a2017-06-09 21:07:32 -0700378 s->start = normalized_position;
379 s->length = position_diff (f, s_end_pos, s->start);
Florin Coras3eb50622017-07-13 01:24:57 -0400380 f->ooos_newest = s - f->ooo_segments;
Dave Barach68b0fb02017-02-28 15:15:56 -0500381 }
382
Dave Barach2c25a622017-06-26 11:35:07 -0400383check_tail:
Florin Coras3eb50622017-07-13 01:24:57 -0400384
385 /* Overlapping tail */
Florin Corasf03a59a2017-06-09 21:07:32 -0700386 if (position_gt (f, normalized_end_position, s_end_pos))
Florin Corasc28764f2017-04-26 00:08:42 -0700387 {
Florin Coras3eb50622017-07-13 01:24:57 -0400388 s->length = position_diff (f, normalized_end_position, s->start);
389
390 /* Remove the completely overlapped segments in the tail */
391 it = ooo_segment_next (f, s);
Florin Corasf03a59a2017-06-09 21:07:32 -0700392 while (it && position_leq (f, ooo_segment_end_pos (f, it),
393 normalized_end_position))
Florin Corasc28764f2017-04-26 00:08:42 -0700394 {
Florin Coras3eb50622017-07-13 01:24:57 -0400395 next = ooo_segment_next (f, it);
Florin Corasc28764f2017-04-26 00:08:42 -0700396 ooo_segment_del (f, it - f->ooo_segments);
397 it = next;
398 }
399
400 /* If partial overlap with last, merge */
Florin Corasf03a59a2017-06-09 21:07:32 -0700401 if (it && position_leq (f, it->start, normalized_end_position))
Florin Corasc28764f2017-04-26 00:08:42 -0700402 {
Florin Coras3eb50622017-07-13 01:24:57 -0400403 s->length = position_diff (f, ooo_segment_end_pos (f, it),
404 s->start);
Florin Corasc28764f2017-04-26 00:08:42 -0700405 ooo_segment_del (f, it - f->ooo_segments);
406 }
Florin Coras3eb50622017-07-13 01:24:57 -0400407 f->ooos_newest = s - f->ooo_segments;
Florin Corasc28764f2017-04-26 00:08:42 -0700408 }
Dave Barach68b0fb02017-02-28 15:15:56 -0500409}
410
411/**
412 * Removes segments that can now be enqueued because the fifo's tail has
413 * advanced. Returns the number of bytes added to tail.
414 */
415static int
416ooo_segment_try_collect (svm_fifo_t * f, u32 n_bytes_enqueued)
417{
418 ooo_segment_t *s;
Florin Corasf03a59a2017-06-09 21:07:32 -0700419 u32 index, bytes = 0;
420 i32 diff;
Dave Barach68b0fb02017-02-28 15:15:56 -0500421
422 s = pool_elt_at_index (f->ooo_segments, f->ooos_list_head);
Dave Barach2c25a622017-06-26 11:35:07 -0400423 diff = ooo_segment_distance_to_tail (f, s->start);
Dave Barach68b0fb02017-02-28 15:15:56 -0500424
Dave Barach2c25a622017-06-26 11:35:07 -0400425 ASSERT (diff != n_bytes_enqueued);
Florin Corasc28764f2017-04-26 00:08:42 -0700426
Florin Corasf03a59a2017-06-09 21:07:32 -0700427 if (diff > n_bytes_enqueued)
Florin Corasb59a7052017-04-18 22:07:29 -0700428 return 0;
429
Dave Barach68b0fb02017-02-28 15:15:56 -0500430 /* If last tail update overlaps one/multiple ooo segments, remove them */
Florin Corasf03a59a2017-06-09 21:07:32 -0700431 while (0 <= diff && diff < n_bytes_enqueued)
Dave Barach68b0fb02017-02-28 15:15:56 -0500432 {
Florin Corasb59a7052017-04-18 22:07:29 -0700433 index = s - f->ooo_segments;
434
435 /* Segment end is beyond the tail. Advance tail and remove segment */
Florin Corasf03a59a2017-06-09 21:07:32 -0700436 if (s->length > diff)
Dave Barach68b0fb02017-02-28 15:15:56 -0500437 {
Florin Corasb59a7052017-04-18 22:07:29 -0700438 bytes = s->length - diff;
Florin Corasf03a59a2017-06-09 21:07:32 -0700439 f->tail += bytes;
440 f->tail %= f->nitems;
Florin Corasb59a7052017-04-18 22:07:29 -0700441 ooo_segment_del (f, index);
Dave Barach68b0fb02017-02-28 15:15:56 -0500442 break;
443 }
Florin Corasb59a7052017-04-18 22:07:29 -0700444
Dave Barach68b0fb02017-02-28 15:15:56 -0500445 /* If we have next go on */
Florin Corasb59a7052017-04-18 22:07:29 -0700446 if (s->next != OOO_SEGMENT_INVALID_INDEX)
Dave Barach68b0fb02017-02-28 15:15:56 -0500447 {
Dave Barach68b0fb02017-02-28 15:15:56 -0500448 s = pool_elt_at_index (f->ooo_segments, s->next);
Dave Barach2c25a622017-06-26 11:35:07 -0400449 diff = ooo_segment_distance_to_tail (f, s->start);
Dave Barach68b0fb02017-02-28 15:15:56 -0500450 ooo_segment_del (f, index);
451 }
452 /* End of search */
453 else
454 {
Florin Corasb59a7052017-04-18 22:07:29 -0700455 ooo_segment_del (f, index);
Dave Barach68b0fb02017-02-28 15:15:56 -0500456 break;
457 }
458 }
459
Chris Lukeab7b8d92017-09-07 07:40:13 -0400460 ASSERT (bytes <= f->nitems);
Dave Barach68b0fb02017-02-28 15:15:56 -0500461 return bytes;
462}
463
Florin Coras983cc7d2018-09-18 23:11:55 -0700464CLIB_MARCH_FN (svm_fifo_enqueue_nowait, int, svm_fifo_t * f, u32 max_bytes,
465 const u8 * copy_from_here)
Dave Barach68b0fb02017-02-28 15:15:56 -0500466{
467 u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
468 u32 cursize, nitems;
469
Florin Coras3e350af2017-03-30 02:54:28 -0700470 /* read cursize, which can only increase while we're working */
471 cursize = svm_fifo_max_dequeue (f);
Florin Corasf03a59a2017-06-09 21:07:32 -0700472 f->ooos_newest = OOO_SEGMENT_INVALID_INDEX;
Florin Coras3e350af2017-03-30 02:54:28 -0700473
474 if (PREDICT_FALSE (cursize == f->nitems))
Florin Coras7fb0fe12018-04-09 09:24:52 -0700475 return SVM_FIFO_FULL;
Dave Barach68b0fb02017-02-28 15:15:56 -0500476
Dave Barach68b0fb02017-02-28 15:15:56 -0500477 nitems = f->nitems;
478
479 /* Number of bytes we're going to copy */
480 total_copy_bytes = (nitems - cursize) < max_bytes ?
481 (nitems - cursize) : max_bytes;
482
483 if (PREDICT_TRUE (copy_from_here != 0))
484 {
485 /* Number of bytes in first copy segment */
486 first_copy_bytes = ((nitems - f->tail) < total_copy_bytes)
487 ? (nitems - f->tail) : total_copy_bytes;
488
Dave Barach178cf492018-11-13 16:34:13 -0500489 clib_memcpy_fast (&f->data[f->tail], copy_from_here, first_copy_bytes);
Dave Barach68b0fb02017-02-28 15:15:56 -0500490 f->tail += first_copy_bytes;
491 f->tail = (f->tail == nitems) ? 0 : f->tail;
492
493 /* Number of bytes in second copy segment, if any */
494 second_copy_bytes = total_copy_bytes - first_copy_bytes;
495 if (second_copy_bytes)
496 {
Dave Barach178cf492018-11-13 16:34:13 -0500497 clib_memcpy_fast (&f->data[f->tail],
498 copy_from_here + first_copy_bytes,
499 second_copy_bytes);
Dave Barach68b0fb02017-02-28 15:15:56 -0500500 f->tail += second_copy_bytes;
501 f->tail = (f->tail == nitems) ? 0 : f->tail;
502 }
503 }
504 else
505 {
Dave Barach2c25a622017-06-26 11:35:07 -0400506 ASSERT (0);
507
Dave Barach68b0fb02017-02-28 15:15:56 -0500508 /* Account for a zero-copy enqueue done elsewhere */
509 ASSERT (max_bytes <= (nitems - cursize));
510 f->tail += max_bytes;
511 f->tail = f->tail % nitems;
512 total_copy_bytes = max_bytes;
513 }
514
Florin Coras3eb50622017-07-13 01:24:57 -0400515 svm_fifo_trace_add (f, f->head, total_copy_bytes, 2);
516
Dave Barach68b0fb02017-02-28 15:15:56 -0500517 /* Any out-of-order segments to collect? */
518 if (PREDICT_FALSE (f->ooos_list_head != OOO_SEGMENT_INVALID_INDEX))
519 total_copy_bytes += ooo_segment_try_collect (f, total_copy_bytes);
520
521 /* Atomically increase the queue length */
Dave Barach2c25a622017-06-26 11:35:07 -0400522 ASSERT (cursize + total_copy_bytes <= nitems);
Sirshak Das19515ac2018-11-07 18:46:42 -0600523 clib_atomic_fetch_add_rel (&f->cursize, total_copy_bytes);
Dave Barach68b0fb02017-02-28 15:15:56 -0500524
525 return (total_copy_bytes);
526}
527
Florin Coras3aa7af32018-06-29 08:44:31 -0700528#ifndef CLIB_MARCH_VARIANT
Dave Barach68b0fb02017-02-28 15:15:56 -0500529int
Florin Coras371ca502018-02-21 12:07:41 -0800530svm_fifo_enqueue_nowait (svm_fifo_t * f, u32 max_bytes,
531 const u8 * copy_from_here)
Dave Barach68b0fb02017-02-28 15:15:56 -0500532{
Florin Coras983cc7d2018-09-18 23:11:55 -0700533 return CLIB_MARCH_FN_SELECT (svm_fifo_enqueue_nowait) (f, max_bytes,
534 copy_from_here);
Dave Barach68b0fb02017-02-28 15:15:56 -0500535}
Florin Coras3aa7af32018-06-29 08:44:31 -0700536#endif
Dave Barach68b0fb02017-02-28 15:15:56 -0500537
Florin Coras6792ec02017-03-13 03:49:51 -0700538/**
539 * Enqueue a future segment.
540 *
Dave Barach68b0fb02017-02-28 15:15:56 -0500541 * Two choices: either copies the entire segment, or copies nothing
542 * Returns 0 of the entire segment was copied
543 * Returns -1 if none of the segment was copied due to lack of space
544 */
Florin Coras983cc7d2018-09-18 23:11:55 -0700545CLIB_MARCH_FN (svm_fifo_enqueue_with_offset, int, svm_fifo_t * f,
546 u32 offset, u32 required_bytes, u8 * copy_from_here)
Dave Barach68b0fb02017-02-28 15:15:56 -0500547{
548 u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
Florin Corasf03a59a2017-06-09 21:07:32 -0700549 u32 cursize, nitems, normalized_offset;
Florin Corasf03a59a2017-06-09 21:07:32 -0700550
551 f->ooos_newest = OOO_SEGMENT_INVALID_INDEX;
Dave Barach68b0fb02017-02-28 15:15:56 -0500552
Florin Coras3e350af2017-03-30 02:54:28 -0700553 /* read cursize, which can only increase while we're working */
554 cursize = svm_fifo_max_dequeue (f);
Dave Barach68b0fb02017-02-28 15:15:56 -0500555 nitems = f->nitems;
Florin Corasf03a59a2017-06-09 21:07:32 -0700556
Dave Barach2c25a622017-06-26 11:35:07 -0400557 ASSERT (required_bytes < nitems);
558
Florin Corasf03a59a2017-06-09 21:07:32 -0700559 normalized_offset = (f->tail + offset) % nitems;
Dave Barach68b0fb02017-02-28 15:15:56 -0500560
561 /* Will this request fit? */
Florin Coras1f152cd2017-08-18 19:28:03 -0700562 if ((required_bytes + offset) > (nitems - cursize))
Dave Barach68b0fb02017-02-28 15:15:56 -0500563 return -1;
564
Florin Coras3eb50622017-07-13 01:24:57 -0400565 svm_fifo_trace_add (f, offset, required_bytes, 1);
566
Dave Barach68b0fb02017-02-28 15:15:56 -0500567 ooo_segment_add (f, offset, required_bytes);
568
569 /* Number of bytes we're going to copy */
570 total_copy_bytes = required_bytes;
Dave Barach68b0fb02017-02-28 15:15:56 -0500571
572 /* Number of bytes in first copy segment */
Dave Barach1f75cfd2017-04-14 16:46:44 -0400573 first_copy_bytes = ((nitems - normalized_offset) < total_copy_bytes)
574 ? (nitems - normalized_offset) : total_copy_bytes;
Dave Barach68b0fb02017-02-28 15:15:56 -0500575
Dave Barach178cf492018-11-13 16:34:13 -0500576 clib_memcpy_fast (&f->data[normalized_offset], copy_from_here,
577 first_copy_bytes);
Dave Barach68b0fb02017-02-28 15:15:56 -0500578
579 /* Number of bytes in second copy segment, if any */
580 second_copy_bytes = total_copy_bytes - first_copy_bytes;
581 if (second_copy_bytes)
582 {
Dave Barach1f75cfd2017-04-14 16:46:44 -0400583 normalized_offset += first_copy_bytes;
584 normalized_offset %= nitems;
Dave Barach68b0fb02017-02-28 15:15:56 -0500585
Dave Barach1f75cfd2017-04-14 16:46:44 -0400586 ASSERT (normalized_offset == 0);
Dave Barach68b0fb02017-02-28 15:15:56 -0500587
Dave Barach178cf492018-11-13 16:34:13 -0500588 clib_memcpy_fast (&f->data[normalized_offset],
589 copy_from_here + first_copy_bytes, second_copy_bytes);
Dave Barach68b0fb02017-02-28 15:15:56 -0500590 }
591
592 return (0);
593}
594
Florin Coras3aa7af32018-06-29 08:44:31 -0700595#ifndef CLIB_MARCH_VARIANT
Dave Barach68b0fb02017-02-28 15:15:56 -0500596
597int
Florin Coras3aa7af32018-06-29 08:44:31 -0700598svm_fifo_enqueue_with_offset (svm_fifo_t * f, u32 offset, u32 required_bytes,
599 u8 * copy_from_here)
Dave Barach68b0fb02017-02-28 15:15:56 -0500600{
Florin Coras983cc7d2018-09-18 23:11:55 -0700601 return CLIB_MARCH_FN_SELECT (svm_fifo_enqueue_with_offset) (f, offset,
602 required_bytes,
603 copy_from_here);
Dave Barach68b0fb02017-02-28 15:15:56 -0500604}
605
Florin Coras7fb0fe12018-04-09 09:24:52 -0700606void
607svm_fifo_overwrite_head (svm_fifo_t * f, u8 * data, u32 len)
608{
609 u32 first_chunk;
Florin Coras8e43d042018-05-04 15:46:57 -0700610 first_chunk = f->nitems - f->head;
Florin Coras7fb0fe12018-04-09 09:24:52 -0700611 ASSERT (len <= f->nitems);
Florin Coras8e43d042018-05-04 15:46:57 -0700612 if (len <= first_chunk)
Dave Barach178cf492018-11-13 16:34:13 -0500613 clib_memcpy_fast (&f->data[f->head], data, len);
Florin Coras7fb0fe12018-04-09 09:24:52 -0700614 else
615 {
Dave Barach178cf492018-11-13 16:34:13 -0500616 clib_memcpy_fast (&f->data[f->head], data, first_chunk);
617 clib_memcpy_fast (&f->data[0], data + first_chunk, len - first_chunk);
Florin Coras7fb0fe12018-04-09 09:24:52 -0700618 }
619}
Florin Coras3aa7af32018-06-29 08:44:31 -0700620#endif
Dave Barach68b0fb02017-02-28 15:15:56 -0500621
Florin Coras983cc7d2018-09-18 23:11:55 -0700622CLIB_MARCH_FN (svm_fifo_dequeue_nowait, int, svm_fifo_t * f, u32 max_bytes,
623 u8 * copy_here)
Dave Barach68b0fb02017-02-28 15:15:56 -0500624{
625 u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
626 u32 cursize, nitems;
627
Florin Coras3e350af2017-03-30 02:54:28 -0700628 /* read cursize, which can only increase while we're working */
629 cursize = svm_fifo_max_dequeue (f);
630 if (PREDICT_FALSE (cursize == 0))
Dave Barach68b0fb02017-02-28 15:15:56 -0500631 return -2; /* nothing in the fifo */
632
Dave Barach68b0fb02017-02-28 15:15:56 -0500633 nitems = f->nitems;
634
635 /* Number of bytes we're going to copy */
636 total_copy_bytes = (cursize < max_bytes) ? cursize : max_bytes;
637
638 if (PREDICT_TRUE (copy_here != 0))
639 {
640 /* Number of bytes in first copy segment */
641 first_copy_bytes = ((nitems - f->head) < total_copy_bytes)
642 ? (nitems - f->head) : total_copy_bytes;
Dave Barach178cf492018-11-13 16:34:13 -0500643 clib_memcpy_fast (copy_here, &f->data[f->head], first_copy_bytes);
Dave Barach68b0fb02017-02-28 15:15:56 -0500644 f->head += first_copy_bytes;
645 f->head = (f->head == nitems) ? 0 : f->head;
646
647 /* Number of bytes in second copy segment, if any */
648 second_copy_bytes = total_copy_bytes - first_copy_bytes;
649 if (second_copy_bytes)
650 {
Dave Barach178cf492018-11-13 16:34:13 -0500651 clib_memcpy_fast (copy_here + first_copy_bytes,
652 &f->data[f->head], second_copy_bytes);
Dave Barach68b0fb02017-02-28 15:15:56 -0500653 f->head += second_copy_bytes;
654 f->head = (f->head == nitems) ? 0 : f->head;
655 }
656 }
657 else
658 {
Dave Barach2c25a622017-06-26 11:35:07 -0400659 ASSERT (0);
Dave Barach68b0fb02017-02-28 15:15:56 -0500660 /* Account for a zero-copy dequeue done elsewhere */
661 ASSERT (max_bytes <= cursize);
662 f->head += max_bytes;
663 f->head = f->head % nitems;
664 cursize -= max_bytes;
665 total_copy_bytes = max_bytes;
666 }
667
Dave Barach2c25a622017-06-26 11:35:07 -0400668 ASSERT (f->head <= nitems);
669 ASSERT (cursize >= total_copy_bytes);
Sirshak Das19515ac2018-11-07 18:46:42 -0600670 clib_atomic_fetch_sub_rel (&f->cursize, total_copy_bytes);
Dave Barach68b0fb02017-02-28 15:15:56 -0500671
672 return (total_copy_bytes);
673}
674
Florin Coras3aa7af32018-06-29 08:44:31 -0700675#ifndef CLIB_MARCH_VARIANT
Florin Corasf6359c82017-06-19 12:26:09 -0400676
Dave Barach68b0fb02017-02-28 15:15:56 -0500677int
Florin Corasf6359c82017-06-19 12:26:09 -0400678svm_fifo_dequeue_nowait (svm_fifo_t * f, u32 max_bytes, u8 * copy_here)
679{
Florin Coras983cc7d2018-09-18 23:11:55 -0700680 return CLIB_MARCH_FN_SELECT (svm_fifo_dequeue_nowait) (f, max_bytes,
681 copy_here);
Florin Corasf6359c82017-06-19 12:26:09 -0400682}
Florin Coras3aa7af32018-06-29 08:44:31 -0700683#endif
Florin Corasf6359c82017-06-19 12:26:09 -0400684
Florin Coras983cc7d2018-09-18 23:11:55 -0700685CLIB_MARCH_FN (svm_fifo_peek, int, svm_fifo_t * f, u32 relative_offset,
686 u32 max_bytes, u8 * copy_here)
Dave Barach68b0fb02017-02-28 15:15:56 -0500687{
688 u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
Florin Coras6792ec02017-03-13 03:49:51 -0700689 u32 cursize, nitems, real_head;
Dave Barach68b0fb02017-02-28 15:15:56 -0500690
Florin Coras3e350af2017-03-30 02:54:28 -0700691 /* read cursize, which can only increase while we're working */
692 cursize = svm_fifo_max_dequeue (f);
Florin Coras93992a92017-05-24 18:03:56 -0700693 if (PREDICT_FALSE (cursize < relative_offset))
Dave Barach68b0fb02017-02-28 15:15:56 -0500694 return -2; /* nothing in the fifo */
695
Dave Barach68b0fb02017-02-28 15:15:56 -0500696 nitems = f->nitems;
Florin Corasb59a7052017-04-18 22:07:29 -0700697 real_head = f->head + relative_offset;
Florin Coras6792ec02017-03-13 03:49:51 -0700698 real_head = real_head >= nitems ? real_head - nitems : real_head;
Dave Barach68b0fb02017-02-28 15:15:56 -0500699
700 /* Number of bytes we're going to copy */
Florin Coras93992a92017-05-24 18:03:56 -0700701 total_copy_bytes = (cursize - relative_offset < max_bytes) ?
702 cursize - relative_offset : max_bytes;
Dave Barach68b0fb02017-02-28 15:15:56 -0500703
704 if (PREDICT_TRUE (copy_here != 0))
705 {
706 /* Number of bytes in first copy segment */
707 first_copy_bytes =
Florin Coras6792ec02017-03-13 03:49:51 -0700708 ((nitems - real_head) < total_copy_bytes) ?
709 (nitems - real_head) : total_copy_bytes;
Dave Barach178cf492018-11-13 16:34:13 -0500710 clib_memcpy_fast (copy_here, &f->data[real_head], first_copy_bytes);
Dave Barach68b0fb02017-02-28 15:15:56 -0500711
712 /* Number of bytes in second copy segment, if any */
713 second_copy_bytes = total_copy_bytes - first_copy_bytes;
714 if (second_copy_bytes)
715 {
Dave Barach178cf492018-11-13 16:34:13 -0500716 clib_memcpy_fast (copy_here + first_copy_bytes, &f->data[0],
717 second_copy_bytes);
Dave Barach68b0fb02017-02-28 15:15:56 -0500718 }
719 }
720 return total_copy_bytes;
721}
722
Florin Coras3aa7af32018-06-29 08:44:31 -0700723#ifndef CLIB_MARCH_VARIANT
Florin Corasf6359c82017-06-19 12:26:09 -0400724
725int
726svm_fifo_peek (svm_fifo_t * f, u32 relative_offset, u32 max_bytes,
727 u8 * copy_here)
728{
Florin Coras983cc7d2018-09-18 23:11:55 -0700729 return CLIB_MARCH_FN_SELECT (svm_fifo_peek) (f, relative_offset, max_bytes,
730 copy_here);
Florin Corasf6359c82017-06-19 12:26:09 -0400731}
732
Dave Barach68b0fb02017-02-28 15:15:56 -0500733int
Florin Corasa5464812017-04-19 13:00:05 -0700734svm_fifo_dequeue_drop (svm_fifo_t * f, u32 max_bytes)
Dave Barach68b0fb02017-02-28 15:15:56 -0500735{
736 u32 total_drop_bytes, first_drop_bytes, second_drop_bytes;
737 u32 cursize, nitems;
738
Florin Coras3e350af2017-03-30 02:54:28 -0700739 /* read cursize, which can only increase while we're working */
740 cursize = svm_fifo_max_dequeue (f);
741 if (PREDICT_FALSE (cursize == 0))
Dave Barach68b0fb02017-02-28 15:15:56 -0500742 return -2; /* nothing in the fifo */
743
Dave Barach68b0fb02017-02-28 15:15:56 -0500744 nitems = f->nitems;
745
746 /* Number of bytes we're going to drop */
747 total_drop_bytes = (cursize < max_bytes) ? cursize : max_bytes;
748
Florin Coras3eb50622017-07-13 01:24:57 -0400749 svm_fifo_trace_add (f, f->tail, total_drop_bytes, 3);
750
Dave Barach68b0fb02017-02-28 15:15:56 -0500751 /* Number of bytes in first copy segment */
752 first_drop_bytes =
753 ((nitems - f->head) < total_drop_bytes) ?
754 (nitems - f->head) : total_drop_bytes;
755 f->head += first_drop_bytes;
756 f->head = (f->head == nitems) ? 0 : f->head;
757
758 /* Number of bytes in second drop segment, if any */
759 second_drop_bytes = total_drop_bytes - first_drop_bytes;
760 if (second_drop_bytes)
761 {
762 f->head += second_drop_bytes;
763 f->head = (f->head == nitems) ? 0 : f->head;
764 }
765
Dave Barach2c25a622017-06-26 11:35:07 -0400766 ASSERT (f->head <= nitems);
767 ASSERT (cursize >= total_drop_bytes);
Sirshak Das19515ac2018-11-07 18:46:42 -0600768 clib_atomic_fetch_sub_rel (&f->cursize, total_drop_bytes);
Dave Barach68b0fb02017-02-28 15:15:56 -0500769
770 return total_drop_bytes;
771}
772
Florin Coras25579b42018-06-06 17:55:02 -0700773void
774svm_fifo_dequeue_drop_all (svm_fifo_t * f)
775{
776 f->head = f->tail;
Sirshak Das19515ac2018-11-07 18:46:42 -0600777 clib_atomic_fetch_sub_rel (&f->cursize, f->cursize);
Florin Coras25579b42018-06-06 17:55:02 -0700778}
779
Florin Coras2cba8532018-09-11 16:33:36 -0700780int
781svm_fifo_segments (svm_fifo_t * f, svm_fifo_segment_t * fs)
782{
783 u32 cursize, nitems;
784
785 /* read cursize, which can only increase while we're working */
786 cursize = svm_fifo_max_dequeue (f);
787 if (PREDICT_FALSE (cursize == 0))
788 return -2;
789
790 nitems = f->nitems;
791
792 fs[0].len = ((nitems - f->head) < cursize) ? (nitems - f->head) : cursize;
793 fs[0].data = f->data + f->head;
794
795 if (fs[0].len < cursize)
796 {
797 fs[1].len = cursize - fs[0].len;
798 fs[1].data = f->data;
799 }
800 else
801 {
802 fs[1].len = 0;
803 fs[1].data = 0;
804 }
805 return cursize;
806}
807
808void
809svm_fifo_segments_free (svm_fifo_t * f, svm_fifo_segment_t * fs)
810{
811 u32 total_drop_bytes;
812
813 ASSERT (fs[0].data == f->data + f->head);
814 if (fs[1].len)
815 {
816 f->head = fs[1].len;
817 total_drop_bytes = fs[0].len + fs[1].len;
818 }
819 else
820 {
821 f->head = (f->head + fs[0].len) % f->nitems;
822 total_drop_bytes = fs[0].len;
823 }
Sirshak Das19515ac2018-11-07 18:46:42 -0600824 clib_atomic_fetch_sub_rel (&f->cursize, total_drop_bytes);
Florin Coras2cba8532018-09-11 16:33:36 -0700825}
826
Dave Barach1f75cfd2017-04-14 16:46:44 -0400827u32
828svm_fifo_number_ooo_segments (svm_fifo_t * f)
829{
830 return pool_elts (f->ooo_segments);
831}
832
833ooo_segment_t *
834svm_fifo_first_ooo_segment (svm_fifo_t * f)
835{
836 return pool_elt_at_index (f->ooo_segments, f->ooos_list_head);
837}
838
Florin Corasc28764f2017-04-26 00:08:42 -0700839/**
840 * Set fifo pointers to requested offset
841 */
842void
843svm_fifo_init_pointers (svm_fifo_t * f, u32 pointer)
844{
845 f->head = f->tail = pointer % f->nitems;
846}
847
Florin Coras3aa7af32018-06-29 08:44:31 -0700848#endif
Dave Barach68b0fb02017-02-28 15:15:56 -0500849/*
850 * fd.io coding-style-patch-verification: ON
851 *
852 * Local Variables:
853 * eval: (c-set-style "gnu")
854 * End:
855 */