blob: 10c319236eeadc652be5a117137c8ddd527c09dd [file] [log] [blame]
Dave Barach68b0fb02017-02-28 15:15:56 -05001/*
2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
Florin Coras6792ec02017-03-13 03:49:51 -070016#include <svm/svm_fifo.h>
Florin Corasf6359c82017-06-19 12:26:09 -040017#include <vppinfra/cpu.h>
Dave Barach68b0fb02017-02-28 15:15:56 -050018
Florin Corasf03a59a2017-06-09 21:07:32 -070019static inline u8
20position_lt (svm_fifo_t * f, u32 a, u32 b)
21{
Dave Barach2c25a622017-06-26 11:35:07 -040022 return (ooo_segment_distance_from_tail (f, a)
23 < ooo_segment_distance_from_tail (f, b));
Florin Corasf03a59a2017-06-09 21:07:32 -070024}
25
26static inline u8
27position_leq (svm_fifo_t * f, u32 a, u32 b)
28{
Dave Barach2c25a622017-06-26 11:35:07 -040029 return (ooo_segment_distance_from_tail (f, a)
30 <= ooo_segment_distance_from_tail (f, b));
Florin Corasf03a59a2017-06-09 21:07:32 -070031}
32
33static inline u8
34position_gt (svm_fifo_t * f, u32 a, u32 b)
35{
Dave Barach2c25a622017-06-26 11:35:07 -040036 return (ooo_segment_distance_from_tail (f, a)
37 > ooo_segment_distance_from_tail (f, b));
Florin Corasf03a59a2017-06-09 21:07:32 -070038}
39
40static inline u32
41position_diff (svm_fifo_t * f, u32 posa, u32 posb)
42{
Dave Barach2c25a622017-06-26 11:35:07 -040043 return ooo_segment_distance_from_tail (f, posa)
44 - ooo_segment_distance_from_tail (f, posb);
Florin Corasf03a59a2017-06-09 21:07:32 -070045}
46
47static inline u32
48ooo_segment_end_pos (svm_fifo_t * f, ooo_segment_t * s)
49{
50 return (s->start + s->length) % f->nitems;
51}
Dave Barach1f75cfd2017-04-14 16:46:44 -040052
53u8 *
54format_ooo_segment (u8 * s, va_list * args)
55{
Florin Coras1f152cd2017-08-18 19:28:03 -070056 svm_fifo_t *f = va_arg (*args, svm_fifo_t *);
Dave Barach1f75cfd2017-04-14 16:46:44 -040057 ooo_segment_t *seg = va_arg (*args, ooo_segment_t *);
Florin Coras1f152cd2017-08-18 19:28:03 -070058 u32 normalized_start = (seg->start + f->nitems - f->tail) % f->nitems;
59 s = format (s, "[%u, %u], len %u, next %d, prev %d", normalized_start,
60 (normalized_start + seg->length) % f->nitems, seg->length,
61 seg->next, seg->prev);
Dave Barach1f75cfd2017-04-14 16:46:44 -040062 return s;
63}
64
65u8 *
Florin Coras3eb50622017-07-13 01:24:57 -040066svm_fifo_dump_trace (u8 * s, svm_fifo_t * f)
67{
68#if SVM_FIFO_TRACE
69 svm_fifo_trace_elem_t *seg = 0;
70 int i = 0;
71
72 if (f->trace)
73 {
74 vec_foreach (seg, f->trace)
75 {
76 s = format (s, "{%u, %u, %u}, ", seg->offset, seg->len, seg->action);
77 i++;
78 if (i % 5 == 0)
79 s = format (s, "\n");
80 }
81 s = format (s, "\n");
82 }
83 return s;
84#else
85 return 0;
86#endif
87}
88
89u8 *
90svm_fifo_replay (u8 * s, svm_fifo_t * f, u8 no_read, u8 verbose)
91{
92 int i, trace_len;
93 u8 *data = 0;
94 svm_fifo_trace_elem_t *trace;
95 u32 offset;
96 svm_fifo_t *dummy_fifo;
97
98 if (!f)
99 return s;
100
101#if SVM_FIFO_TRACE
102 trace = f->trace;
103 trace_len = vec_len (trace);
104#else
105 trace = 0;
106 trace_len = 0;
107#endif
108
109 dummy_fifo = svm_fifo_create (f->nitems);
110 memset (f->data, 0xFF, f->nitems);
111
112 vec_validate (data, f->nitems);
113 for (i = 0; i < vec_len (data); i++)
114 data[i] = i;
115
116 for (i = 0; i < trace_len; i++)
117 {
118 offset = trace[i].offset;
119 if (trace[i].action == 1)
120 {
121 if (verbose)
122 s = format (s, "adding [%u, %u]:", trace[i].offset,
123 (trace[i].offset +
124 trace[i].len) % dummy_fifo->nitems);
125 svm_fifo_enqueue_with_offset (dummy_fifo, trace[i].offset,
126 trace[i].len, &data[offset]);
127 }
128 else if (trace[i].action == 2)
129 {
130 if (verbose)
131 s = format (s, "adding [%u, %u]:", 0, trace[i].len);
132 svm_fifo_enqueue_nowait (dummy_fifo, trace[i].len, &data[offset]);
133 }
134 else if (!no_read)
135 {
136 if (verbose)
137 s = format (s, "read: %u", trace[i].len);
138 svm_fifo_dequeue_drop (dummy_fifo, trace[i].len);
139 }
140 if (verbose)
141 s = format (s, "%U", format_svm_fifo, dummy_fifo, 1);
142 }
143
144 s = format (s, "result: %U", format_svm_fifo, dummy_fifo, 1);
145
146 return s;
147}
148
149u8 *
Dave Barach1f75cfd2017-04-14 16:46:44 -0400150format_ooo_list (u8 * s, va_list * args)
151{
152 svm_fifo_t *f = va_arg (*args, svm_fifo_t *);
153 u32 ooo_segment_index = f->ooos_list_head;
154 ooo_segment_t *seg;
155
156 while (ooo_segment_index != OOO_SEGMENT_INVALID_INDEX)
157 {
158 seg = pool_elt_at_index (f->ooo_segments, ooo_segment_index);
Florin Coras1f152cd2017-08-18 19:28:03 -0700159 s = format (s, " %U\n", format_ooo_segment, f, seg);
Dave Barach1f75cfd2017-04-14 16:46:44 -0400160 ooo_segment_index = seg->next;
161 }
Florin Coras3eb50622017-07-13 01:24:57 -0400162
Dave Barach1f75cfd2017-04-14 16:46:44 -0400163 return s;
164}
165
Florin Corasb59a7052017-04-18 22:07:29 -0700166u8 *
167format_svm_fifo (u8 * s, va_list * args)
168{
169 svm_fifo_t *f = va_arg (*args, svm_fifo_t *);
170 int verbose = va_arg (*args, int);
171
Florin Coras7fb0fe12018-04-09 09:24:52 -0700172 if (!s)
173 return s;
174
Florin Corasb59a7052017-04-18 22:07:29 -0700175 s = format (s, "cursize %u nitems %u has_event %d\n",
176 f->cursize, f->nitems, f->has_event);
Florin Corasb2371c22018-05-31 17:14:10 -0700177 s = format (s, " head %d tail %d segment manager %u\n", f->head, f->tail,
178 f->segment_manager);
Florin Corasb59a7052017-04-18 22:07:29 -0700179
180 if (verbose > 1)
181 s = format
Florin Corasbb292f42017-05-19 09:49:19 -0700182 (s, " server session %d thread %d client session %d thread %d\n",
Florin Corasa5464812017-04-19 13:00:05 -0700183 f->master_session_index, f->master_thread_index,
Florin Corasb59a7052017-04-18 22:07:29 -0700184 f->client_session_index, f->client_thread_index);
185
186 if (verbose)
187 {
Florin Coras3eb50622017-07-13 01:24:57 -0400188 s = format (s, " ooo pool %d active elts newest %u\n",
189 pool_elts (f->ooo_segments), f->ooos_newest);
Florin Corasbb292f42017-05-19 09:49:19 -0700190 if (svm_fifo_has_ooo_data (f))
Florin Coras3eb50622017-07-13 01:24:57 -0400191 s = format (s, " %U", format_ooo_list, f, verbose);
Florin Corasb59a7052017-04-18 22:07:29 -0700192 }
193 return s;
194}
195
Dave Barach68b0fb02017-02-28 15:15:56 -0500196/** create an svm fifo, in the current heap. Fails vs blow up the process */
197svm_fifo_t *
198svm_fifo_create (u32 data_size_in_bytes)
199{
200 svm_fifo_t *f;
Dave Barach818eb542017-08-02 13:56:13 -0400201 u32 rounded_data_size;
Dave Barach68b0fb02017-02-28 15:15:56 -0500202
Dave Barach818eb542017-08-02 13:56:13 -0400203 /* always round fifo data size to the next highest power-of-two */
204 rounded_data_size = (1 << (max_log2 (data_size_in_bytes)));
205 f = clib_mem_alloc_aligned_or_null (sizeof (*f) + rounded_data_size,
Dave Barach68b0fb02017-02-28 15:15:56 -0500206 CLIB_CACHE_LINE_BYTES);
207 if (f == 0)
208 return 0;
209
Dave Barach2c25a622017-06-26 11:35:07 -0400210 memset (f, 0, sizeof (*f));
Dave Barach68b0fb02017-02-28 15:15:56 -0500211 f->nitems = data_size_in_bytes;
212 f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX;
Dave Barach52851e62017-08-07 09:35:25 -0400213 f->refcnt = 1;
Dave Barach68b0fb02017-02-28 15:15:56 -0500214 return (f);
215}
216
Florin Coras6cf30ad2017-04-04 23:08:23 -0700217void
218svm_fifo_free (svm_fifo_t * f)
219{
Dave Barach52851e62017-08-07 09:35:25 -0400220 ASSERT (f->refcnt > 0);
221
222 if (--f->refcnt == 0)
223 {
224 pool_free (f->ooo_segments);
225 clib_mem_free (f);
226 }
Florin Coras6cf30ad2017-04-04 23:08:23 -0700227}
228
Dave Barach68b0fb02017-02-28 15:15:56 -0500229always_inline ooo_segment_t *
230ooo_segment_new (svm_fifo_t * f, u32 start, u32 length)
231{
232 ooo_segment_t *s;
233
234 pool_get (f->ooo_segments, s);
235
Dave Barach1f75cfd2017-04-14 16:46:44 -0400236 s->start = start;
Dave Barach68b0fb02017-02-28 15:15:56 -0500237 s->length = length;
238
239 s->prev = s->next = OOO_SEGMENT_INVALID_INDEX;
240
241 return s;
242}
243
244always_inline void
245ooo_segment_del (svm_fifo_t * f, u32 index)
246{
247 ooo_segment_t *cur, *prev = 0, *next = 0;
248 cur = pool_elt_at_index (f->ooo_segments, index);
249
250 if (cur->next != OOO_SEGMENT_INVALID_INDEX)
251 {
252 next = pool_elt_at_index (f->ooo_segments, cur->next);
253 next->prev = cur->prev;
254 }
255
256 if (cur->prev != OOO_SEGMENT_INVALID_INDEX)
257 {
258 prev = pool_elt_at_index (f->ooo_segments, cur->prev);
259 prev->next = cur->next;
260 }
261 else
262 {
263 f->ooos_list_head = cur->next;
264 }
265
266 pool_put (f->ooo_segments, cur);
267}
268
269/**
270 * Add segment to fifo's out-of-order segment list. Takes care of merging
271 * adjacent segments and removing overlapping ones.
272 */
273static void
274ooo_segment_add (svm_fifo_t * f, u32 offset, u32 length)
275{
276 ooo_segment_t *s, *new_s, *prev, *next, *it;
Florin Corasf03a59a2017-06-09 21:07:32 -0700277 u32 new_index, s_end_pos, s_index;
278 u32 normalized_position, normalized_end_position;
Dave Barach68b0fb02017-02-28 15:15:56 -0500279
Florin Coras3eb50622017-07-13 01:24:57 -0400280 ASSERT (offset + length <= ooo_segment_distance_from_tail (f, f->head));
Florin Corasf03a59a2017-06-09 21:07:32 -0700281 normalized_position = (f->tail + offset) % f->nitems;
282 normalized_end_position = (f->tail + offset + length) % f->nitems;
283
284 f->ooos_newest = OOO_SEGMENT_INVALID_INDEX;
Dave Barach68b0fb02017-02-28 15:15:56 -0500285
286 if (f->ooos_list_head == OOO_SEGMENT_INVALID_INDEX)
287 {
Florin Corasf03a59a2017-06-09 21:07:32 -0700288 s = ooo_segment_new (f, normalized_position, length);
Dave Barach68b0fb02017-02-28 15:15:56 -0500289 f->ooos_list_head = s - f->ooo_segments;
290 f->ooos_newest = f->ooos_list_head;
291 return;
292 }
293
294 /* Find first segment that starts after new segment */
295 s = pool_elt_at_index (f->ooo_segments, f->ooos_list_head);
296 while (s->next != OOO_SEGMENT_INVALID_INDEX
Florin Corasf03a59a2017-06-09 21:07:32 -0700297 && position_lt (f, s->start, normalized_position))
Dave Barach68b0fb02017-02-28 15:15:56 -0500298 s = pool_elt_at_index (f->ooo_segments, s->next);
299
Florin Corasc28764f2017-04-26 00:08:42 -0700300 /* If we have a previous and we overlap it, use it as starting point */
301 prev = ooo_segment_get_prev (f, s);
Florin Corasf03a59a2017-06-09 21:07:32 -0700302 if (prev
303 && position_leq (f, normalized_position, ooo_segment_end_pos (f, prev)))
Florin Corasc28764f2017-04-26 00:08:42 -0700304 {
305 s = prev;
Florin Corasf03a59a2017-06-09 21:07:32 -0700306 s_end_pos = ooo_segment_end_pos (f, s);
Dave Barach2c25a622017-06-26 11:35:07 -0400307
Florin Coras3eb50622017-07-13 01:24:57 -0400308 /* Since we have previous, normalized start position cannot be smaller
309 * than prev->start. Check tail */
310 ASSERT (position_lt (f, s->start, normalized_position));
Dave Barach2c25a622017-06-26 11:35:07 -0400311 goto check_tail;
Florin Corasc28764f2017-04-26 00:08:42 -0700312 }
313
Dave Barach68b0fb02017-02-28 15:15:56 -0500314 s_index = s - f->ooo_segments;
Florin Corasf03a59a2017-06-09 21:07:32 -0700315 s_end_pos = ooo_segment_end_pos (f, s);
Dave Barach68b0fb02017-02-28 15:15:56 -0500316
317 /* No overlap, add before current segment */
Florin Corasf03a59a2017-06-09 21:07:32 -0700318 if (position_lt (f, normalized_end_position, s->start))
Dave Barach68b0fb02017-02-28 15:15:56 -0500319 {
Florin Corasf03a59a2017-06-09 21:07:32 -0700320 new_s = ooo_segment_new (f, normalized_position, length);
Dave Barach68b0fb02017-02-28 15:15:56 -0500321 new_index = new_s - f->ooo_segments;
322
323 /* Pool might've moved, get segment again */
324 s = pool_elt_at_index (f->ooo_segments, s_index);
Dave Barach68b0fb02017-02-28 15:15:56 -0500325 if (s->prev != OOO_SEGMENT_INVALID_INDEX)
326 {
327 new_s->prev = s->prev;
Dave Barach68b0fb02017-02-28 15:15:56 -0500328 prev = pool_elt_at_index (f->ooo_segments, new_s->prev);
329 prev->next = new_index;
330 }
331 else
332 {
333 /* New head */
334 f->ooos_list_head = new_index;
335 }
336
Florin Corasf03a59a2017-06-09 21:07:32 -0700337 new_s->next = s_index;
Dave Barach68b0fb02017-02-28 15:15:56 -0500338 s->prev = new_index;
339 f->ooos_newest = new_index;
340 return;
341 }
342 /* No overlap, add after current segment */
Florin Corasf03a59a2017-06-09 21:07:32 -0700343 else if (position_gt (f, normalized_position, s_end_pos))
Dave Barach68b0fb02017-02-28 15:15:56 -0500344 {
Florin Corasf03a59a2017-06-09 21:07:32 -0700345 new_s = ooo_segment_new (f, normalized_position, length);
Dave Barach68b0fb02017-02-28 15:15:56 -0500346 new_index = new_s - f->ooo_segments;
347
348 /* Pool might've moved, get segment again */
349 s = pool_elt_at_index (f->ooo_segments, s_index);
350
Florin Coras3eb50622017-07-13 01:24:57 -0400351 /* Needs to be last */
Florin Corasf03a59a2017-06-09 21:07:32 -0700352 ASSERT (s->next == OOO_SEGMENT_INVALID_INDEX);
Dave Barach68b0fb02017-02-28 15:15:56 -0500353
Florin Corasf03a59a2017-06-09 21:07:32 -0700354 new_s->prev = s_index;
Dave Barach68b0fb02017-02-28 15:15:56 -0500355 s->next = new_index;
356 f->ooos_newest = new_index;
357
358 return;
359 }
360
361 /*
362 * Merge needed
363 */
364
365 /* Merge at head */
Florin Corasf03a59a2017-06-09 21:07:32 -0700366 if (position_lt (f, normalized_position, s->start))
Dave Barach68b0fb02017-02-28 15:15:56 -0500367 {
Florin Corasf03a59a2017-06-09 21:07:32 -0700368 s->start = normalized_position;
369 s->length = position_diff (f, s_end_pos, s->start);
Florin Coras3eb50622017-07-13 01:24:57 -0400370 f->ooos_newest = s - f->ooo_segments;
Dave Barach68b0fb02017-02-28 15:15:56 -0500371 }
372
Dave Barach2c25a622017-06-26 11:35:07 -0400373check_tail:
Florin Coras3eb50622017-07-13 01:24:57 -0400374
375 /* Overlapping tail */
Florin Corasf03a59a2017-06-09 21:07:32 -0700376 if (position_gt (f, normalized_end_position, s_end_pos))
Florin Corasc28764f2017-04-26 00:08:42 -0700377 {
Florin Coras3eb50622017-07-13 01:24:57 -0400378 s->length = position_diff (f, normalized_end_position, s->start);
379
380 /* Remove the completely overlapped segments in the tail */
381 it = ooo_segment_next (f, s);
Florin Corasf03a59a2017-06-09 21:07:32 -0700382 while (it && position_leq (f, ooo_segment_end_pos (f, it),
383 normalized_end_position))
Florin Corasc28764f2017-04-26 00:08:42 -0700384 {
Florin Coras3eb50622017-07-13 01:24:57 -0400385 next = ooo_segment_next (f, it);
Florin Corasc28764f2017-04-26 00:08:42 -0700386 ooo_segment_del (f, it - f->ooo_segments);
387 it = next;
388 }
389
390 /* If partial overlap with last, merge */
Florin Corasf03a59a2017-06-09 21:07:32 -0700391 if (it && position_leq (f, it->start, normalized_end_position))
Florin Corasc28764f2017-04-26 00:08:42 -0700392 {
Florin Coras3eb50622017-07-13 01:24:57 -0400393 s->length = position_diff (f, ooo_segment_end_pos (f, it),
394 s->start);
Florin Corasc28764f2017-04-26 00:08:42 -0700395 ooo_segment_del (f, it - f->ooo_segments);
396 }
Florin Coras3eb50622017-07-13 01:24:57 -0400397 f->ooos_newest = s - f->ooo_segments;
Florin Corasc28764f2017-04-26 00:08:42 -0700398 }
Dave Barach68b0fb02017-02-28 15:15:56 -0500399}
400
401/**
402 * Removes segments that can now be enqueued because the fifo's tail has
403 * advanced. Returns the number of bytes added to tail.
404 */
405static int
406ooo_segment_try_collect (svm_fifo_t * f, u32 n_bytes_enqueued)
407{
408 ooo_segment_t *s;
Florin Corasf03a59a2017-06-09 21:07:32 -0700409 u32 index, bytes = 0;
410 i32 diff;
Dave Barach68b0fb02017-02-28 15:15:56 -0500411
412 s = pool_elt_at_index (f->ooo_segments, f->ooos_list_head);
Dave Barach2c25a622017-06-26 11:35:07 -0400413 diff = ooo_segment_distance_to_tail (f, s->start);
Dave Barach68b0fb02017-02-28 15:15:56 -0500414
Dave Barach2c25a622017-06-26 11:35:07 -0400415 ASSERT (diff != n_bytes_enqueued);
Florin Corasc28764f2017-04-26 00:08:42 -0700416
Florin Corasf03a59a2017-06-09 21:07:32 -0700417 if (diff > n_bytes_enqueued)
Florin Corasb59a7052017-04-18 22:07:29 -0700418 return 0;
419
Dave Barach68b0fb02017-02-28 15:15:56 -0500420 /* If last tail update overlaps one/multiple ooo segments, remove them */
Florin Corasf03a59a2017-06-09 21:07:32 -0700421 while (0 <= diff && diff < n_bytes_enqueued)
Dave Barach68b0fb02017-02-28 15:15:56 -0500422 {
Florin Corasb59a7052017-04-18 22:07:29 -0700423 index = s - f->ooo_segments;
424
425 /* Segment end is beyond the tail. Advance tail and remove segment */
Florin Corasf03a59a2017-06-09 21:07:32 -0700426 if (s->length > diff)
Dave Barach68b0fb02017-02-28 15:15:56 -0500427 {
Florin Corasb59a7052017-04-18 22:07:29 -0700428 bytes = s->length - diff;
Florin Corasf03a59a2017-06-09 21:07:32 -0700429 f->tail += bytes;
430 f->tail %= f->nitems;
Florin Corasb59a7052017-04-18 22:07:29 -0700431 ooo_segment_del (f, index);
Dave Barach68b0fb02017-02-28 15:15:56 -0500432 break;
433 }
Florin Corasb59a7052017-04-18 22:07:29 -0700434
Dave Barach68b0fb02017-02-28 15:15:56 -0500435 /* If we have next go on */
Florin Corasb59a7052017-04-18 22:07:29 -0700436 if (s->next != OOO_SEGMENT_INVALID_INDEX)
Dave Barach68b0fb02017-02-28 15:15:56 -0500437 {
Dave Barach68b0fb02017-02-28 15:15:56 -0500438 s = pool_elt_at_index (f->ooo_segments, s->next);
Dave Barach2c25a622017-06-26 11:35:07 -0400439 diff = ooo_segment_distance_to_tail (f, s->start);
Dave Barach68b0fb02017-02-28 15:15:56 -0500440 ooo_segment_del (f, index);
441 }
442 /* End of search */
443 else
444 {
Florin Corasb59a7052017-04-18 22:07:29 -0700445 ooo_segment_del (f, index);
Dave Barach68b0fb02017-02-28 15:15:56 -0500446 break;
447 }
448 }
449
Chris Lukeab7b8d92017-09-07 07:40:13 -0400450 ASSERT (bytes <= f->nitems);
Dave Barach68b0fb02017-02-28 15:15:56 -0500451 return bytes;
452}
453
454static int
Florin Coras371ca502018-02-21 12:07:41 -0800455svm_fifo_enqueue_internal (svm_fifo_t * f, u32 max_bytes,
456 const u8 * copy_from_here)
Dave Barach68b0fb02017-02-28 15:15:56 -0500457{
458 u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
459 u32 cursize, nitems;
460
Florin Coras3e350af2017-03-30 02:54:28 -0700461 /* read cursize, which can only increase while we're working */
462 cursize = svm_fifo_max_dequeue (f);
Florin Corasf03a59a2017-06-09 21:07:32 -0700463 f->ooos_newest = OOO_SEGMENT_INVALID_INDEX;
Florin Coras3e350af2017-03-30 02:54:28 -0700464
465 if (PREDICT_FALSE (cursize == f->nitems))
Florin Coras7fb0fe12018-04-09 09:24:52 -0700466 return SVM_FIFO_FULL;
Dave Barach68b0fb02017-02-28 15:15:56 -0500467
Dave Barach68b0fb02017-02-28 15:15:56 -0500468 nitems = f->nitems;
469
470 /* Number of bytes we're going to copy */
471 total_copy_bytes = (nitems - cursize) < max_bytes ?
472 (nitems - cursize) : max_bytes;
473
474 if (PREDICT_TRUE (copy_from_here != 0))
475 {
476 /* Number of bytes in first copy segment */
477 first_copy_bytes = ((nitems - f->tail) < total_copy_bytes)
478 ? (nitems - f->tail) : total_copy_bytes;
479
480 clib_memcpy (&f->data[f->tail], copy_from_here, first_copy_bytes);
481 f->tail += first_copy_bytes;
482 f->tail = (f->tail == nitems) ? 0 : f->tail;
483
484 /* Number of bytes in second copy segment, if any */
485 second_copy_bytes = total_copy_bytes - first_copy_bytes;
486 if (second_copy_bytes)
487 {
488 clib_memcpy (&f->data[f->tail], copy_from_here + first_copy_bytes,
489 second_copy_bytes);
490 f->tail += second_copy_bytes;
491 f->tail = (f->tail == nitems) ? 0 : f->tail;
492 }
493 }
494 else
495 {
Dave Barach2c25a622017-06-26 11:35:07 -0400496 ASSERT (0);
497
Dave Barach68b0fb02017-02-28 15:15:56 -0500498 /* Account for a zero-copy enqueue done elsewhere */
499 ASSERT (max_bytes <= (nitems - cursize));
500 f->tail += max_bytes;
501 f->tail = f->tail % nitems;
502 total_copy_bytes = max_bytes;
503 }
504
Florin Coras3eb50622017-07-13 01:24:57 -0400505 svm_fifo_trace_add (f, f->head, total_copy_bytes, 2);
506
Dave Barach68b0fb02017-02-28 15:15:56 -0500507 /* Any out-of-order segments to collect? */
508 if (PREDICT_FALSE (f->ooos_list_head != OOO_SEGMENT_INVALID_INDEX))
509 total_copy_bytes += ooo_segment_try_collect (f, total_copy_bytes);
510
511 /* Atomically increase the queue length */
Dave Barach2c25a622017-06-26 11:35:07 -0400512 ASSERT (cursize + total_copy_bytes <= nitems);
Dave Barach68b0fb02017-02-28 15:15:56 -0500513 __sync_fetch_and_add (&f->cursize, total_copy_bytes);
514
515 return (total_copy_bytes);
516}
517
Florin Corasf6359c82017-06-19 12:26:09 -0400518#define SVM_ENQUEUE_CLONE_TEMPLATE(arch, fn, tgt) \
519 uword \
520 __attribute__ ((flatten)) \
521 __attribute__ ((target (tgt))) \
522 CLIB_CPU_OPTIMIZED \
523 fn ## _ ## arch ( svm_fifo_t * f, u32 max_bytes, u8 * copy_from_here) \
524 { return fn (f, max_bytes, copy_from_here);}
525
526static int
527svm_fifo_enqueue_nowait_ma (svm_fifo_t * f, u32 max_bytes,
Florin Coras371ca502018-02-21 12:07:41 -0800528 const u8 * copy_from_here)
Florin Corasf6359c82017-06-19 12:26:09 -0400529{
530 return svm_fifo_enqueue_internal (f, max_bytes, copy_from_here);
531}
532
533foreach_march_variant (SVM_ENQUEUE_CLONE_TEMPLATE,
534 svm_fifo_enqueue_nowait_ma);
535CLIB_MULTIARCH_SELECT_FN (svm_fifo_enqueue_nowait_ma);
536
Dave Barach68b0fb02017-02-28 15:15:56 -0500537int
Florin Coras371ca502018-02-21 12:07:41 -0800538svm_fifo_enqueue_nowait (svm_fifo_t * f, u32 max_bytes,
539 const u8 * copy_from_here)
Dave Barach68b0fb02017-02-28 15:15:56 -0500540{
Florin Corasf6359c82017-06-19 12:26:09 -0400541#if CLIB_DEBUG > 0
542 return svm_fifo_enqueue_nowait_ma (f, max_bytes, copy_from_here);
543#else
Florin Coras371ca502018-02-21 12:07:41 -0800544 static int (*fp) (svm_fifo_t *, u32, const u8 *);
Florin Corasf6359c82017-06-19 12:26:09 -0400545
546 if (PREDICT_FALSE (fp == 0))
547 fp = (void *) svm_fifo_enqueue_nowait_ma_multiarch_select ();
548
549 return (*fp) (f, max_bytes, copy_from_here);
550#endif
Dave Barach68b0fb02017-02-28 15:15:56 -0500551}
552
Florin Coras6792ec02017-03-13 03:49:51 -0700553/**
554 * Enqueue a future segment.
555 *
Dave Barach68b0fb02017-02-28 15:15:56 -0500556 * Two choices: either copies the entire segment, or copies nothing
557 * Returns 0 of the entire segment was copied
558 * Returns -1 if none of the segment was copied due to lack of space
559 */
Dave Barach68b0fb02017-02-28 15:15:56 -0500560static int
Florin Coras6792ec02017-03-13 03:49:51 -0700561svm_fifo_enqueue_with_offset_internal (svm_fifo_t * f,
Florin Coras6792ec02017-03-13 03:49:51 -0700562 u32 offset,
563 u32 required_bytes,
564 u8 * copy_from_here)
Dave Barach68b0fb02017-02-28 15:15:56 -0500565{
566 u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
Florin Corasf03a59a2017-06-09 21:07:32 -0700567 u32 cursize, nitems, normalized_offset;
Florin Corasf03a59a2017-06-09 21:07:32 -0700568
569 f->ooos_newest = OOO_SEGMENT_INVALID_INDEX;
Dave Barach68b0fb02017-02-28 15:15:56 -0500570
Florin Coras3e350af2017-03-30 02:54:28 -0700571 /* read cursize, which can only increase while we're working */
572 cursize = svm_fifo_max_dequeue (f);
Dave Barach68b0fb02017-02-28 15:15:56 -0500573 nitems = f->nitems;
Florin Corasf03a59a2017-06-09 21:07:32 -0700574
Dave Barach2c25a622017-06-26 11:35:07 -0400575 ASSERT (required_bytes < nitems);
576
Florin Corasf03a59a2017-06-09 21:07:32 -0700577 normalized_offset = (f->tail + offset) % nitems;
Dave Barach68b0fb02017-02-28 15:15:56 -0500578
579 /* Will this request fit? */
Florin Coras1f152cd2017-08-18 19:28:03 -0700580 if ((required_bytes + offset) > (nitems - cursize))
Dave Barach68b0fb02017-02-28 15:15:56 -0500581 return -1;
582
Florin Coras3eb50622017-07-13 01:24:57 -0400583 svm_fifo_trace_add (f, offset, required_bytes, 1);
584
Dave Barach68b0fb02017-02-28 15:15:56 -0500585 ooo_segment_add (f, offset, required_bytes);
586
587 /* Number of bytes we're going to copy */
588 total_copy_bytes = required_bytes;
Dave Barach68b0fb02017-02-28 15:15:56 -0500589
590 /* Number of bytes in first copy segment */
Dave Barach1f75cfd2017-04-14 16:46:44 -0400591 first_copy_bytes = ((nitems - normalized_offset) < total_copy_bytes)
592 ? (nitems - normalized_offset) : total_copy_bytes;
Dave Barach68b0fb02017-02-28 15:15:56 -0500593
Dave Barach1f75cfd2017-04-14 16:46:44 -0400594 clib_memcpy (&f->data[normalized_offset], copy_from_here, first_copy_bytes);
Dave Barach68b0fb02017-02-28 15:15:56 -0500595
596 /* Number of bytes in second copy segment, if any */
597 second_copy_bytes = total_copy_bytes - first_copy_bytes;
598 if (second_copy_bytes)
599 {
Dave Barach1f75cfd2017-04-14 16:46:44 -0400600 normalized_offset += first_copy_bytes;
601 normalized_offset %= nitems;
Dave Barach68b0fb02017-02-28 15:15:56 -0500602
Dave Barach1f75cfd2017-04-14 16:46:44 -0400603 ASSERT (normalized_offset == 0);
Dave Barach68b0fb02017-02-28 15:15:56 -0500604
Dave Barach1f75cfd2017-04-14 16:46:44 -0400605 clib_memcpy (&f->data[normalized_offset],
Dave Barach68b0fb02017-02-28 15:15:56 -0500606 copy_from_here + first_copy_bytes, second_copy_bytes);
607 }
608
609 return (0);
610}
611
612
613int
614svm_fifo_enqueue_with_offset (svm_fifo_t * f,
Dave Barach68b0fb02017-02-28 15:15:56 -0500615 u32 offset,
616 u32 required_bytes, u8 * copy_from_here)
617{
Florin Corasa5464812017-04-19 13:00:05 -0700618 return svm_fifo_enqueue_with_offset_internal (f, offset, required_bytes,
619 copy_from_here);
Dave Barach68b0fb02017-02-28 15:15:56 -0500620}
621
Florin Coras7fb0fe12018-04-09 09:24:52 -0700622void
623svm_fifo_overwrite_head (svm_fifo_t * f, u8 * data, u32 len)
624{
625 u32 first_chunk;
Florin Coras8e43d042018-05-04 15:46:57 -0700626 first_chunk = f->nitems - f->head;
Florin Coras7fb0fe12018-04-09 09:24:52 -0700627 ASSERT (len <= f->nitems);
Florin Coras8e43d042018-05-04 15:46:57 -0700628 if (len <= first_chunk)
Florin Coras7fb0fe12018-04-09 09:24:52 -0700629 clib_memcpy (&f->data[f->head], data, len);
630 else
631 {
Florin Coras7fb0fe12018-04-09 09:24:52 -0700632 clib_memcpy (&f->data[f->head], data, first_chunk);
Florin Coras8e43d042018-05-04 15:46:57 -0700633 clib_memcpy (&f->data[0], data + first_chunk, len - first_chunk);
Florin Coras7fb0fe12018-04-09 09:24:52 -0700634 }
635}
Dave Barach68b0fb02017-02-28 15:15:56 -0500636
637static int
Florin Corasa5464812017-04-19 13:00:05 -0700638svm_fifo_dequeue_internal (svm_fifo_t * f, u32 max_bytes, u8 * copy_here)
Dave Barach68b0fb02017-02-28 15:15:56 -0500639{
640 u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
641 u32 cursize, nitems;
642
Florin Coras3e350af2017-03-30 02:54:28 -0700643 /* read cursize, which can only increase while we're working */
644 cursize = svm_fifo_max_dequeue (f);
645 if (PREDICT_FALSE (cursize == 0))
Dave Barach68b0fb02017-02-28 15:15:56 -0500646 return -2; /* nothing in the fifo */
647
Dave Barach68b0fb02017-02-28 15:15:56 -0500648 nitems = f->nitems;
649
650 /* Number of bytes we're going to copy */
651 total_copy_bytes = (cursize < max_bytes) ? cursize : max_bytes;
652
653 if (PREDICT_TRUE (copy_here != 0))
654 {
655 /* Number of bytes in first copy segment */
656 first_copy_bytes = ((nitems - f->head) < total_copy_bytes)
657 ? (nitems - f->head) : total_copy_bytes;
658 clib_memcpy (copy_here, &f->data[f->head], first_copy_bytes);
659 f->head += first_copy_bytes;
660 f->head = (f->head == nitems) ? 0 : f->head;
661
662 /* Number of bytes in second copy segment, if any */
663 second_copy_bytes = total_copy_bytes - first_copy_bytes;
664 if (second_copy_bytes)
665 {
666 clib_memcpy (copy_here + first_copy_bytes,
667 &f->data[f->head], second_copy_bytes);
668 f->head += second_copy_bytes;
669 f->head = (f->head == nitems) ? 0 : f->head;
670 }
671 }
672 else
673 {
Dave Barach2c25a622017-06-26 11:35:07 -0400674 ASSERT (0);
Dave Barach68b0fb02017-02-28 15:15:56 -0500675 /* Account for a zero-copy dequeue done elsewhere */
676 ASSERT (max_bytes <= cursize);
677 f->head += max_bytes;
678 f->head = f->head % nitems;
679 cursize -= max_bytes;
680 total_copy_bytes = max_bytes;
681 }
682
Dave Barach2c25a622017-06-26 11:35:07 -0400683 ASSERT (f->head <= nitems);
684 ASSERT (cursize >= total_copy_bytes);
Dave Barach68b0fb02017-02-28 15:15:56 -0500685 __sync_fetch_and_sub (&f->cursize, total_copy_bytes);
686
687 return (total_copy_bytes);
688}
689
Florin Corasf6359c82017-06-19 12:26:09 -0400690static int
691svm_fifo_dequeue_nowait_ma (svm_fifo_t * f, u32 max_bytes, u8 * copy_here)
Dave Barach68b0fb02017-02-28 15:15:56 -0500692{
Florin Corasa5464812017-04-19 13:00:05 -0700693 return svm_fifo_dequeue_internal (f, max_bytes, copy_here);
Dave Barach68b0fb02017-02-28 15:15:56 -0500694}
695
Florin Corasf6359c82017-06-19 12:26:09 -0400696#define SVM_FIFO_DEQUEUE_CLONE_TEMPLATE(arch, fn, tgt) \
697 uword \
698 __attribute__ ((flatten)) \
699 __attribute__ ((target (tgt))) \
700 CLIB_CPU_OPTIMIZED \
701 fn ## _ ## arch ( svm_fifo_t * f, u32 max_bytes, \
702 u8 * copy_here) \
703 { return fn (f, max_bytes, copy_here);}
704
705foreach_march_variant (SVM_FIFO_DEQUEUE_CLONE_TEMPLATE,
706 svm_fifo_dequeue_nowait_ma);
707CLIB_MULTIARCH_SELECT_FN (svm_fifo_dequeue_nowait_ma);
708
Dave Barach68b0fb02017-02-28 15:15:56 -0500709int
Florin Corasf6359c82017-06-19 12:26:09 -0400710svm_fifo_dequeue_nowait (svm_fifo_t * f, u32 max_bytes, u8 * copy_here)
711{
712#if CLIB_DEBUG > 0
713 return svm_fifo_dequeue_nowait_ma (f, max_bytes, copy_here);
714#else
715 static int (*fp) (svm_fifo_t *, u32, u8 *);
716
717 if (PREDICT_FALSE (fp == 0))
718 fp = (void *) svm_fifo_dequeue_nowait_ma_multiarch_select ();
719
720 return (*fp) (f, max_bytes, copy_here);
721#endif
722}
723
724static int
725svm_fifo_peek_ma (svm_fifo_t * f, u32 relative_offset, u32 max_bytes,
726 u8 * copy_here)
Dave Barach68b0fb02017-02-28 15:15:56 -0500727{
728 u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
Florin Coras6792ec02017-03-13 03:49:51 -0700729 u32 cursize, nitems, real_head;
Dave Barach68b0fb02017-02-28 15:15:56 -0500730
Florin Coras3e350af2017-03-30 02:54:28 -0700731 /* read cursize, which can only increase while we're working */
732 cursize = svm_fifo_max_dequeue (f);
Florin Coras93992a92017-05-24 18:03:56 -0700733 if (PREDICT_FALSE (cursize < relative_offset))
Dave Barach68b0fb02017-02-28 15:15:56 -0500734 return -2; /* nothing in the fifo */
735
Dave Barach68b0fb02017-02-28 15:15:56 -0500736 nitems = f->nitems;
Florin Corasb59a7052017-04-18 22:07:29 -0700737 real_head = f->head + relative_offset;
Florin Coras6792ec02017-03-13 03:49:51 -0700738 real_head = real_head >= nitems ? real_head - nitems : real_head;
Dave Barach68b0fb02017-02-28 15:15:56 -0500739
740 /* Number of bytes we're going to copy */
Florin Coras93992a92017-05-24 18:03:56 -0700741 total_copy_bytes = (cursize - relative_offset < max_bytes) ?
742 cursize - relative_offset : max_bytes;
Dave Barach68b0fb02017-02-28 15:15:56 -0500743
744 if (PREDICT_TRUE (copy_here != 0))
745 {
746 /* Number of bytes in first copy segment */
747 first_copy_bytes =
Florin Coras6792ec02017-03-13 03:49:51 -0700748 ((nitems - real_head) < total_copy_bytes) ?
749 (nitems - real_head) : total_copy_bytes;
750 clib_memcpy (copy_here, &f->data[real_head], first_copy_bytes);
Dave Barach68b0fb02017-02-28 15:15:56 -0500751
752 /* Number of bytes in second copy segment, if any */
753 second_copy_bytes = total_copy_bytes - first_copy_bytes;
754 if (second_copy_bytes)
755 {
756 clib_memcpy (copy_here + first_copy_bytes, &f->data[0],
757 second_copy_bytes);
758 }
759 }
760 return total_copy_bytes;
761}
762
Florin Corasf6359c82017-06-19 12:26:09 -0400763#define SVM_FIFO_PEEK_CLONE_TEMPLATE(arch, fn, tgt) \
764 uword \
765 __attribute__ ((flatten)) \
766 __attribute__ ((target (tgt))) \
767 CLIB_CPU_OPTIMIZED \
768 fn ## _ ## arch ( svm_fifo_t * f, u32 relative_offset, u32 max_bytes, \
769 u8 * copy_here) \
770 { return fn (f, relative_offset, max_bytes, copy_here);}
771
772foreach_march_variant (SVM_FIFO_PEEK_CLONE_TEMPLATE, svm_fifo_peek_ma);
773CLIB_MULTIARCH_SELECT_FN (svm_fifo_peek_ma);
774
775int
776svm_fifo_peek (svm_fifo_t * f, u32 relative_offset, u32 max_bytes,
777 u8 * copy_here)
778{
779#if CLIB_DEBUG > 0
780 return svm_fifo_peek_ma (f, relative_offset, max_bytes, copy_here);
781#else
782 static int (*fp) (svm_fifo_t *, u32, u32, u8 *);
783
784 if (PREDICT_FALSE (fp == 0))
785 fp = (void *) svm_fifo_peek_ma_multiarch_select ();
786
787 return (*fp) (f, relative_offset, max_bytes, copy_here);
788#endif
789}
790
Dave Barach68b0fb02017-02-28 15:15:56 -0500791int
Florin Corasa5464812017-04-19 13:00:05 -0700792svm_fifo_dequeue_drop (svm_fifo_t * f, u32 max_bytes)
Dave Barach68b0fb02017-02-28 15:15:56 -0500793{
794 u32 total_drop_bytes, first_drop_bytes, second_drop_bytes;
795 u32 cursize, nitems;
796
Florin Coras3e350af2017-03-30 02:54:28 -0700797 /* read cursize, which can only increase while we're working */
798 cursize = svm_fifo_max_dequeue (f);
799 if (PREDICT_FALSE (cursize == 0))
Dave Barach68b0fb02017-02-28 15:15:56 -0500800 return -2; /* nothing in the fifo */
801
Dave Barach68b0fb02017-02-28 15:15:56 -0500802 nitems = f->nitems;
803
804 /* Number of bytes we're going to drop */
805 total_drop_bytes = (cursize < max_bytes) ? cursize : max_bytes;
806
Florin Coras3eb50622017-07-13 01:24:57 -0400807 svm_fifo_trace_add (f, f->tail, total_drop_bytes, 3);
808
Dave Barach68b0fb02017-02-28 15:15:56 -0500809 /* Number of bytes in first copy segment */
810 first_drop_bytes =
811 ((nitems - f->head) < total_drop_bytes) ?
812 (nitems - f->head) : total_drop_bytes;
813 f->head += first_drop_bytes;
814 f->head = (f->head == nitems) ? 0 : f->head;
815
816 /* Number of bytes in second drop segment, if any */
817 second_drop_bytes = total_drop_bytes - first_drop_bytes;
818 if (second_drop_bytes)
819 {
820 f->head += second_drop_bytes;
821 f->head = (f->head == nitems) ? 0 : f->head;
822 }
823
Dave Barach2c25a622017-06-26 11:35:07 -0400824 ASSERT (f->head <= nitems);
825 ASSERT (cursize >= total_drop_bytes);
Dave Barach68b0fb02017-02-28 15:15:56 -0500826 __sync_fetch_and_sub (&f->cursize, total_drop_bytes);
827
828 return total_drop_bytes;
829}
830
Dave Barach1f75cfd2017-04-14 16:46:44 -0400831u32
832svm_fifo_number_ooo_segments (svm_fifo_t * f)
833{
834 return pool_elts (f->ooo_segments);
835}
836
837ooo_segment_t *
838svm_fifo_first_ooo_segment (svm_fifo_t * f)
839{
840 return pool_elt_at_index (f->ooo_segments, f->ooos_list_head);
841}
842
Florin Corasc28764f2017-04-26 00:08:42 -0700843/**
844 * Set fifo pointers to requested offset
845 */
846void
847svm_fifo_init_pointers (svm_fifo_t * f, u32 pointer)
848{
849 f->head = f->tail = pointer % f->nitems;
850}
851
Dave Barach68b0fb02017-02-28 15:15:56 -0500852/*
853 * fd.io coding-style-patch-verification: ON
854 *
855 * Local Variables:
856 * eval: (c-set-style "gnu")
857 * End:
858 */