blob: 47c7f5635428e9c015585ce1504cf13083708e90 [file] [log] [blame]
Florin Coras6cf30ad2017-04-04 23:08:23 -07001/*
Florin Coras288eaab2019-02-03 15:26:14 -08002 * Copyright (c) 2017-2019 Cisco and/or its affiliates.
Florin Coras6cf30ad2017-04-04 23:08:23 -07003 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16#include <vnet/session/segment_manager.h>
17#include <vnet/session/session.h>
18#include <vnet/session/application.h>
19
Florin Coras88001c62019-04-24 14:44:46 -070020typedef struct segment_manager_main_
21{
22 segment_manager_t *segment_managers; /**< Pool of segment managers */
23 clib_valloc_main_t va_allocator; /**< Virtual address allocator */
24 u32 seg_name_counter; /**< Counter for segment names */
Florin Corasa332c462018-01-31 06:52:17 -080025
Florin Coras88001c62019-04-24 14:44:46 -070026 /*
27 * Configuration
28 */
29 u32 default_fifo_size; /**< default rx/tx fifo size */
30 u32 default_segment_size; /**< default fifo segment size */
31 u32 default_app_mq_size; /**< default app msg q size */
32} segment_manager_main_t;
Florin Corasa5464812017-04-19 13:00:05 -070033
Florin Coras88001c62019-04-24 14:44:46 -070034static segment_manager_main_t sm_main;
Florin Coras6cf30ad2017-04-04 23:08:23 -070035
Florin Coras88001c62019-04-24 14:44:46 -070036#define segment_manager_foreach_segment_w_lock(VAR, SM, BODY) \
37do { \
38 clib_rwlock_reader_lock (&(SM)->segments_rwlock); \
39 pool_foreach((VAR), ((SM)->segments), (BODY)); \
40 clib_rwlock_reader_unlock (&(SM)->segments_rwlock); \
41} while (0)
42
43static segment_manager_props_t *
Florin Corasa332c462018-01-31 06:52:17 -080044segment_manager_properties_get (segment_manager_t * sm)
Florin Corasad0c77f2017-11-09 18:00:15 -080045{
Florin Corasab2f6db2018-08-31 14:31:41 -070046 app_worker_t *app_wrk = app_worker_get (sm->app_wrk_index);
47 return application_get_segment_manager_properties (app_wrk->app_index);
Florin Corasa332c462018-01-31 06:52:17 -080048}
49
Florin Coras88001c62019-04-24 14:44:46 -070050segment_manager_props_t *
51segment_manager_props_init (segment_manager_props_t * props)
Florin Corasa332c462018-01-31 06:52:17 -080052{
Florin Coras88001c62019-04-24 14:44:46 -070053 props->add_segment_size = sm_main.default_segment_size;
54 props->rx_fifo_size = sm_main.default_fifo_size;
55 props->tx_fifo_size = sm_main.default_fifo_size;
56 props->evt_q_size = sm_main.default_app_mq_size;
Florin Corasad0c77f2017-11-09 18:00:15 -080057 return props;
58}
59
Florin Coras9d063042017-09-14 03:08:00 -040060static u8
61segment_manager_app_detached (segment_manager_t * sm)
62{
Florin Coras15531972018-08-12 23:50:53 -070063 return (sm->app_wrk_index == SEGMENT_MANAGER_INVALID_APP_INDEX);
Dave Wallace7b749fe2017-07-05 14:30:46 -040064}
65
Florin Coras4e4531e2017-11-06 23:27:56 -080066void
67segment_manager_app_detach (segment_manager_t * sm)
68{
Florin Coras15531972018-08-12 23:50:53 -070069 sm->app_wrk_index = SEGMENT_MANAGER_INVALID_APP_INDEX;
Florin Coras4e4531e2017-11-06 23:27:56 -080070}
71
Florin Corasa332c462018-01-31 06:52:17 -080072always_inline u32
Florin Coras88001c62019-04-24 14:44:46 -070073segment_manager_segment_index (segment_manager_t * sm, fifo_segment_t * seg)
Florin Corasc87c91d2017-08-16 19:55:49 -070074{
Florin Corasa332c462018-01-31 06:52:17 -080075 return (seg - sm->segments);
76}
77
78/**
Florin Coras88001c62019-04-24 14:44:46 -070079 * Adds segment to segment manager's pool
80 *
81 * If needed a writer's lock is acquired before allocating a new segment
82 * to avoid affecting any of the segments pool readers.
83 */
84int
85segment_manager_add_segment (segment_manager_t * sm, u32 segment_size)
86{
87 segment_manager_main_t *smm = &sm_main;
88 u32 rnd_margin = 128 << 10, fs_index = ~0, page_size;
89 uword baseva = (uword) ~ 0ULL, alloc_size;
90 segment_manager_props_t *props;
91 fifo_segment_t *fs;
92 u8 *seg_name;
93 int rv;
94
95 props = segment_manager_properties_get (sm);
96
97 /* Not configured for addition of new segments and not first */
98 if (!props->add_segment && !segment_size)
99 {
100 clib_warning ("cannot allocate new segment");
101 return VNET_API_ERROR_INVALID_VALUE;
102 }
103
104 /*
105 * Allocate fifo segment and lock if needed
106 */
107 if (vlib_num_workers ())
108 clib_rwlock_writer_lock (&sm->segments_rwlock);
109
110 pool_get_zero (sm->segments, fs);
111
112 /*
113 * Initialize ssvm segment and svm fifo private header
114 */
115 segment_size = segment_size ? segment_size : props->add_segment_size;
116 page_size = clib_mem_get_page_size ();
117 segment_size = (segment_size + page_size - 1) & ~(page_size - 1);
118 if (props->segment_type != SSVM_SEGMENT_PRIVATE)
119 {
120 seg_name = format (0, "%d-%d%c", getpid (), smm->seg_name_counter++, 0);
121 alloc_size = (uword) segment_size + rnd_margin;
122 baseva = clib_valloc_alloc (&smm->va_allocator, alloc_size, 0);
123 if (!baseva)
124 {
125 clib_warning ("out of space for segments");
126 pool_put (sm->segments, fs);
127 goto done;
128 }
129 }
130 else
131 seg_name = format (0, "%s%c", "process-private-segment", 0);
132
133 fs->ssvm.ssvm_size = segment_size;
134 fs->ssvm.name = seg_name;
135 fs->ssvm.requested_va = baseva;
136
137 if ((rv = ssvm_master_init (&fs->ssvm, props->segment_type)))
138 {
139 clib_warning ("svm_master_init ('%v', %u) failed", seg_name,
140 segment_size);
141
142 if (props->segment_type != SSVM_SEGMENT_PRIVATE)
143 clib_valloc_free (&smm->va_allocator, baseva);
144 pool_put (sm->segments, fs);
145 goto done;
146 }
147
148 fifo_segment_init (fs);
149
150 /*
151 * Save segment index before dropping lock, if any held
152 */
153 fs_index = fs - sm->segments;
154
155done:
156
157 if (vlib_num_workers ())
158 clib_rwlock_writer_unlock (&sm->segments_rwlock);
159
160 return fs_index;
161}
162
163/**
Florin Corasa332c462018-01-31 06:52:17 -0800164 * Remove segment without lock
165 */
Florin Corasf8f516a2018-02-08 15:10:09 -0800166void
Florin Coras88001c62019-04-24 14:44:46 -0700167segment_manager_del_segment (segment_manager_t * sm, fifo_segment_t * fs)
Florin Corasa332c462018-01-31 06:52:17 -0800168{
Florin Coras88001c62019-04-24 14:44:46 -0700169 segment_manager_main_t *smm = &sm_main;
Florin Corasa332c462018-01-31 06:52:17 -0800170
171 if (ssvm_type (&fs->ssvm) != SSVM_SEGMENT_PRIVATE)
Florin Corasa0dbf9e2019-03-01 17:12:02 -0800172 {
173 clib_valloc_free (&smm->va_allocator, fs->ssvm.requested_va);
174
175 if (sm->app_wrk_index != SEGMENT_MANAGER_INVALID_APP_INDEX)
176 {
177 app_worker_t *app_wrk;
178 u64 segment_handle;
179 app_wrk = app_worker_get (sm->app_wrk_index);
180 segment_handle = segment_manager_segment_handle (sm, fs);
181 app_worker_del_segment_notify (app_wrk, segment_handle);
182 }
183 }
Florin Corasa332c462018-01-31 06:52:17 -0800184
185 ssvm_delete (&fs->ssvm);
186
187 if (CLIB_DEBUG)
Dave Barachb7b92992018-10-17 10:38:51 -0400188 clib_memset (fs, 0xfb, sizeof (*fs));
Florin Corasa332c462018-01-31 06:52:17 -0800189 pool_put (sm->segments, fs);
190}
191
192/**
193 * Removes segment after acquiring writer lock
194 */
Florin Coras99368312018-08-02 10:45:44 -0700195static inline void
Florin Corasa332c462018-01-31 06:52:17 -0800196segment_manager_lock_and_del_segment (segment_manager_t * sm, u32 fs_index)
197{
Florin Coras88001c62019-04-24 14:44:46 -0700198 fifo_segment_t *fs;
Florin Corasa332c462018-01-31 06:52:17 -0800199 u8 is_prealloc;
200
201 clib_rwlock_writer_lock (&sm->segments_rwlock);
202 fs = segment_manager_get_segment (sm, fs_index);
Florin Coras88001c62019-04-24 14:44:46 -0700203 is_prealloc = fifo_segment_flags (fs) & FIFO_SEGMENT_F_IS_PREALLOCATED;
Florin Corasa332c462018-01-31 06:52:17 -0800204 if (is_prealloc && !segment_manager_app_detached (sm))
Florin Coras9d063042017-09-14 03:08:00 -0400205 {
Florin Corasa332c462018-01-31 06:52:17 -0800206 clib_rwlock_writer_unlock (&sm->segments_rwlock);
Florin Coras9d063042017-09-14 03:08:00 -0400207 return;
208 }
Florin Corasa332c462018-01-31 06:52:17 -0800209
210 segment_manager_del_segment (sm, fs);
211 clib_rwlock_writer_unlock (&sm->segments_rwlock);
212}
213
214/**
215 * Reads a segment from the segment manager's pool without lock
216 */
Florin Coras88001c62019-04-24 14:44:46 -0700217fifo_segment_t *
Florin Corasa332c462018-01-31 06:52:17 -0800218segment_manager_get_segment (segment_manager_t * sm, u32 segment_index)
219{
220 return pool_elt_at_index (sm->segments, segment_index);
221}
222
Florin Corasfa76a762018-11-29 12:40:10 -0800223u64
224segment_manager_segment_handle (segment_manager_t * sm,
Florin Coras88001c62019-04-24 14:44:46 -0700225 fifo_segment_t * segment)
Florin Corasfa76a762018-11-29 12:40:10 -0800226{
227 u32 segment_index = segment_manager_segment_index (sm, segment);
228 return (((u64) segment_manager_index (sm) << 32) | segment_index);
229}
230
Florin Coras88001c62019-04-24 14:44:46 -0700231static void
Florin Corasfa76a762018-11-29 12:40:10 -0800232segment_manager_parse_segment_handle (u64 segment_handle, u32 * sm_index,
233 u32 * segment_index)
234{
235 *sm_index = segment_handle >> 32;
236 *segment_index = segment_handle & 0xFFFFFFFF;
237}
238
Florin Coras88001c62019-04-24 14:44:46 -0700239u64
240segment_manager_make_segment_handle (u32 segment_manager_index,
241 u32 segment_index)
242{
243 return (((u64) segment_manager_index << 32) | segment_index);
244}
245
246fifo_segment_t *
Florin Corasfa76a762018-11-29 12:40:10 -0800247segment_manager_get_segment_w_handle (u64 segment_handle)
248{
249 u32 sm_index, segment_index;
250 segment_manager_t *sm;
251
252 segment_manager_parse_segment_handle (segment_handle, &sm_index,
253 &segment_index);
254 sm = segment_manager_get (sm_index);
255 if (!sm || pool_is_free_index (sm->segments, segment_index))
256 return 0;
257 return pool_elt_at_index (sm->segments, segment_index);
258}
259
Florin Corasa332c462018-01-31 06:52:17 -0800260/**
261 * Reads a segment from the segment manager's pool and acquires reader lock
262 *
263 * Caller must drop the reader's lock by calling
264 * @ref segment_manager_segment_reader_unlock once it finishes working with
265 * the segment.
266 */
Florin Coras88001c62019-04-24 14:44:46 -0700267fifo_segment_t *
Florin Corasa332c462018-01-31 06:52:17 -0800268segment_manager_get_segment_w_lock (segment_manager_t * sm, u32 segment_index)
269{
270 clib_rwlock_reader_lock (&sm->segments_rwlock);
271 return pool_elt_at_index (sm->segments, segment_index);
272}
273
274void
275segment_manager_segment_reader_unlock (segment_manager_t * sm)
276{
Florin Corasf8f516a2018-02-08 15:10:09 -0800277 ASSERT (sm->segments_rwlock->n_readers > 0);
Florin Corasa332c462018-01-31 06:52:17 -0800278 clib_rwlock_reader_unlock (&sm->segments_rwlock);
279}
280
281void
282segment_manager_segment_writer_unlock (segment_manager_t * sm)
283{
284 clib_rwlock_writer_unlock (&sm->segments_rwlock);
285}
286
Florin Corasa332c462018-01-31 06:52:17 -0800287segment_manager_t *
Florin Coras88001c62019-04-24 14:44:46 -0700288segment_manager_alloc (void)
Florin Corasa332c462018-01-31 06:52:17 -0800289{
Florin Coras88001c62019-04-24 14:44:46 -0700290 segment_manager_main_t *smm = &sm_main;
Florin Corasa332c462018-01-31 06:52:17 -0800291 segment_manager_t *sm;
Florin Coras88001c62019-04-24 14:44:46 -0700292
293 pool_get_zero (smm->segment_managers, sm);
Florin Corasa332c462018-01-31 06:52:17 -0800294 clib_rwlock_init (&sm->segments_rwlock);
295 return sm;
296}
297
298/**
299 * Initializes segment manager based on options provided.
300 * Returns error if ssvm segment(s) allocation fails.
301 */
302int
303segment_manager_init (segment_manager_t * sm, u32 first_seg_size,
Florin Corasf8f516a2018-02-08 15:10:09 -0800304 u32 prealloc_fifo_pairs)
Florin Corasa332c462018-01-31 06:52:17 -0800305{
306 u32 rx_fifo_size, tx_fifo_size, pair_size;
307 u32 rx_rounded_data_size, tx_rounded_data_size;
Florin Coras86f04502018-09-12 16:08:01 -0700308 u64 approx_total_size, max_seg_size = ((u64) 1 << 32) - (128 << 10);
Florin Coras88001c62019-04-24 14:44:46 -0700309 segment_manager_props_t *props;
310 fifo_segment_t *segment;
Florin Corasa332c462018-01-31 06:52:17 -0800311 u32 approx_segment_count;
312 int seg_index, i;
313
314 props = segment_manager_properties_get (sm);
Florin Coras88001c62019-04-24 14:44:46 -0700315 first_seg_size = clib_max (first_seg_size, sm_main.default_segment_size);
Florin Corasa332c462018-01-31 06:52:17 -0800316
317 if (prealloc_fifo_pairs)
318 {
319 /* Figure out how many segments should be preallocated */
320 rx_rounded_data_size = (1 << (max_log2 (props->rx_fifo_size)));
321 tx_rounded_data_size = (1 << (max_log2 (props->tx_fifo_size)));
322
323 rx_fifo_size = sizeof (svm_fifo_t) + rx_rounded_data_size;
324 tx_fifo_size = sizeof (svm_fifo_t) + tx_rounded_data_size;
325 pair_size = rx_fifo_size + tx_fifo_size;
326
327 approx_total_size = (u64) prealloc_fifo_pairs *pair_size;
328 if (first_seg_size > approx_total_size)
329 max_seg_size = first_seg_size;
330 approx_segment_count = (approx_total_size + (max_seg_size - 1))
331 / max_seg_size;
332
333 /* Allocate the segments */
334 for (i = 0; i < approx_segment_count + 1; i++)
335 {
336 seg_index = segment_manager_add_segment (sm, max_seg_size);
337 if (seg_index < 0)
338 {
339 clib_warning ("Failed to preallocate segment %d", i);
340 return seg_index;
341 }
342
Florin Corasa332c462018-01-31 06:52:17 -0800343 segment = segment_manager_get_segment (sm, seg_index);
Florin Corasf8f516a2018-02-08 15:10:09 -0800344 if (i == 0)
Florin Coras99368312018-08-02 10:45:44 -0700345 sm->event_queue = segment_manager_alloc_queue (segment, props);
Florin Corasf8f516a2018-02-08 15:10:09 -0800346
Florin Coras88001c62019-04-24 14:44:46 -0700347 fifo_segment_preallocate_fifo_pairs (segment,
348 props->rx_fifo_size,
349 props->tx_fifo_size,
350 &prealloc_fifo_pairs);
351 fifo_segment_flags (segment) = FIFO_SEGMENT_F_IS_PREALLOCATED;
Florin Corasa332c462018-01-31 06:52:17 -0800352 if (prealloc_fifo_pairs == 0)
353 break;
354 }
355 }
356 else
357 {
358 seg_index = segment_manager_add_segment (sm, first_seg_size);
Florin Coras402377e2019-04-17 10:23:23 -0700359 if (seg_index < 0)
Florin Corasa332c462018-01-31 06:52:17 -0800360 {
361 clib_warning ("Failed to allocate segment");
362 return seg_index;
363 }
Florin Corasf8f516a2018-02-08 15:10:09 -0800364 segment = segment_manager_get_segment (sm, seg_index);
Florin Coras99368312018-08-02 10:45:44 -0700365 sm->event_queue = segment_manager_alloc_queue (segment, props);
Florin Corasa332c462018-01-31 06:52:17 -0800366 }
367
368 return 0;
369}
370
Florin Corasc87c91d2017-08-16 19:55:49 -0700371/**
Florin Coras88001c62019-04-24 14:44:46 -0700372 * Cleanup segment manager.
Florin Coras6cf30ad2017-04-04 23:08:23 -0700373 */
374void
Florin Coras88001c62019-04-24 14:44:46 -0700375segment_manager_free (segment_manager_t * sm)
Florin Coras6cf30ad2017-04-04 23:08:23 -0700376{
Florin Coras88001c62019-04-24 14:44:46 -0700377 segment_manager_main_t *smm = &sm_main;
378 fifo_segment_t *fifo_segment;
Dave Wallace7b749fe2017-07-05 14:30:46 -0400379
Florin Coras9d063042017-09-14 03:08:00 -0400380 ASSERT (!segment_manager_has_fifos (sm)
381 && segment_manager_app_detached (sm));
382
383 /* If we have empty preallocated segments that haven't been removed, remove
384 * them now. Apart from that, the first segment in the first segment manager
385 * is not removed when all fifos are removed. It can only be removed when
386 * the manager is explicitly deleted/detached by the app. */
Florin Corasa332c462018-01-31 06:52:17 -0800387 clib_rwlock_writer_lock (&sm->segments_rwlock);
388
389 /* *INDENT-OFF* */
390 pool_foreach (fifo_segment, sm->segments, ({
391 segment_manager_del_segment (sm, fifo_segment);
392 }));
393 /* *INDENT-ON* */
394
395 clib_rwlock_writer_unlock (&sm->segments_rwlock);
396
397 clib_rwlock_free (&sm->segments_rwlock);
Florin Corasc87c91d2017-08-16 19:55:49 -0700398 if (CLIB_DEBUG)
Dave Barachb7b92992018-10-17 10:38:51 -0400399 clib_memset (sm, 0xfe, sizeof (*sm));
Florin Corasa332c462018-01-31 06:52:17 -0800400 pool_put (smm->segment_managers, sm);
Florin Coras6cf30ad2017-04-04 23:08:23 -0700401}
402
Florin Corasc87c91d2017-08-16 19:55:49 -0700403void
Florin Coras88001c62019-04-24 14:44:46 -0700404segment_manager_init_free (segment_manager_t * sm)
Florin Coras6cf30ad2017-04-04 23:08:23 -0700405{
Florin Coras4e4531e2017-11-06 23:27:56 -0800406 segment_manager_app_detach (sm);
Florin Corasc87c91d2017-08-16 19:55:49 -0700407 if (segment_manager_has_fifos (sm))
408 segment_manager_del_sessions (sm);
409 else
410 {
Florin Coras9d063042017-09-14 03:08:00 -0400411 ASSERT (!sm->first_is_protected || segment_manager_app_detached (sm));
Florin Coras88001c62019-04-24 14:44:46 -0700412 segment_manager_free (sm);
Florin Corasc87c91d2017-08-16 19:55:49 -0700413 }
Florin Coras6cf30ad2017-04-04 23:08:23 -0700414}
415
Florin Coras88001c62019-04-24 14:44:46 -0700416segment_manager_t *
417segment_manager_get (u32 index)
418{
419 return pool_elt_at_index (sm_main.segment_managers, index);
420}
421
422segment_manager_t *
423segment_manager_get_if_valid (u32 index)
424{
425 if (pool_is_free_index (sm_main.segment_managers, index))
426 return 0;
427 return pool_elt_at_index (sm_main.segment_managers, index);
428}
429
430u32
431segment_manager_index (segment_manager_t * sm)
432{
433 return sm - sm_main.segment_managers;
434}
435
436u8
437segment_manager_has_fifos (segment_manager_t * sm)
438{
439 fifo_segment_t *seg;
440 u8 first = 1;
441
442 /* *INDENT-OFF* */
443 segment_manager_foreach_segment_w_lock (seg, sm, ({
444 if (CLIB_DEBUG && !first && !fifo_segment_has_fifos (seg)
445 && !(fifo_segment_flags (seg) & FIFO_SEGMENT_F_IS_PREALLOCATED))
446 {
447 clib_warning ("segment %d has no fifos!",
448 segment_manager_segment_index (sm, seg));
449 first = 0;
450 }
451 if (fifo_segment_has_fifos (seg))
452 {
453 segment_manager_segment_reader_unlock (sm);
454 return 1;
455 }
456 }));
457 /* *INDENT-ON* */
458
459 return 0;
460}
461
462/**
463 * Initiate disconnects for all sessions 'owned' by a segment manager
464 */
465void
466segment_manager_del_sessions (segment_manager_t * sm)
467{
468 fifo_segment_t *fifo_segment;
469 session_handle_t *handles = 0, *handle;
470 session_t *session;
471 svm_fifo_t *fifo;
472
473 ASSERT (pool_elts (sm->segments) != 0);
474
475 /* Across all fifo segments used by the server */
476 /* *INDENT-OFF* */
477 segment_manager_foreach_segment_w_lock (fifo_segment, sm, ({
478 fifo = fifo_segment_get_fifo_list (fifo_segment);
479
480 /*
481 * Remove any residual sessions from the session lookup table
482 * Don't bother deleting the individual fifos, we're going to
483 * throw away the fifo segment in a minute.
484 */
485 while (fifo)
486 {
487 session = session_get_if_valid (fifo->master_session_index,
488 fifo->master_thread_index);
489 if (session)
490 vec_add1 (handles, session_handle (session));
491 fifo = fifo->next;
492 }
493
494 /* Instead of removing the segment, test when cleaning up disconnected
495 * sessions if the segment can be removed.
496 */
497 }));
498 /* *INDENT-ON* */
499
500 vec_foreach (handle, handles)
501 session_close (session_get_from_handle (*handle));
502}
503
Florin Corasf8f516a2018-02-08 15:10:09 -0800504int
Florin Coras88001c62019-04-24 14:44:46 -0700505segment_manager_try_alloc_fifos (fifo_segment_t * fifo_segment,
Florin Corasf8f516a2018-02-08 15:10:09 -0800506 u32 rx_fifo_size, u32 tx_fifo_size,
507 svm_fifo_t ** rx_fifo, svm_fifo_t ** tx_fifo)
Florin Corasa332c462018-01-31 06:52:17 -0800508{
Florin Coras88001c62019-04-24 14:44:46 -0700509 rx_fifo_size = clib_max (rx_fifo_size, sm_main.default_fifo_size);
510 *rx_fifo = fifo_segment_alloc_fifo (fifo_segment, rx_fifo_size,
511 FIFO_SEGMENT_RX_FIFO);
Florin Corasa332c462018-01-31 06:52:17 -0800512
Florin Coras88001c62019-04-24 14:44:46 -0700513 tx_fifo_size = clib_max (tx_fifo_size, sm_main.default_fifo_size);
514 *tx_fifo = fifo_segment_alloc_fifo (fifo_segment, tx_fifo_size,
515 FIFO_SEGMENT_TX_FIFO);
Florin Corasa332c462018-01-31 06:52:17 -0800516
517 if (*rx_fifo == 0)
518 {
519 /* This would be very odd, but handle it... */
520 if (*tx_fifo != 0)
521 {
Florin Coras88001c62019-04-24 14:44:46 -0700522 fifo_segment_free_fifo (fifo_segment, *tx_fifo);
Florin Corasa332c462018-01-31 06:52:17 -0800523 *tx_fifo = 0;
524 }
525 return -1;
526 }
527 if (*tx_fifo == 0)
528 {
529 if (*rx_fifo != 0)
530 {
Florin Coras88001c62019-04-24 14:44:46 -0700531 fifo_segment_free_fifo (fifo_segment, *rx_fifo);
Florin Corasa332c462018-01-31 06:52:17 -0800532 *rx_fifo = 0;
533 }
534 return -1;
535 }
536
537 return 0;
538}
539
Florin Coras6cf30ad2017-04-04 23:08:23 -0700540int
541segment_manager_alloc_session_fifos (segment_manager_t * sm,
Florin Corasb384b542018-01-15 01:08:33 -0800542 svm_fifo_t ** rx_fifo,
Florin Coras1219b2d2019-04-23 15:53:43 -0700543 svm_fifo_t ** tx_fifo)
Florin Coras6cf30ad2017-04-04 23:08:23 -0700544{
Florin Corasa332c462018-01-31 06:52:17 -0800545 int alloc_fail = 1, rv = 0, new_fs_index;
Florin Coras88001c62019-04-24 14:44:46 -0700546 segment_manager_props_t *props;
547 fifo_segment_t *fs = 0;
548 u32 sm_index, fs_index;
Florin Coras6cf30ad2017-04-04 23:08:23 -0700549 u8 added_a_segment = 0;
Florin Coras88001c62019-04-24 14:44:46 -0700550 u64 fs_handle;
Florin Coras6cf30ad2017-04-04 23:08:23 -0700551
Florin Corasa332c462018-01-31 06:52:17 -0800552 props = segment_manager_properties_get (sm);
Florin Coras6cf30ad2017-04-04 23:08:23 -0700553
Florin Corasa332c462018-01-31 06:52:17 -0800554 /*
555 * Find the first free segment to allocate the fifos in
556 */
Florin Corasa5464812017-04-19 13:00:05 -0700557
Florin Corasa332c462018-01-31 06:52:17 -0800558 /* *INDENT-OFF* */
Florin Coras88001c62019-04-24 14:44:46 -0700559 segment_manager_foreach_segment_w_lock (fs, sm, ({
560 alloc_fail = segment_manager_try_alloc_fifos (fs,
Florin Corasf8f516a2018-02-08 15:10:09 -0800561 props->rx_fifo_size,
562 props->tx_fifo_size,
563 rx_fifo, tx_fifo);
Florin Corasa332c462018-01-31 06:52:17 -0800564 /* Exit with lock held, drop it after notifying app */
565 if (!alloc_fail)
566 goto alloc_success;
567 }));
568 /* *INDENT-ON* */
569
570alloc_check:
571
572 if (!alloc_fail)
Florin Coras6cf30ad2017-04-04 23:08:23 -0700573 {
Florin Coras6cf30ad2017-04-04 23:08:23 -0700574
Florin Corasa332c462018-01-31 06:52:17 -0800575 alloc_success:
Florin Coras6cf30ad2017-04-04 23:08:23 -0700576
Florin Corasa332c462018-01-31 06:52:17 -0800577 ASSERT (rx_fifo && tx_fifo);
578 sm_index = segment_manager_index (sm);
Florin Coras88001c62019-04-24 14:44:46 -0700579 fs_index = segment_manager_segment_index (sm, fs);
Florin Corasa332c462018-01-31 06:52:17 -0800580 (*tx_fifo)->segment_manager = sm_index;
581 (*rx_fifo)->segment_manager = sm_index;
Florin Coras88001c62019-04-24 14:44:46 -0700582 (*tx_fifo)->segment_index = fs_index;
583 (*rx_fifo)->segment_index = fs_index;
Florin Coras6cf30ad2017-04-04 23:08:23 -0700584
Florin Corasa332c462018-01-31 06:52:17 -0800585 if (added_a_segment)
Florin Corasfa76a762018-11-29 12:40:10 -0800586 {
Florin Coras2b81e3c2019-02-27 07:55:46 -0800587 app_worker_t *app_wrk;
Florin Coras88001c62019-04-24 14:44:46 -0700588 fs_handle = segment_manager_segment_handle (sm, fs);
Florin Coras2b81e3c2019-02-27 07:55:46 -0800589 app_wrk = app_worker_get (sm->app_wrk_index);
Florin Coras88001c62019-04-24 14:44:46 -0700590 rv = app_worker_add_segment_notify (app_wrk, fs_handle);
Florin Corasfa76a762018-11-29 12:40:10 -0800591 }
Florin Corasa332c462018-01-31 06:52:17 -0800592 /* Drop the lock after app is notified */
593 segment_manager_segment_reader_unlock (sm);
594 return rv;
Florin Coras6cf30ad2017-04-04 23:08:23 -0700595 }
596
Florin Corasa332c462018-01-31 06:52:17 -0800597 /*
598 * Allocation failed, see if we can add a new segment
599 */
600 if (props->add_segment)
Florin Coras6cf30ad2017-04-04 23:08:23 -0700601 {
Florin Corasa332c462018-01-31 06:52:17 -0800602 if (added_a_segment)
Florin Coras6cf30ad2017-04-04 23:08:23 -0700603 {
Florin Corasa332c462018-01-31 06:52:17 -0800604 clib_warning ("Added a segment, still can't allocate a fifo");
605 segment_manager_segment_reader_unlock (sm);
606 return SESSION_ERROR_NEW_SEG_NO_SPACE;
Florin Coras6cf30ad2017-04-04 23:08:23 -0700607 }
Florin Corasa332c462018-01-31 06:52:17 -0800608 if ((new_fs_index = segment_manager_add_segment (sm, 0)) < 0)
Florin Coras6cf30ad2017-04-04 23:08:23 -0700609 {
Florin Corasa332c462018-01-31 06:52:17 -0800610 clib_warning ("Failed to add new segment");
611 return SESSION_ERROR_SEG_CREATE;
Florin Coras6cf30ad2017-04-04 23:08:23 -0700612 }
Florin Coras88001c62019-04-24 14:44:46 -0700613 fs = segment_manager_get_segment_w_lock (sm, new_fs_index);
614 alloc_fail = segment_manager_try_alloc_fifos (fs, props->rx_fifo_size,
Florin Corasf8f516a2018-02-08 15:10:09 -0800615 props->tx_fifo_size,
616 rx_fifo, tx_fifo);
Florin Corasa332c462018-01-31 06:52:17 -0800617 added_a_segment = 1;
618 goto alloc_check;
Florin Coras6cf30ad2017-04-04 23:08:23 -0700619 }
Florin Corasa332c462018-01-31 06:52:17 -0800620 else
621 {
622 clib_warning ("Can't add new seg and no space to allocate fifos!");
623 return SESSION_ERROR_NO_SPACE;
624 }
Florin Coras6cf30ad2017-04-04 23:08:23 -0700625}
626
627void
Florin Coras19223e02019-03-03 14:56:05 -0800628segment_manager_dealloc_fifos (svm_fifo_t * rx_fifo, svm_fifo_t * tx_fifo)
Florin Coras6cf30ad2017-04-04 23:08:23 -0700629{
Florin Coras88001c62019-04-24 14:44:46 -0700630 fifo_segment_t *fs;
Florin Corasa332c462018-01-31 06:52:17 -0800631 segment_manager_t *sm;
Florin Coras19223e02019-03-03 14:56:05 -0800632 u32 segment_index;
Florin Corasa5464812017-04-19 13:00:05 -0700633
Florin Coras58a93e82019-01-14 23:33:46 -0800634 if (!rx_fifo || !tx_fifo)
635 return;
636
Florin Corasa5464812017-04-19 13:00:05 -0700637 /* It's possible to have no segment manager if the session was removed
Florin Corasc87c91d2017-08-16 19:55:49 -0700638 * as result of a detach. */
Florin Corasa332c462018-01-31 06:52:17 -0800639 if (!(sm = segment_manager_get_if_valid (rx_fifo->segment_manager)))
Florin Corasa5464812017-04-19 13:00:05 -0700640 return;
641
Florin Coras19223e02019-03-03 14:56:05 -0800642 segment_index = rx_fifo->segment_index;
Florin Coras88001c62019-04-24 14:44:46 -0700643 fs = segment_manager_get_segment_w_lock (sm, segment_index);
644 fifo_segment_free_fifo (fs, rx_fifo);
645 fifo_segment_free_fifo (fs, tx_fifo);
Florin Coras6cf30ad2017-04-04 23:08:23 -0700646
Florin Corasc87c91d2017-08-16 19:55:49 -0700647 /*
648 * Try to remove svm segment if it has no fifos. This can be done only if
649 * the segment is not the first in the segment manager or if it is first
650 * and it is not protected. Moreover, if the segment is first and the app
651 * has detached from the segment manager, remove the segment manager.
652 */
Florin Coras88001c62019-04-24 14:44:46 -0700653 if (!fifo_segment_has_fifos (fs))
Florin Coras6cf30ad2017-04-04 23:08:23 -0700654 {
Florin Corasa332c462018-01-31 06:52:17 -0800655 segment_manager_segment_reader_unlock (sm);
Florin Corasc87c91d2017-08-16 19:55:49 -0700656
657 /* Remove segment if it holds no fifos or first but not protected */
Florin Corasa332c462018-01-31 06:52:17 -0800658 if (segment_index != 0 || !sm->first_is_protected)
659 segment_manager_lock_and_del_segment (sm, segment_index);
Florin Corasc87c91d2017-08-16 19:55:49 -0700660
661 /* Remove segment manager if no sessions and detached from app */
Florin Coras9d063042017-09-14 03:08:00 -0400662 if (segment_manager_app_detached (sm)
663 && !segment_manager_has_fifos (sm))
Florin Coras568ebc72018-09-18 16:12:50 -0700664 {
Florin Coras88001c62019-04-24 14:44:46 -0700665 segment_manager_free (sm);
Florin Coras568ebc72018-09-18 16:12:50 -0700666 }
Florin Coras6cf30ad2017-04-04 23:08:23 -0700667 }
Florin Corasa332c462018-01-31 06:52:17 -0800668 else
669 segment_manager_segment_reader_unlock (sm);
Florin Coras6cf30ad2017-04-04 23:08:23 -0700670}
671
Florin Coras3c2fed52018-07-04 04:15:05 -0700672u32
673segment_manager_evt_q_expected_size (u32 q_len)
674{
675 u32 fifo_evt_size, notif_q_size, q_hdrs;
676 u32 msg_q_sz, fifo_evt_ring_sz, session_ntf_ring_sz;
677
Florin Coras52207f12018-07-12 14:48:06 -0700678 fifo_evt_size = 1 << max_log2 (sizeof (session_event_t));
Florin Coras3c2fed52018-07-04 04:15:05 -0700679 notif_q_size = clib_max (16, q_len >> 4);
680
681 msg_q_sz = q_len * sizeof (svm_msg_q_msg_t);
682 fifo_evt_ring_sz = q_len * fifo_evt_size;
683 session_ntf_ring_sz = notif_q_size * 256;
684 q_hdrs = sizeof (svm_queue_t) + sizeof (svm_msg_q_t);
685
686 return (msg_q_sz + fifo_evt_ring_sz + session_ntf_ring_sz + q_hdrs);
687}
688
Florin Corasa5464812017-04-19 13:00:05 -0700689/**
690 * Allocates shm queue in the first segment
Florin Corasa332c462018-01-31 06:52:17 -0800691 *
692 * Must be called with lock held
Florin Corasa5464812017-04-19 13:00:05 -0700693 */
Florin Coras3c2fed52018-07-04 04:15:05 -0700694svm_msg_q_t *
Florin Coras88001c62019-04-24 14:44:46 -0700695segment_manager_alloc_queue (fifo_segment_t * segment,
696 segment_manager_props_t * props)
Florin Corasa5464812017-04-19 13:00:05 -0700697{
Florin Coras3c2fed52018-07-04 04:15:05 -0700698 u32 fifo_evt_size, session_evt_size = 256, notif_q_size;
699 svm_msg_q_cfg_t _cfg, *cfg = &_cfg;
700 svm_msg_q_t *q;
Florin Corasa5464812017-04-19 13:00:05 -0700701 void *oldheap;
702
Florin Coras52207f12018-07-12 14:48:06 -0700703 fifo_evt_size = sizeof (session_event_t);
Florin Coras99368312018-08-02 10:45:44 -0700704 notif_q_size = clib_max (16, props->evt_q_size >> 4);
Florin Coras3c2fed52018-07-04 04:15:05 -0700705 /* *INDENT-OFF* */
706 svm_msg_q_ring_cfg_t rc[SESSION_MQ_N_RINGS] = {
Florin Coras99368312018-08-02 10:45:44 -0700707 {props->evt_q_size, fifo_evt_size, 0},
Florin Coras3c2fed52018-07-04 04:15:05 -0700708 {notif_q_size, session_evt_size, 0}
709 };
710 /* *INDENT-ON* */
711 cfg->consumer_pid = 0;
712 cfg->n_rings = 2;
Florin Coras99368312018-08-02 10:45:44 -0700713 cfg->q_nitems = props->evt_q_size;
Florin Coras3c2fed52018-07-04 04:15:05 -0700714 cfg->ring_cfgs = rc;
Florin Corasa5464812017-04-19 13:00:05 -0700715
Florin Coras3c2fed52018-07-04 04:15:05 -0700716 oldheap = ssvm_push_heap (segment->ssvm.sh);
717 q = svm_msg_q_alloc (cfg);
Florin Corasa5464812017-04-19 13:00:05 -0700718 ssvm_pop_heap (oldheap);
Florin Coras99368312018-08-02 10:45:44 -0700719
720 if (props->use_mq_eventfd)
721 {
722 if (svm_msg_q_alloc_producer_eventfd (q))
723 clib_warning ("failed to alloc eventfd");
724 }
Florin Corasa5464812017-04-19 13:00:05 -0700725 return q;
726}
727
Florin Coras88001c62019-04-24 14:44:46 -0700728svm_msg_q_t *
729segment_manager_event_queue (segment_manager_t * sm)
730{
731 return sm->event_queue;
732}
733
Florin Corasa5464812017-04-19 13:00:05 -0700734/**
735 * Frees shm queue allocated in the first segment
736 */
737void
Florin Corase86a8ed2018-01-05 03:20:25 -0800738segment_manager_dealloc_queue (segment_manager_t * sm, svm_queue_t * q)
Florin Corasa5464812017-04-19 13:00:05 -0700739{
Florin Coras88001c62019-04-24 14:44:46 -0700740 fifo_segment_t *segment;
Florin Corasa332c462018-01-31 06:52:17 -0800741 ssvm_shared_header_t *sh;
Florin Corasa5464812017-04-19 13:00:05 -0700742 void *oldheap;
743
Florin Corasa332c462018-01-31 06:52:17 -0800744 ASSERT (!pool_is_free_index (sm->segments, 0));
Florin Corasa5464812017-04-19 13:00:05 -0700745
Florin Corasa332c462018-01-31 06:52:17 -0800746 segment = segment_manager_get_segment_w_lock (sm, 0);
Florin Corasa5464812017-04-19 13:00:05 -0700747 sh = segment->ssvm.sh;
748
749 oldheap = ssvm_push_heap (sh);
Florin Corase86a8ed2018-01-05 03:20:25 -0800750 svm_queue_free (q);
Florin Corasa5464812017-04-19 13:00:05 -0700751 ssvm_pop_heap (oldheap);
Florin Corasa332c462018-01-31 06:52:17 -0800752 segment_manager_segment_reader_unlock (sm);
753}
754
755/*
756 * Init segment vm address allocator
757 */
758void
759segment_manager_main_init (segment_manager_main_init_args_t * a)
760{
Florin Coras88001c62019-04-24 14:44:46 -0700761 segment_manager_main_t *sm = &sm_main;
Florin Corasa332c462018-01-31 06:52:17 -0800762 clib_valloc_chunk_t _ip, *ip = &_ip;
763
764 ip->baseva = a->baseva;
765 ip->size = a->size;
766
767 clib_valloc_init (&sm->va_allocator, ip, 1 /* lock */ );
Florin Coras88001c62019-04-24 14:44:46 -0700768
769 sm->default_fifo_size = 1 << 12;
770 sm->default_segment_size = 1 << 20;
771 sm->default_app_mq_size = 128;
Florin Corasa5464812017-04-19 13:00:05 -0700772}
773
Florin Corasc87c91d2017-08-16 19:55:49 -0700774static clib_error_t *
775segment_manager_show_fn (vlib_main_t * vm, unformat_input_t * input,
776 vlib_cli_command_t * cmd)
777{
Florin Coras88001c62019-04-24 14:44:46 -0700778 segment_manager_main_t *smm = &sm_main;
779 fifo_segment_t *seg;
Florin Corasc87c91d2017-08-16 19:55:49 -0700780 segment_manager_t *sm;
Florin Corasb384b542018-01-15 01:08:33 -0800781 u8 show_segments = 0, verbose = 0;
David Johnsond9818dd2018-12-14 14:53:41 -0500782 char *address;
783 size_t size;
Dave Barach91f3e742017-09-01 19:12:11 -0400784 u32 active_fifos;
785 u32 free_fifos;
786
Florin Corasc87c91d2017-08-16 19:55:49 -0700787 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
788 {
789 if (unformat (input, "segments"))
790 show_segments = 1;
791 else if (unformat (input, "verbose"))
792 verbose = 1;
793 else
794 return clib_error_return (0, "unknown input `%U'",
795 format_unformat_error, input);
796 }
797 vlib_cli_output (vm, "%d segment managers allocated",
Florin Corasa332c462018-01-31 06:52:17 -0800798 pool_elts (smm->segment_managers));
799 if (verbose && pool_elts (smm->segment_managers))
Florin Corasc87c91d2017-08-16 19:55:49 -0700800 {
801 vlib_cli_output (vm, "%-10s%=15s%=12s", "Index", "App Index",
802 "Segments");
803
804 /* *INDENT-OFF* */
Florin Corasa332c462018-01-31 06:52:17 -0800805 pool_foreach (sm, smm->segment_managers, ({
Florin Coras15531972018-08-12 23:50:53 -0700806 vlib_cli_output (vm, "%-10d%=15d%=12d", segment_manager_index (sm),
807 sm->app_wrk_index, pool_elts (sm->segments));
Florin Corasc87c91d2017-08-16 19:55:49 -0700808 }));
809 /* *INDENT-ON* */
810
811 }
812 if (show_segments)
813 {
Florin Coras2f8d8fa2018-01-26 06:36:04 -0800814 vlib_cli_output (vm, "%-15s%15s%15s%15s%15s%15s", "Name", "Type",
Dave Barach91f3e742017-09-01 19:12:11 -0400815 "HeapSize (M)", "ActiveFifos", "FreeFifos", "Address");
Florin Corasc87c91d2017-08-16 19:55:49 -0700816
817 /* *INDENT-OFF* */
Florin Corasa332c462018-01-31 06:52:17 -0800818 pool_foreach (sm, smm->segment_managers, ({
819 segment_manager_foreach_segment_w_lock (seg, sm, ({
Florin Coras88001c62019-04-24 14:44:46 -0700820 fifo_segment_info (seg, &address, &size);
821 active_fifos = fifo_segment_num_fifos (seg);
822 free_fifos = fifo_segment_num_free_fifos (seg, ~0 /* size */);
Florin Corasa332c462018-01-31 06:52:17 -0800823 vlib_cli_output (vm, "%-15v%15U%15llu%15u%15u%15llx",
Florin Coras88001c62019-04-24 14:44:46 -0700824 ssvm_name (&seg->ssvm), format_fifo_segment_type,
Florin Corasa332c462018-01-31 06:52:17 -0800825 seg, size >> 20ULL, active_fifos, free_fifos,
826 address);
827 if (verbose)
Florin Coras88001c62019-04-24 14:44:46 -0700828 vlib_cli_output (vm, "%U", format_fifo_segment, seg, verbose);
Florin Corasa332c462018-01-31 06:52:17 -0800829 }));
Florin Corasc87c91d2017-08-16 19:55:49 -0700830 }));
831 /* *INDENT-ON* */
832
833 }
834 return 0;
835}
836
Florin Corasad0c77f2017-11-09 18:00:15 -0800837/* *INDENT-OFF* */
Florin Corasc87c91d2017-08-16 19:55:49 -0700838VLIB_CLI_COMMAND (segment_manager_show_command, static) =
839{
840 .path = "show segment-manager",
Dave Barach91f3e742017-09-01 19:12:11 -0400841 .short_help = "show segment-manager [segments][verbose]",
Florin Corasc87c91d2017-08-16 19:55:49 -0700842 .function = segment_manager_show_fn,
843};
844/* *INDENT-ON* */
845
Florin Coras88001c62019-04-24 14:44:46 -0700846void
847segment_manager_format_sessions (segment_manager_t * sm, int verbose)
848{
849 fifo_segment_t *fifo_segment;
850 vlib_main_t *vm = vlib_get_main ();
851 app_worker_t *app_wrk;
852 const u8 *app_name;
853 u8 *s = 0;
854
855 if (!sm)
856 {
857 if (verbose)
858 vlib_cli_output (vm, "%-40s%-20s%-15s%-10s", "Connection", "App",
859 "API Client", "SegManager");
860 else
861 vlib_cli_output (vm, "%-40s%-20s", "Connection", "App");
862 return;
863 }
864
865 app_wrk = app_worker_get (sm->app_wrk_index);
866 app_name = application_name_from_index (app_wrk->app_index);
867
868 clib_rwlock_reader_lock (&sm->segments_rwlock);
869
870 /* *INDENT-OFF* */
871 pool_foreach (fifo_segment, sm->segments, ({
872 svm_fifo_t *fifo;
873 u8 *str;
874
875 fifo = fifo_segment_get_fifo_list (fifo_segment);
876 while (fifo)
877 {
878 u32 session_index, thread_index;
879 session_t *session;
880
881 session_index = fifo->master_session_index;
882 thread_index = fifo->master_thread_index;
883
884 session = session_get (session_index, thread_index);
885 str = format (0, "%U", format_session, session, verbose);
886
887 if (verbose)
888 s = format (s, "%-40s%-20s%-15u%-10u", str, app_name,
889 app_wrk->api_client_index, app_wrk->connects_seg_manager);
890 else
891 s = format (s, "%-40s%-20s", str, app_name);
892
893 vlib_cli_output (vm, "%v", s);
894 vec_reset_length (s);
895 vec_free (str);
896
897 fifo = fifo->next;
898 }
899 vec_free (s);
900 }));
901 /* *INDENT-ON* */
902
903 clib_rwlock_reader_unlock (&sm->segments_rwlock);
904}
905
Florin Coras6cf30ad2017-04-04 23:08:23 -0700906/*
907 * fd.io coding-style-patch-verification: ON
908 *
909 * Local Variables:
910 * eval: (c-set-style "gnu")
911 * End:
912 */