rdma: implement multiseg rx without striding rq
Change-Id: I623617ad3c80610805dd3cf2a5f371e6677f4844
Signed-off-by: Mohammed Hawari <mohammed@hawari.fr>
Type: improvement
diff --git a/src/plugins/rdma/api.c b/src/plugins/rdma/api.c
index fe1103f..06e7385 100644
--- a/src/plugins/rdma/api.c
+++ b/src/plugins/rdma/api.c
@@ -45,6 +45,37 @@
}
static void
+vl_api_rdma_create_v2_t_handler (vl_api_rdma_create_v2_t * mp)
+{
+ vlib_main_t *vm = vlib_get_main ();
+ rdma_main_t *rm = &rdma_main;
+ vl_api_rdma_create_v2_reply_t *rmp;
+ rdma_create_if_args_t args;
+ int rv;
+
+ clib_memset (&args, 0, sizeof (rdma_create_if_args_t));
+
+ args.ifname = mp->host_if;
+ args.name = mp->name;
+ args.rxq_num = ntohs (mp->rxq_num);
+ args.rxq_size = ntohs (mp->rxq_size);
+ args.txq_size = ntohs (mp->txq_size);
+ args.mode = rdma_api_mode (mp->mode);
+ args.disable_striding_rq = 0;
+ args.no_multi_seg = mp->no_multi_seg;
+ args.max_pktlen = ntohs (mp->max_pktlen);
+ rdma_create_if (vm, &args);
+ rv = args.rv;
+
+ /* *INDENT-OFF* */
+ REPLY_MACRO2 (VL_API_RDMA_CREATE_V2_REPLY + rm->msg_id_base,
+ ({
+ rmp->sw_if_index = ntohl (args.sw_if_index);
+ }));
+ /* *INDENT-ON* */
+}
+
+static void
vl_api_rdma_create_t_handler (vl_api_rdma_create_t * mp)
{
vlib_main_t *vm = vlib_get_main ();
@@ -61,6 +92,9 @@
args.rxq_size = ntohs (mp->rxq_size);
args.txq_size = ntohs (mp->txq_size);
args.mode = rdma_api_mode (mp->mode);
+ args.disable_striding_rq = 0;
+ args.no_multi_seg = 1;
+ args.max_pktlen = 0;
rdma_create_if (vm, &args);
rv = args.rv;
diff --git a/src/plugins/rdma/cli.c b/src/plugins/rdma/cli.c
index 918cd77..8f191e3 100644
--- a/src/plugins/rdma/cli.c
+++ b/src/plugins/rdma/cli.c
@@ -49,7 +49,9 @@
.path = "create interface rdma",
.short_help = "create interface rdma <host-if ifname> [name <name>]"
" [rx-queue-size <size>] [tx-queue-size <size>]"
- " [num-rx-queues <size>] [mode <auto|ibv|dv]",
+ " [num-rx-queues <size>] [mode <auto|ibv|dv]"
+ " [no-multi-seg] [no-striding]"
+ " [max-pktlen <size>]",
.function = rdma_create_command_fn,
};
/* *INDENT-ON* */
diff --git a/src/plugins/rdma/device.c b/src/plugins/rdma/device.c
index 9b6fda9..043232d 100644
--- a/src/plugins/rdma/device.c
+++ b/src/plugins/rdma/device.c
@@ -419,7 +419,8 @@
}
static clib_error_t *
-rdma_rxq_init (vlib_main_t * vm, rdma_device_t * rd, u16 qid, u32 n_desc)
+rdma_rxq_init (vlib_main_t * vm, rdma_device_t * rd, u16 qid, u32 n_desc,
+ u8 no_multi_seg, u16 max_pktlen)
{
rdma_rxq_t *rxq;
struct ibv_wq_init_attr wqia;
@@ -427,17 +428,18 @@
struct ibv_wq_attr wqa;
struct ibv_cq_ex *cqex;
struct mlx5dv_wq_init_attr dv_wqia = { };
+ int is_mlx5dv = ! !(rd->flags & RDMA_DEVICE_F_MLX5DV);
+ int is_striding = ! !(rd->flags & RDMA_DEVICE_F_STRIDING_RQ);
vec_validate_aligned (rd->rxqs, qid, CLIB_CACHE_LINE_BYTES);
rxq = vec_elt_at_index (rd->rxqs, qid);
rxq->size = n_desc;
rxq->log_wqe_sz = 0;
- rxq->log_stride_per_wqe = 0;
rxq->buf_sz = vlib_buffer_get_default_data_size (vm);
vec_validate_aligned (rxq->bufs, n_desc - 1, CLIB_CACHE_LINE_BYTES);
cqa.cqe = n_desc;
- if (rd->flags & RDMA_DEVICE_F_MLX5DV)
+ if (is_mlx5dv)
{
struct mlx5dv_cq_init_attr dvcq = { };
dvcq.comp_mask = MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE;
@@ -460,14 +462,14 @@
wqia.max_sge = 1;
wqia.pd = rd->pd;
wqia.cq = rxq->cq;
- if (rd->flags & RDMA_DEVICE_F_MLX5DV)
+ if (is_mlx5dv)
{
- if (rd->flags & RDMA_DEVICE_F_STRIDING_RQ)
+ if (is_striding)
{
/* In STRIDING_RQ mode, map a descriptor to a stride, not a full WQE buffer */
uword data_seg_log2_sz =
min_log2 (vlib_buffer_get_default_data_size (vm));
-
+ rxq->buf_sz = 1 << data_seg_log2_sz;
/* The trick is also to map a descriptor to a data segment in the WQE SG list
The number of strides per WQE and the size of a WQE (in 16-bytes words) both
must be powers of two.
@@ -478,24 +480,44 @@
a stride, and a vlib_buffer)
- RDMA_RXQ_MAX_CHAIN_SZ-1 null data segments
*/
-
- wqia.max_sge = RDMA_RXQ_MAX_CHAIN_SZ;
+ int max_chain_log_sz =
+ max_pktlen ? max_log2 ((max_pktlen /
+ (rxq->buf_sz)) +
+ 1) : RDMA_RXQ_MAX_CHAIN_LOG_SZ;
+ max_chain_log_sz = clib_max (max_chain_log_sz, 3);
+ wqia.max_sge = 1 << max_chain_log_sz;
dv_wqia.comp_mask = MLX5DV_WQ_INIT_ATTR_MASK_STRIDING_RQ;
dv_wqia.striding_rq_attrs.two_byte_shift_en = 0;
dv_wqia.striding_rq_attrs.single_wqe_log_num_of_strides =
- RDMA_RXQ_MAX_CHAIN_LOG_SZ;
+ max_chain_log_sz;
dv_wqia.striding_rq_attrs.single_stride_log_num_of_bytes =
data_seg_log2_sz;
- wqia.max_wr >>= RDMA_RXQ_MAX_CHAIN_LOG_SZ;
- rxq->log_wqe_sz = RDMA_RXQ_MAX_CHAIN_LOG_SZ + 1;
- rxq->log_stride_per_wqe = RDMA_RXQ_MAX_CHAIN_LOG_SZ;
- rxq->buf_sz = 1 << data_seg_log2_sz;
+ wqia.max_wr >>= max_chain_log_sz;
+ rxq->log_wqe_sz = max_chain_log_sz + 1;
+ rxq->log_stride_per_wqe = max_chain_log_sz;
}
else
{
- /* For now, in non STRIDING_RQ mode, SG operations/chained buffers
- are not supported */
- wqia.max_sge = 1;
+ /* In non STRIDING_RQ mode and if multiseg is not disabled, each WQE is a SG list of data
+ segments, each pointing to a vlib_buffer. */
+ if (no_multi_seg)
+ {
+ wqia.max_sge = 1;
+ rxq->log_wqe_sz = 0;
+ rxq->n_ds_per_wqe = 1;
+ }
+ else
+ {
+ int max_chain_sz =
+ max_pktlen ? (max_pktlen /
+ (rxq->buf_sz)) +
+ 1 : RDMA_RXQ_LEGACY_MODE_MAX_CHAIN_SZ;
+ int max_chain_log_sz = max_log2 (max_chain_sz);
+ wqia.max_sge = 1 << max_chain_log_sz;
+ rxq->log_wqe_sz = max_chain_log_sz;
+ rxq->n_ds_per_wqe = max_chain_sz;
+ }
+
}
if ((rxq->wq = mlx5dv_create_wq (rd->ctx, &wqia, &dv_wqia)))
@@ -516,13 +538,14 @@
if (ibv_modify_wq (rxq->wq, &wqa) != 0)
return clib_error_return_unix (0, "Modify WQ (RDY) Failed");
- if (rd->flags & RDMA_DEVICE_F_MLX5DV)
+ if (is_mlx5dv)
{
struct mlx5dv_obj obj = { };
struct mlx5dv_cq dv_cq;
struct mlx5dv_rwq dv_rwq;
u64 qw0;
u64 qw0_nullseg;
+ u32 wqe_sz_mask = (1 << rxq->log_wqe_sz) - 1;
obj.cq.in = rxq->cq;
obj.cq.out = &dv_cq;
@@ -550,19 +573,36 @@
qw0 |= (u64) clib_host_to_net_u32 (rd->lkey) << 32;
qw0_nullseg |= (u64) clib_host_to_net_u32 (rd->lkey) << 32;
-/* Prefill the different 16 bytes words of the WQ. If not in striding RQ mode,
- init with qw0 only with segments of rxq->buf_sz. Otherwise, for each WQE, the
- RDMA_RXQ_MAX_CHAIN_SZ + 1 first 16-bytes words are initialised with qw0, the rest
- are null segments */
+/* Prefill the different 16 bytes words of the WQ.
+ - If not in striding RQ mode, for each WQE, init with qw0 the first
+ RDMA_RXQ_LEGACY_MODE_MAX_CHAIN_SZ, and init the rest of the WQE
+ with null segments.
+ - If in striding RQ mode, for each WQE, the RDMA_RXQ_MAX_CHAIN_SZ + 1
+ first 16-bytes words are initialised with qw0, the rest are null segments */
+
for (int i = 0; i < rxq->wqe_cnt << rxq->log_wqe_sz; i++)
- if (!(rd->flags & RDMA_DEVICE_F_STRIDING_RQ)
- || (i == 0) || !(((i - 1) >> rxq->log_stride_per_wqe) & 0x1))
+ if ((!is_striding
+ && ((i & wqe_sz_mask) < rxq->n_ds_per_wqe))
+ || (is_striding
+ && ((i == 0)
+ || !(((i - 1) >> rxq->log_stride_per_wqe) & 0x1))))
rxq->wqes[i].dsz_and_lkey = qw0;
else
rxq->wqes[i].dsz_and_lkey = qw0_nullseg;
for (int i = 0; i < (1 << rxq->log2_cq_size); i++)
rxq->cqes[i].opcode_cqefmt_se_owner = 0xff;
+
+ if (!is_striding)
+ {
+ vec_validate_aligned (rxq->second_bufs, n_desc - 1,
+ CLIB_CACHE_LINE_BYTES);
+ vec_validate_aligned (rxq->n_used_per_chain, n_desc - 1,
+ CLIB_CACHE_LINE_BYTES);
+ rxq->n_total_additional_segs = n_desc * (rxq->n_ds_per_wqe - 1);
+ for (int i = 0; i < n_desc; i++)
+ rxq->n_used_per_chain[i] = rxq->n_ds_per_wqe - 1;
+ }
}
return 0;
@@ -719,12 +759,15 @@
}
static clib_error_t *
-rdma_dev_init (vlib_main_t * vm, rdma_device_t * rd, u32 rxq_size,
- u32 txq_size, u32 rxq_num)
+rdma_dev_init (vlib_main_t * vm, rdma_device_t * rd,
+ rdma_create_if_args_t * args)
{
clib_error_t *err;
vlib_buffer_main_t *bm = vm->buffer_main;
vlib_thread_main_t *tm = vlib_get_thread_main ();
+ u32 rxq_num = args->rxq_num;
+ u32 rxq_size = args->rxq_size;
+ u32 txq_size = args->txq_size;
u32 i;
if (rd->ctx == 0)
@@ -758,7 +801,9 @@
return err;
for (i = 0; i < rxq_num; i++)
- if ((err = rdma_rxq_init (vm, rd, i, rxq_size)))
+ if ((err =
+ rdma_rxq_init (vm, rd, i, rxq_size,
+ args->no_multi_seg, args->max_pktlen)))
return err;
if ((err = rdma_rxq_finalize (vm, rd)))
return err;
@@ -799,7 +844,7 @@
args->rxq_size = args->rxq_size ? args->rxq_size : 1024;
args->txq_size = args->txq_size ? args->txq_size : 1024;
- args->rxq_num = args->rxq_num ? args->rxq_num : 1;
+ args->rxq_num = args->rxq_num ? args->rxq_num : 2;
if (!is_pow2 (args->rxq_num))
{
@@ -896,7 +941,10 @@
if ((mlx5dv_attrs.flags & MLX5DV_CONTEXT_FLAGS_CQE_V1))
rd->flags |= RDMA_DEVICE_F_MLX5DV;
- if (data_seg_log2_sz <=
+/* Enable striding RQ if neither multiseg nor striding rq
+are explicitly disabled, and if the interface supports it.*/
+ if (!args->no_multi_seg && !args->disable_striding_rq
+ && data_seg_log2_sz <=
mlx5dv_attrs.striding_rq_caps.max_single_stride_log_num_of_bytes
&& data_seg_log2_sz >=
mlx5dv_attrs.striding_rq_caps.min_single_stride_log_num_of_bytes
@@ -917,8 +965,7 @@
}
}
- if ((args->error = rdma_dev_init (vm, rd, args->rxq_size, args->txq_size,
- args->rxq_num)))
+ if ((args->error = rdma_dev_init (vm, rd, args)))
goto err2;
if ((args->error = rdma_register_interface (vnm, rd)))
diff --git a/src/plugins/rdma/input.c b/src/plugins/rdma/input.c
index 52e921d..d7fbe96 100644
--- a/src/plugins/rdma/input.c
+++ b/src/plugins/rdma/input.c
@@ -53,21 +53,106 @@
w[0].num_sge = 1;
}
+static_always_inline u32
+rdma_device_legacy_input_refill_additional (vlib_main_t * vm,
+ rdma_device_t * rd,
+ rdma_rxq_t * rxq,
+ rdma_per_thread_data_t * ptd,
+ vlib_buffer_t * bt,
+ u32 first_slot, u32 n_alloc)
+{
+ int i;
+ u8 log_wqe_sz = rxq->log_wqe_sz;
+ u32 *bi = ptd->tmp_bi;
+ vlib_buffer_t **bufs = ptd->tmp_bufs;
+
+ for (i = 0; i < n_alloc; i++)
+ {
+ u8 chain_sz = rxq->n_used_per_chain[first_slot + i];
+ u8 chain_sz_alloc;
+ mlx5dv_wqe_ds_t *current_wqe =
+ rxq->wqes + ((first_slot + i) << log_wqe_sz);
+ if (chain_sz == 0)
+ continue;
+ if (PREDICT_FALSE ((chain_sz_alloc =
+ vlib_buffer_alloc_from_pool (vm, bi, chain_sz,
+ rd->pool)) !=
+ chain_sz))
+ {
+ vlib_buffer_free (vm, bi, chain_sz_alloc);
+ break;
+ }
+ /*Build the chain */
+ vlib_get_buffers (vm, bi, bufs, chain_sz);
+ for (int j = 0; j < chain_sz - 1; j++)
+ {
+ vlib_buffer_copy_template (bufs[j], bt);
+ bufs[j]->next_buffer = bi[j + 1];
+ bufs[j]->flags |= VLIB_BUFFER_NEXT_PRESENT;
+ }
+ /* The chain starting at the second buffer is pre-initialised */
+ vlib_buffer_copy_template (bufs[chain_sz - 1], bt);
+ /* Stick with the already existing chain */
+ if (chain_sz < rxq->n_ds_per_wqe - 1)
+ {
+ bufs[chain_sz - 1]->next_buffer = rxq->second_bufs[first_slot + i];
+ bufs[chain_sz - 1]->flags |= VLIB_BUFFER_NEXT_PRESENT;
+ }
+ else
+ {
+ bufs[chain_sz - 1]->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
+ }
+
+ /* Update the wqes */
+ for (int j = 0; j < chain_sz; j++)
+ {
+ u64 addr;
+ vlib_get_buffers_with_offset (vm, bi + j,
+ (void *) &addr, 1,
+ sizeof (vlib_buffer_t));
+ current_wqe[j + 1].addr = clib_host_to_net_u64 (addr);
+ }
+ rxq->n_used_per_chain[first_slot + i] = 0;
+ rxq->n_total_additional_segs -= chain_sz;
+ rxq->second_bufs[first_slot + i] = bi[0];
+ }
+ return i;
+}
+
static_always_inline void
rdma_device_input_refill (vlib_main_t * vm, rdma_device_t * rd,
- rdma_rxq_t * rxq, int is_mlx5dv, int is_striding)
+ rdma_rxq_t * rxq, vlib_buffer_t * bt,
+ const int is_mlx5dv, const int is_striding)
{
u32 n_alloc, n;
u16 ring_space;
struct ibv_recv_wr wr[VLIB_FRAME_SIZE], *w = wr;
struct ibv_sge sge[VLIB_FRAME_SIZE], *s = sge;
+ rdma_per_thread_data_t *ptd =
+ &rdma_main.per_thread_data[vlib_get_thread_index ()];
u32 mask = rxq->size - 1;
u32 slot = rxq->tail & mask;
u32 *bufs = rxq->bufs + slot;
u32 data_size = rxq->buf_sz;
u32 lkey = rd->lkey;
- int log_stride_per_wqe = rxq->log_stride_per_wqe;
- int log_wqe_sz = rxq->log_wqe_sz;
+ const int log_stride_per_wqe = is_striding ? rxq->log_stride_per_wqe : 0;
+ const int log_wqe_sz = rxq->log_wqe_sz;
+
+ /*In legacy mode, maybe some buffers chains are incomplete? */
+ if (PREDICT_FALSE
+ (is_mlx5dv && !is_striding && (rxq->incomplete_tail != rxq->tail)))
+ {
+ int n_incomplete = rxq->incomplete_tail - rxq->tail;
+ int n_completed =
+ rdma_device_legacy_input_refill_additional (vm, rd, rxq, ptd, bt,
+ slot,
+ n_incomplete);
+ rxq->tail += n_completed;
+ slot = rxq->tail & mask;
+ /* Don't start recycling head buffers if there are incomplete chains */
+ if (n_completed != n_incomplete)
+ return;
+ }
/* refilled buffers must be a multiple of 8 and of strides per WQE */
u32 alloc_multiple = 1 << (clib_max (3, log_stride_per_wqe));
@@ -115,12 +200,16 @@
/* slot does not necessarily correspond to the slot
in the wqes ring (in 16B words) */
u32 wqes_slot = slot << (log_wqe_sz - log_stride_per_wqe);
- u32 wqe_cnt = rxq->wqe_cnt;
+ const u32 wqe_cnt = rxq->wqe_cnt;
mlx5dv_wqe_ds_t *wqe = rxq->wqes + wqes_slot;
- int wqe_sz = 1 << log_wqe_sz;
- int stride_per_wqe = 1 << log_stride_per_wqe;
+ const int wqe_sz = 1 << log_wqe_sz;
+ const int stride_per_wqe = 1 << log_stride_per_wqe;
int current_data_seg = 0;
+ /* In legacy mode, this function only refills head descriptors for each
+ WQE, so RDMA_RXQ_MAX_CHAIN_SZ-1 data segments are skipped per WQE */
+ const int log_skip_wqe = is_striding ? 0 : log_wqe_sz;
+
while (n >= 1)
{
vlib_get_buffers_with_offset (vm, rxq->bufs + slot, (void **) va, 8,
@@ -149,25 +238,38 @@
};
/* *INDENT-ON* */
+ /* TODO: when log_skip_wqe > 2, hw_prefetcher doesn't work, lots of LLC store
+ misses occur for wqes, to be fixed... */
if (!is_striding || !(current_data_seg & ~(stride_per_wqe - 1)))
{
- wqe[0 + is_striding].addr = va[0];
- wqe[1 + is_striding].addr = va[1];
- wqe[2 + is_striding].addr = va[2];
- wqe[3 + is_striding].addr = va[3];
- wqe[4 + is_striding].addr = va[4];
- wqe[5 + is_striding].addr = va[5];
- wqe[6 + is_striding].addr = va[6];
- wqe[7 + is_striding].addr = va[7];
+ wqe[(0 << log_skip_wqe) + is_striding].addr = va[0];
+ wqe[(1 << log_skip_wqe) + is_striding].addr = va[1];
+ wqe[(2 << log_skip_wqe) + is_striding].addr = va[2];
+ wqe[(3 << log_skip_wqe) + is_striding].addr = va[3];
+ wqe[(4 << log_skip_wqe) + is_striding].addr = va[4];
+ wqe[(5 << log_skip_wqe) + is_striding].addr = va[5];
+ wqe[(6 << log_skip_wqe) + is_striding].addr = va[6];
+ wqe[(7 << log_skip_wqe) + is_striding].addr = va[7];
slot += 8;
n -= 8;
}
- wqe += 8;
- wqes_slot += 8;
+ wqe += 8 << log_skip_wqe;
+ wqes_slot += 8 << log_skip_wqe;
current_data_seg += 8;
current_data_seg &= wqe_sz - 1;
}
+ /* In legacy mode, there is some work required to finish building the SG lists */
+ if (!is_striding)
+ {
+ int first_slot = slot - n_alloc;
+ rxq->incomplete_tail += n_alloc;
+ if (PREDICT_FALSE (rxq->n_total_additional_segs))
+ n_alloc =
+ rdma_device_legacy_input_refill_additional (vm, rd, rxq, ptd,
+ bt, first_slot,
+ n_alloc);
+ }
CLIB_MEMORY_STORE_BARRIER ();
rxq->tail += n_alloc;
if (is_striding)
@@ -536,6 +638,33 @@
}
static_always_inline int
+rdma_device_mlx5dv_legacy_rq_slow_path_needed (u32 buf_sz, int n_rx_packets,
+ u32 * bc)
+{
+#if defined CLIB_HAVE_VEC256
+ u32x8 thresh8 = u32x8_splat (buf_sz);
+ for (int i = 0; i < n_rx_packets; i += 8)
+ if (!u32x8_is_all_zero (*(u32x8 *) (bc + i) > thresh8))
+ return 1;
+#elif defined CLIB_HAVE_VEC128
+ u32x4 thresh4 = u32x4_splat (buf_sz);
+ for (int i = 0; i < n_rx_packets; i += 4)
+ if (!u32x4_is_all_zero (*(u32x4 *) (bc + i) > thresh4))
+ return 1;
+#else
+ while (n_rx_packets)
+ {
+ if (*bc > buf_sz)
+ return 1;
+ bc++;
+ n_rx_packets--;
+ }
+#endif
+
+ return 0;
+}
+
+static_always_inline int
rdma_device_mlx5dv_l3_validate_and_swap_bc (rdma_per_thread_data_t
* ptd, int n_rx_packets, u32 * bc)
{
@@ -585,11 +714,11 @@
static_always_inline u32
rdma_device_mlx5dv_fast_input (vlib_main_t * vm, rdma_rxq_t * rxq,
+ vlib_buffer_t ** bufs,
u32 qs_mask, vlib_buffer_t * bt,
u32 * to_next, u32 n_rx_segs, u32 * bc,
u32 bc_mask)
{
- vlib_buffer_t *bufs[VLIB_FRAME_SIZE];
vlib_buffer_t **b = bufs;
u32 n_left = n_rx_segs;
u32 n_rx_bytes = 0;
@@ -629,6 +758,59 @@
return n_rx_bytes;
}
+static_always_inline void
+rdma_device_mlx5dv_legacy_rq_fix_chains (vlib_main_t * vm, rdma_rxq_t * rxq,
+ vlib_buffer_t ** bufs, u32 qs_mask,
+ u32 n)
+{
+ u32 buf_sz = rxq->buf_sz;
+ uword slot = (rxq->head - n) & qs_mask;
+ u32 *second = &rxq->second_bufs[slot];
+ u32 n_wrap_around = (slot + n) & (qs_mask + 1) ? (slot + n) & qs_mask : 0;
+ u8 *n_used_per_chain = &rxq->n_used_per_chain[slot];
+ n -= n_wrap_around;
+wrap_around:
+ while (n > 0)
+ {
+ u16 total_length = bufs[0]->current_length;
+ if (total_length > buf_sz)
+ {
+ vlib_buffer_t *current_buf = bufs[0];
+ u8 current_chain_sz = 0;
+ current_buf->current_length = buf_sz;
+ total_length -= buf_sz;
+ current_buf->total_length_not_including_first_buffer = total_length;
+ current_buf->flags |= VLIB_BUFFER_NEXT_PRESENT;
+ current_buf->next_buffer = second[0];
+ do
+ {
+ current_buf = vlib_get_buffer (vm, current_buf->next_buffer);
+ current_buf->current_length = clib_min (buf_sz, total_length);
+ total_length -= current_buf->current_length;
+ current_chain_sz++;
+ }
+ while (total_length > 0);
+ current_buf->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
+ second[0] = current_buf->next_buffer;
+ current_buf->next_buffer = 0;
+ rxq->n_total_additional_segs += current_chain_sz;
+ n_used_per_chain[0] = current_chain_sz;
+ }
+ bufs++;
+ second++;
+ n_used_per_chain++;
+ n--;
+ }
+ if (PREDICT_FALSE (n_wrap_around))
+ {
+ n = n_wrap_around;
+ n_wrap_around = 0;
+ second = rxq->second_bufs;
+ n_used_per_chain = rxq->n_used_per_chain;
+ goto wrap_around;
+ }
+}
+
static_always_inline u32
rdma_device_mlx5dv_striding_rq_input (vlib_main_t * vm,
rdma_per_thread_data_t * ptd,
@@ -641,8 +823,9 @@
u32 n_rx_bytes = 0;
if (PREDICT_TRUE (!slow_path_needed))
{
+ vlib_buffer_t *bufs[VLIB_FRAME_SIZE];
n_rx_bytes +=
- rdma_device_mlx5dv_fast_input (vm, rxq, mask, bt, to_next,
+ rdma_device_mlx5dv_fast_input (vm, rxq, bufs, mask, bt, to_next,
n_rx_segs, bc, CQE_BC_BYTE_COUNT_MASK);
}
else /* Slow path with multiseg */
@@ -729,7 +912,7 @@
static_always_inline uword
rdma_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
vlib_frame_t * frame, rdma_device_t * rd,
- u16 qid, int use_mlx5dv)
+ u16 qid, const int use_mlx5dv)
{
rdma_main_t *rm = &rdma_main;
vnet_main_t *vnm = vnet_get_main ();
@@ -742,6 +925,7 @@
u32 next_index, *to_next, n_left_to_next, n_rx_bytes = 0;
int n_rx_packets, skip_ip4_cksum = 0;
u32 mask = rxq->size - 1;
+ const int is_striding = ! !(rd->flags & RDMA_DEVICE_F_STRIDING_RQ);
if (use_mlx5dv)
n_rx_packets = rdma_device_poll_cq_mlx5dv (rd, rxq, byte_cnts,
@@ -749,14 +933,14 @@
else
n_rx_packets = ibv_poll_cq (rxq->cq, VLIB_FRAME_SIZE, wc);
- if (PREDICT_FALSE (n_rx_packets <= 0))
- goto refill;
-
/* init buffer template */
vlib_buffer_copy_template (&bt, &ptd->buffer_template);
vnet_buffer (&bt)->sw_if_index[VLIB_RX] = rd->sw_if_index;
bt.buffer_pool_index = rd->pool;
+ if (PREDICT_FALSE (n_rx_packets <= 0))
+ goto refill;
+
/* update buffer template for input feature arcs if any */
next_index = rd->per_interface_next_index;
if (PREDICT_FALSE (vnet_device_input_have_features (rd->sw_if_index)))
@@ -770,7 +954,7 @@
int slow_path_needed;
skip_ip4_cksum =
rdma_device_mlx5dv_l3_validate_and_swap_bc (ptd, n_rx_packets, bc);
- if (rd->flags & RDMA_DEVICE_F_STRIDING_RQ)
+ if (is_striding)
{
int n_rx_segs = 0;
slow_path_needed =
@@ -784,12 +968,20 @@
}
else
{
- /*For now, legacy path doesn't support multiseg */
+ vlib_buffer_t *bufs[VLIB_FRAME_SIZE];
+ slow_path_needed =
+ rdma_device_mlx5dv_legacy_rq_slow_path_needed (rxq->buf_sz,
+ n_rx_packets, bc);
n_rx_bytes =
- rdma_device_mlx5dv_fast_input (vm, rxq, mask, &bt, to_next,
+ rdma_device_mlx5dv_fast_input (vm, rxq, bufs, mask, &bt, to_next,
n_rx_packets, bc, ~1);
- }
+ /* If there are chained buffers, some of the head buffers have a current length
+ higher than buf_sz: it needs to be fixed */
+ if (PREDICT_FALSE (slow_path_needed))
+ rdma_device_mlx5dv_legacy_rq_fix_chains (vm, rxq, bufs, mask,
+ n_rx_packets);
+ }
}
else
{
@@ -817,8 +1009,7 @@
vm->thread_index, rd->hw_if_index,
n_rx_packets, n_rx_bytes);
refill:
- rdma_device_input_refill (vm, rd, rxq, use_mlx5dv,
- ! !(rd->flags & RDMA_DEVICE_F_STRIDING_RQ));
+ rdma_device_input_refill (vm, rd, rxq, &bt, use_mlx5dv, is_striding);
return n_rx_packets;
}
diff --git a/src/plugins/rdma/rdma.api b/src/plugins/rdma/rdma.api
index 668f238..4519e23 100644
--- a/src/plugins/rdma/rdma.api
+++ b/src/plugins/rdma/rdma.api
@@ -15,7 +15,7 @@
*------------------------------------------------------------------
*/
-option version = "1.0.0";
+option version = "2.0.0";
import "vnet/interface_types.api";
enum rdma_mode
@@ -39,6 +39,7 @@
define rdma_create
{
+ option deprecated="21.01";
u32 client_index;
u32 context;
@@ -52,6 +53,36 @@
};
/** \brief
+ @param client_index - opaque cookie to identify the sender
+ @param context - sender context, to match reply w/ request
+ @param host_if - Linux netdev interface name
+ @param name - new rdma interface name
+ @param rxq_num - number of receive queues (optional)
+ @param rxq_size - receive queue size (optional)
+ @param txq_size - transmit queue size (optional)
+ @param mode - operation mode (optional)
+ @param no_multi_seg (optional) - disable chained buffer RX
+ @param max_pktlen (optional) - maximal RX packet size.
+*/
+
+define rdma_create_v2
+{
+ u32 client_index;
+ u32 context;
+
+ string host_if[64];
+ string name[64];
+ u16 rxq_num [default=1];
+ u16 rxq_size [default=1024];
+ u16 txq_size [default=1024];
+ vl_api_rdma_mode_t mode [default=0];
+ bool no_multi_seg [default=0];
+ u16 max_pktlen [default=0];
+ option vat_help = "<host-if ifname> [name <name>] [rx-queue-size <size>] [tx-queue-size <size>] [num-rx-queues <size>] [mode <auto|ibv|dv>] [no-multi-seg] [max-pktlen <size>]";
+};
+
+
+/** \brief
@param context - sender context, to match reply w/ request
@param retval - return value for request
@param sw_if_index - software index for the new rdma interface
@@ -65,6 +96,19 @@
};
/** \brief
+ @param context - sender context, to match reply w/ request
+ @param retval - return value for request
+ @param sw_if_index - software index for the new rdma interface
+*/
+
+define rdma_create_v2_reply
+{
+ u32 context;
+ i32 retval;
+ vl_api_interface_index_t sw_if_index;
+};
+
+/** \brief
@param client_index - opaque cookie to identify the sender
@param context - sender context, to match reply w/ request
@param sw_if_index - interface index
diff --git a/src/plugins/rdma/rdma.h b/src/plugins/rdma/rdma.h
index e6d0abc..a72765d 100644
--- a/src/plugins/rdma/rdma.h
+++ b/src/plugins/rdma/rdma.h
@@ -90,9 +90,27 @@
u32 wqe_cnt;
u32 wq_stride;
u32 buf_sz;
- u32 striding_wqe_tail;
+ union
+ {
+ struct
+ {
+ u32 striding_wqe_tail; /* Striding RQ: number of released whole WQE */
+ u8 log_stride_per_wqe; /* Striding RQ: number of strides in a single WQE */
+ };
+
+ struct
+ {
+ u8 *n_used_per_chain; /* Legacy RQ: for each buffer chain, how many additional segments are needed */
+
+ u32 *second_bufs; /* Legacy RQ: ring of second buffers of each chain */
+ u32 incomplete_tail; /* Legacy RQ: tail index in bufs,
+ corresponds to buffer chains with recycled valid head buffer,
+ but whose other buffers are not yet recycled (due to pool exhaustion). */
+ u16 n_total_additional_segs;
+ u8 n_ds_per_wqe; /* Legacy RQ: number of nonnull data segs per WQE */
+ };
+ };
u8 log_wqe_sz; /* log-size of a single WQE (in data segments) */
- u8 log_stride_per_wqe; /* Striding RQ: number of strides in a single WQE */
} rdma_rxq_t;
typedef struct
@@ -200,8 +218,20 @@
u16x8 cqe_flags8[VLIB_FRAME_SIZE / 8];
u16x16 cqe_flags16[VLIB_FRAME_SIZE / 16];
};
- u32 current_segs[VLIB_FRAME_SIZE];
- u32 to_free_buffers[VLIB_FRAME_SIZE];
+ union
+ {
+ struct
+ {
+ u32 current_segs[VLIB_FRAME_SIZE];
+ u32 to_free_buffers[VLIB_FRAME_SIZE];
+ }; /* Specific to STRIDING RQ mode */
+ struct
+ {
+ u32 tmp_bi[VLIB_FRAME_SIZE];
+ vlib_buffer_t *tmp_bufs[VLIB_FRAME_SIZE];
+ }; /* Specific to LEGACY RQ mode */
+ };
+
vlib_buffer_t buffer_template;
} rdma_per_thread_data_t;
@@ -230,6 +260,9 @@
u32 txq_size;
u32 rxq_num;
rdma_mode_t mode;
+ u8 no_multi_seg;
+ u8 disable_striding_rq;
+ u16 max_pktlen;
/* return */
int rv;
diff --git a/src/plugins/rdma/test_api.c b/src/plugins/rdma/test_api.c
index 19c3592..ff5dec1 100644
--- a/src/plugins/rdma/test_api.c
+++ b/src/plugins/rdma/test_api.c
@@ -87,6 +87,36 @@
return ret;
}
+static int
+api_rdma_create_v2 (vat_main_t * vam)
+{
+ vl_api_rdma_create_v2_t *mp;
+ rdma_create_if_args_t args;
+ int ret;
+
+ if (!unformat_user (vam->input, unformat_rdma_create_if_args, &args))
+ {
+ clib_warning ("unknown input `%U'", format_unformat_error, vam->input);
+ return -99;
+ }
+
+ M (RDMA_CREATE_V2, mp);
+
+ snprintf ((char *) mp->host_if, sizeof (mp->host_if), "%s", args.ifname);
+ snprintf ((char *) mp->name, sizeof (mp->name), "%s", args.name);
+ mp->rxq_num = clib_host_to_net_u16 (args.rxq_num);
+ mp->rxq_size = clib_host_to_net_u16 (args.rxq_size);
+ mp->txq_size = clib_host_to_net_u16 (args.txq_size);
+ mp->mode = api_rdma_mode (args.mode);
+ mp->no_multi_seg = args.no_multi_seg;
+ mp->max_pktlen = clib_host_to_net_u16 (args.max_pktlen);
+
+ S (mp);
+ W (ret);
+
+ return ret;
+}
+
/* rdma-create reply handler */
static void
vl_api_rdma_create_reply_t_handler (vl_api_rdma_create_reply_t * mp)
@@ -105,6 +135,24 @@
vam->regenerate_interface_table = 1;
}
+/* rdma-create reply handler */
+static void
+vl_api_rdma_create_v2_reply_t_handler (vl_api_rdma_create_v2_reply_t * mp)
+{
+ vat_main_t *vam = rdma_test_main.vat_main;
+ i32 retval = ntohl (mp->retval);
+
+ if (retval == 0)
+ {
+ fformat (vam->ofp, "created rdma with sw_if_index %d\n",
+ ntohl (mp->sw_if_index));
+ }
+
+ vam->retval = retval;
+ vam->result_ready = 1;
+ vam->regenerate_interface_table = 1;
+}
+
/* rdma delete API */
static int
api_rdma_delete (vat_main_t * vam)
diff --git a/src/plugins/rdma/unformat.c b/src/plugins/rdma/unformat.c
index 2bbb266..26c1847 100644
--- a/src/plugins/rdma/unformat.c
+++ b/src/plugins/rdma/unformat.c
@@ -24,7 +24,7 @@
rdma_create_if_args_t *args = va_arg (*vargs, rdma_create_if_args_t *);
unformat_input_t _line_input, *line_input = &_line_input;
uword ret = 1;
-
+ u32 tmp;
if (!unformat_user (input, unformat_line_input, line_input))
return 0;
@@ -48,6 +48,12 @@
args->mode = RDMA_MODE_IBV;
else if (unformat (line_input, "mode dv"))
args->mode = RDMA_MODE_DV;
+ else if (unformat (line_input, "no-striding"))
+ args->disable_striding_rq = 1;
+ else if (unformat (line_input, "no-multi-seg"))
+ args->no_multi_seg = 1;
+ else if (unformat (line_input, "max-pktlen %u", &tmp))
+ args->max_pktlen = tmp;
else
{
/* return failure on unknown input */