virtio: virtio: implement packed queues from virtio 1.1
Type: feature
Change-Id: I12703371541298efa029903d6762b1cd1f7322ca
Signed-off-by: Mohsin Kazmi <sykazmi@cisco.com>
diff --git a/src/vnet/devices/virtio/FEATURE.yaml b/src/vnet/devices/virtio/FEATURE.yaml
index ac7133b..7b2fb59 100644
--- a/src/vnet/devices/virtio/FEATURE.yaml
+++ b/src/vnet/devices/virtio/FEATURE.yaml
@@ -7,6 +7,7 @@
- Device mode to emulate vhost-user interface presented to VPP from the
guest VM.
- Support virtio 1.0 in virtio
+ - Support virtio 1.1 packed ring in virtio [experimental]
- Support multi-queue, GSO, checksum offload, indirect descriptor,
jumbo frame, and packed ring.
- Support virtio 1.1 packed ring in vhost
diff --git a/src/vnet/devices/virtio/cli.c b/src/vnet/devices/virtio/cli.c
index 8a9c97b..05f9eab 100644
--- a/src/vnet/devices/virtio/cli.c
+++ b/src/vnet/devices/virtio/cli.c
@@ -49,10 +49,12 @@
args.checksum_offload_enabled = 1;
else if (unformat (line_input, "buffering"))
{
- args.virtio_flags = VIRTIO_FLAG_BUFFERING;
+ args.virtio_flags |= VIRTIO_FLAG_BUFFERING;
if (unformat (line_input, "size %u", &buffering_size))
args.buffering_size = buffering_size;
}
+ else if (unformat (line_input, "packed"))
+ args.virtio_flags |= VIRTIO_FLAG_PACKED;
else
return clib_error_return (0, "unknown input `%U'",
format_unformat_error, input);
@@ -69,7 +71,7 @@
.path = "create interface virtio",
.short_help = "create interface virtio <pci-address> "
"[feature-mask <hex-mask>] [gso-enabled] [csum-enabled] "
- "[buffering [size <buffering-szie>]]",
+ "[buffering [size <buffering-szie>]] [packed]",
.function = virtio_pci_create_command_fn,
};
/* *INDENT-ON* */
diff --git a/src/vnet/devices/virtio/device.c b/src/vnet/devices/virtio/device.c
index add82b7..7e57952 100644
--- a/src/vnet/devices/virtio/device.c
+++ b/src/vnet/devices/virtio/device.c
@@ -161,8 +161,8 @@
}
static_always_inline void
-virtio_free_used_device_desc (vlib_main_t * vm, virtio_vring_t * vring,
- uword node_index)
+virtio_free_used_device_desc_split (vlib_main_t * vm, virtio_vring_t * vring,
+ uword node_index)
{
u16 used = vring->desc_in_use;
u16 sz = vring->size;
@@ -227,7 +227,60 @@
}
static_always_inline void
-set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr, int is_l2)
+virtio_free_used_device_desc_packed (vlib_main_t * vm, virtio_vring_t * vring,
+ uword node_index)
+{
+ vring_packed_desc_t *d;
+ u16 sz = vring->size;
+ u16 last = vring->last_used_idx;
+ u16 n_buffers = 0, start;
+ u16 flags;
+
+ if (vring->desc_in_use == 0)
+ return;
+
+ d = &vring->packed_desc[last];
+ flags = d->flags;
+ start = d->id;
+
+ while ((flags & VRING_DESC_F_AVAIL) == (vring->used_wrap_counter << 7) &&
+ (flags & VRING_DESC_F_USED) == (vring->used_wrap_counter << 15))
+ {
+ last++;
+ n_buffers++;
+
+ if (last >= sz)
+ {
+ last = 0;
+ vring->used_wrap_counter ^= 1;
+ }
+ d = &vring->packed_desc[last];
+ flags = d->flags;
+ }
+
+ if (n_buffers)
+ {
+ vlib_buffer_free_from_ring (vm, vring->buffers, start, sz, n_buffers);
+ virtio_memset_ring_u32 (vring->buffers, start, sz, n_buffers);
+ vring->desc_in_use -= n_buffers;
+ vring->last_used_idx = last;
+ }
+}
+
+static_always_inline void
+virtio_free_used_device_desc (vlib_main_t * vm, virtio_vring_t * vring,
+ uword node_index, int packed)
+{
+ if (packed)
+ virtio_free_used_device_desc_packed (vm, vring, node_index);
+ else
+ virtio_free_used_device_desc_split (vm, vring, node_index);
+
+}
+
+static_always_inline void
+set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr,
+ const int is_l2)
{
if (b->flags & VNET_BUFFER_F_IS_IP4)
{
@@ -274,7 +327,8 @@
}
static_always_inline void
-set_gso_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr, int is_l2)
+set_gso_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr,
+ const int is_l2)
{
if (b->flags & VNET_BUFFER_F_IS_IP4)
{
@@ -504,6 +558,238 @@
return n_added;
}
+static_always_inline u16
+add_buffer_to_slot_packed (vlib_main_t * vm, vlib_node_runtime_t * node,
+ virtio_vring_t * vring, u32 bi, u16 next,
+ int hdr_sz, int do_gso, int csum_offload,
+ int is_pci, int is_tun, int is_indirect,
+ int is_any_layout)
+{
+ u16 n_added = 0, flags = 0;
+ int is_l2 = !is_tun;
+ vring_packed_desc_t *d = &vring->packed_desc[next];
+ vlib_buffer_t *b = vlib_get_buffer (vm, bi);
+ virtio_net_hdr_v1_t *hdr = vlib_buffer_get_current (b) - hdr_sz;
+ u32 drop_inline = ~0;
+
+ clib_memset (hdr, 0, hdr_sz);
+
+ if (b->flags & VNET_BUFFER_F_GSO)
+ {
+ if (do_gso)
+ set_gso_offsets (b, hdr, is_l2);
+ else
+ {
+ drop_inline = VIRTIO_TX_ERROR_GSO_PACKET_DROP;
+ goto done;
+ }
+ }
+ else if (b->flags & (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
+ VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
+ {
+ if (csum_offload)
+ set_checksum_offsets (b, hdr, is_l2);
+ else
+ {
+ drop_inline = VIRTIO_TX_ERROR_CSUM_OFFLOAD_PACKET_DROP;
+ goto done;
+ }
+ }
+ if (PREDICT_FALSE (b->flags & VLIB_BUFFER_IS_TRACED))
+ {
+ virtio_tx_trace (vm, node, b, bi, is_tun);
+ }
+
+ if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
+ {
+ d->addr =
+ ((is_pci) ? vlib_buffer_get_current_pa (vm,
+ b) :
+ pointer_to_uword (vlib_buffer_get_current (b))) - hdr_sz;
+ d->len = b->current_length + hdr_sz;
+ }
+ else if (is_indirect)
+ {
+ /*
+ * We are using single vlib_buffer_t for indirect descriptor(s)
+ * chain. Single descriptor is 16 bytes and vlib_buffer_t
+ * has 2048 bytes space. So maximum long chain can have 128
+ * (=2048/16) indirect descriptors.
+ * It can easily support 65535 bytes of Jumbo frames with
+ * each data buffer size of 512 bytes minimum.
+ */
+ u32 indirect_buffer = 0;
+ if (PREDICT_FALSE (vlib_buffer_alloc (vm, &indirect_buffer, 1) == 0))
+ {
+ drop_inline = VIRTIO_TX_ERROR_INDIRECT_DESC_ALLOC_FAILED;
+ goto done;
+ }
+
+ vlib_buffer_t *indirect_desc = vlib_get_buffer (vm, indirect_buffer);
+ indirect_desc->current_data = 0;
+ indirect_desc->flags |= VLIB_BUFFER_NEXT_PRESENT;
+ indirect_desc->next_buffer = bi;
+ bi = indirect_buffer;
+
+ vring_packed_desc_t *id =
+ (vring_packed_desc_t *) vlib_buffer_get_current (indirect_desc);
+ u32 count = 1;
+ if (is_pci)
+ {
+ d->addr = vlib_physmem_get_pa (vm, id);
+ id->addr = vlib_buffer_get_current_pa (vm, b) - hdr_sz;
+
+ /*
+ * If VIRTIO_F_ANY_LAYOUT is not negotiated, then virtio_net_hdr
+ * should be presented in separate descriptor and data will start
+ * from next descriptor.
+ */
+ if (is_any_layout)
+ id->len = b->current_length + hdr_sz;
+ else
+ {
+ id->len = hdr_sz;
+ id->flags = 0;
+ id->id = 0;
+ count++;
+ id++;
+ id->addr = vlib_buffer_get_current_pa (vm, b);
+ id->len = b->current_length;
+ }
+ while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
+ {
+ id->flags = 0;
+ id->id = 0;
+ count++;
+ id++;
+ b = vlib_get_buffer (vm, b->next_buffer);
+ id->addr = vlib_buffer_get_current_pa (vm, b);
+ id->len = b->current_length;
+ }
+ }
+ id->flags = 0;
+ id->id = 0;
+ d->len = count * sizeof (vring_packed_desc_t);
+ flags = VRING_DESC_F_INDIRECT;
+ }
+ else
+ {
+ ASSERT (0);
+ }
+ if (vring->avail_wrap_counter)
+ {
+ flags |= VRING_DESC_F_AVAIL;
+ flags &= ~VRING_DESC_F_USED;
+ }
+ else
+ {
+ flags &= ~VRING_DESC_F_AVAIL;
+ flags |= VRING_DESC_F_USED;
+ }
+
+ d->id = next;
+ d->flags = flags;
+ vring->buffers[next] = bi;
+ n_added++;
+
+done:
+ if (drop_inline != ~0)
+ virtio_interface_drop_inline (vm, node->node_index, &bi, 1, drop_inline);
+
+ return n_added;
+}
+
+static_always_inline uword
+virtio_interface_tx_packed_gso_inline (vlib_main_t * vm,
+ vlib_node_runtime_t * node,
+ virtio_if_t * vif,
+ virtio_if_type_t type,
+ virtio_vring_t * vring,
+ u32 * buffers, u16 n_left,
+ const int do_gso,
+ const int csum_offload)
+{
+ int is_pci = (type == VIRTIO_IF_TYPE_PCI);
+ int is_tun = (type == VIRTIO_IF_TYPE_TUN);
+ int is_indirect =
+ ((vif->features & VIRTIO_FEATURE (VIRTIO_RING_F_INDIRECT_DESC)) != 0);
+ int is_any_layout =
+ ((vif->features & VIRTIO_FEATURE (VIRTIO_F_ANY_LAYOUT)) != 0);
+ const int hdr_sz = vif->virtio_net_hdr_sz;
+ u16 sz = vring->size;
+ u16 used, next, n_buffers = 0, n_buffers_left = 0;
+ u16 n_vectors = n_left;
+
+
+ used = vring->desc_in_use;
+ next = vring->desc_next;
+
+ if (vif->packet_buffering)
+ {
+ n_buffers = n_buffers_left = virtio_vring_n_buffers (vring->buffering);
+
+ while (n_buffers_left && used < sz)
+ {
+ u16 n_added = 0;
+
+ u32 bi = virtio_vring_buffering_read_from_front (vring->buffering);
+ if (bi == ~0)
+ break;
+ n_added = add_buffer_to_slot_packed (vm, node,
+ vring, bi, next,
+ hdr_sz, do_gso, csum_offload,
+ is_pci, is_tun, is_indirect,
+ is_any_layout);
+ n_buffers_left--;
+ if (PREDICT_FALSE (n_added == 0))
+ continue;
+
+ used++;
+ next++;
+ if (next >= sz)
+ {
+ next = 0;
+ vring->avail_wrap_counter ^= 1;
+ }
+ }
+ }
+
+ while (n_left && used < sz)
+ {
+ u16 n_added = 0;
+
+ n_added = add_buffer_to_slot_packed (vm, node,
+ vring, buffers[0], next,
+ hdr_sz, do_gso, csum_offload,
+ is_pci, is_tun, is_indirect,
+ is_any_layout);
+ buffers++;
+ n_left--;
+ if (PREDICT_FALSE (n_added == 0))
+ continue;
+
+ used++;
+ next++;
+ if (next >= sz)
+ {
+ next = 0;
+ vring->avail_wrap_counter ^= 1;
+ }
+ }
+
+ if (n_left != n_vectors || n_buffers != n_buffers_left)
+ {
+ CLIB_MEMORY_STORE_BARRIER ();
+ vring->desc_next = next;
+ vring->desc_in_use = used;
+ CLIB_MEMORY_BARRIER ();
+ if (vring->device_event->flags != VRING_EVENT_F_DISABLE)
+ virtio_kick (vm, vring, vif);
+ }
+
+ return n_left;
+}
+
static_always_inline void
virtio_find_free_desc (virtio_vring_t * vring, u16 size, u16 mask,
u16 req, u16 next, u32 * first_free_desc_index,
@@ -541,11 +827,13 @@
}
static_always_inline u16
-virtio_interface_tx_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
- virtio_if_t * vif,
- virtio_if_type_t type, virtio_vring_t * vring,
- u32 * buffers, u16 n_left, int do_gso,
- int csum_offload)
+virtio_interface_tx_split_gso_inline (vlib_main_t * vm,
+ vlib_node_runtime_t * node,
+ virtio_if_t * vif,
+ virtio_if_type_t type,
+ virtio_vring_t * vring, u32 * buffers,
+ u16 n_left, int do_gso,
+ int csum_offload)
{
u16 used, next, avail, n_buffers = 0, n_buffers_left = 0;
int is_pci = (type == VIRTIO_IF_TYPE_PCI);
@@ -650,26 +938,46 @@
}
static_always_inline u16
+virtio_interface_tx_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
+ virtio_if_t * vif,
+ virtio_if_type_t type, virtio_vring_t * vring,
+ u32 * buffers, u16 n_left, int packed,
+ int do_gso, int csum_offload)
+{
+ if (packed)
+ return virtio_interface_tx_packed_gso_inline (vm, node, vif, type, vring,
+ buffers, n_left,
+ 1 /* do_gso */ ,
+ 1 /* checksum offload */ );
+ else
+ return virtio_interface_tx_split_gso_inline (vm, node, vif, type, vring,
+ buffers, n_left,
+ 1 /* do_gso */ ,
+ 1 /* checksum offload */ );
+}
+
+static_always_inline u16
virtio_interface_tx_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
virtio_if_t * vif,
virtio_vring_t * vring, virtio_if_type_t type,
- u32 * buffers, u16 n_left)
+ u32 * buffers, u16 n_left, int packed)
{
vnet_main_t *vnm = vnet_get_main ();
vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, vif->hw_if_index);
if (hw->flags & VNET_HW_INTERFACE_FLAG_SUPPORTS_GSO)
return virtio_interface_tx_gso_inline (vm, node, vif, type, vring,
- buffers, n_left, 1 /* do_gso */ ,
+ buffers, n_left, packed,
+ 1 /* do_gso */ ,
1 /* checksum offload */ );
else if (hw->flags & VNET_HW_INTERFACE_FLAG_SUPPORTS_TX_L4_CKSUM_OFFLOAD)
return virtio_interface_tx_gso_inline (vm, node, vif, type, vring,
- buffers, n_left,
+ buffers, n_left, packed,
0 /* no do_gso */ ,
1 /* checksum offload */ );
else
return virtio_interface_tx_gso_inline (vm, node, vif, type, vring,
- buffers, n_left,
+ buffers, n_left, packed,
0 /* no do_gso */ ,
0 /* no checksum offload */ );
}
@@ -686,6 +994,7 @@
u16 n_left = frame->n_vectors;
u32 *buffers = vlib_frame_vector_args (frame);
u32 to[GRO_TO_VECTOR_SIZE (n_left)];
+ int packed = vif->is_packed;
clib_spinlock_lock_if_init (&vring->lockp);
@@ -703,23 +1012,23 @@
retry:
/* free consumed buffers */
- virtio_free_used_device_desc (vm, vring, node->node_index);
+ virtio_free_used_device_desc (vm, vring, node->node_index, packed);
if (vif->type == VIRTIO_IF_TYPE_TAP)
n_left = virtio_interface_tx_inline (vm, node, vif, vring,
VIRTIO_IF_TYPE_TAP,
&buffers[frame->n_vectors - n_left],
- n_left);
+ n_left, packed);
else if (vif->type == VIRTIO_IF_TYPE_PCI)
n_left = virtio_interface_tx_inline (vm, node, vif, vring,
VIRTIO_IF_TYPE_PCI,
&buffers[frame->n_vectors - n_left],
- n_left);
+ n_left, packed);
else if (vif->type == VIRTIO_IF_TYPE_TUN)
n_left = virtio_interface_tx_inline (vm, node, vif, vring,
VIRTIO_IF_TYPE_TUN,
&buffers[frame->n_vectors - n_left],
- n_left);
+ n_left, packed);
else
ASSERT (0);
diff --git a/src/vnet/devices/virtio/node.c b/src/vnet/devices/virtio/node.c
index 91e788e..a45b52c 100644
--- a/src/vnet/devices/virtio/node.c
+++ b/src/vnet/devices/virtio/node.c
@@ -81,9 +81,9 @@
}
static_always_inline void
-virtio_refill_vring (vlib_main_t * vm, virtio_if_t * vif,
- virtio_if_type_t type, virtio_vring_t * vring,
- const int hdr_sz, u32 node_index)
+virtio_refill_vring_split (vlib_main_t * vm, virtio_if_t * vif,
+ virtio_if_type_t type, virtio_vring_t * vring,
+ const int hdr_sz, u32 node_index)
{
u16 used, next, avail, n_slots, n_refill;
u16 sz = vring->size;
@@ -149,6 +149,88 @@
}
static_always_inline void
+virtio_refill_vring_packed (vlib_main_t * vm, virtio_if_t * vif,
+ virtio_if_type_t type, virtio_vring_t * vring,
+ const int hdr_sz, u32 node_index)
+{
+ u16 used, next, n_slots, n_refill, flags = 0, first_desc_flags;
+ u16 sz = vring->size;
+
+more:
+ used = vring->desc_in_use;
+
+ if (sz == used)
+ return;
+
+ /* deliver free buffers in chunks of 64 */
+ n_refill = clib_min (sz - used, 64);
+
+ next = vring->desc_next;
+ first_desc_flags = vring->packed_desc[next].flags;
+ n_slots =
+ vlib_buffer_alloc_to_ring_from_pool (vm, vring->buffers, next,
+ sz, n_refill,
+ vring->buffer_pool_index);
+
+ if (PREDICT_FALSE (n_slots != n_refill))
+ {
+ vlib_error_count (vm, node_index,
+ VIRTIO_INPUT_ERROR_BUFFER_ALLOC, n_refill - n_slots);
+ if (n_slots == 0)
+ return;
+ }
+
+ while (n_slots)
+ {
+ vring_packed_desc_t *d = &vring->packed_desc[next];
+ vlib_buffer_t *b = vlib_get_buffer (vm, vring->buffers[next]);
+ /*
+ * current_data may not be initialized with 0 and may contain
+ * previous offset. Here we want to make sure, it should be 0
+ * initialized.
+ */
+ b->current_data = -hdr_sz;
+ memset (vlib_buffer_get_current (b), 0, hdr_sz);
+ d->addr =
+ ((type == VIRTIO_IF_TYPE_PCI) ? vlib_buffer_get_current_pa (vm,
+ b) :
+ pointer_to_uword (vlib_buffer_get_current (b)));
+ d->len = vlib_buffer_get_default_data_size (vm) + hdr_sz;
+
+ if (vring->avail_wrap_counter)
+ flags = (VRING_DESC_F_AVAIL | VRING_DESC_F_WRITE);
+ else
+ flags = (VRING_DESC_F_USED | VRING_DESC_F_WRITE);
+
+ d->id = next;
+ if (vring->desc_next == next)
+ first_desc_flags = flags;
+ else
+ d->flags = flags;
+
+ next++;
+ if (next >= sz)
+ {
+ next = 0;
+ vring->avail_wrap_counter ^= 1;
+ }
+ n_slots--;
+ used++;
+ }
+ CLIB_MEMORY_STORE_BARRIER ();
+ vring->packed_desc[vring->desc_next].flags = first_desc_flags;
+ vring->desc_next = next;
+ vring->desc_in_use = used;
+ CLIB_MEMORY_BARRIER ();
+ if (vring->device_event->flags != VRING_EVENT_F_DISABLE)
+ {
+ virtio_kick (vm, vring, vif);
+ }
+
+ goto more;
+}
+
+static_always_inline void
virtio_needs_csum (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr,
u8 * l4_proto, u8 * l4_hdr_sz, virtio_if_type_t type)
{
@@ -258,11 +340,51 @@
}
}
+static_always_inline u16
+virtio_n_left_to_process (virtio_vring_t * vring, const int packed)
+{
+ if (packed)
+ return vring->desc_in_use;
+ else
+ return vring->used->idx - vring->last_used_idx;
+}
+
+static_always_inline u16
+virtio_get_slot_id (virtio_vring_t * vring, const int packed, u16 last,
+ u16 mask)
+{
+ if (packed)
+ return vring->packed_desc[last].id;
+ else
+ return vring->used->ring[last & mask].id;
+}
+
+static_always_inline u16
+virtio_get_len (virtio_vring_t * vring, const int packed, const int hdr_sz,
+ u16 last, u16 mask)
+{
+ if (packed)
+ return vring->packed_desc[last].len - hdr_sz;
+ else
+ return vring->used->ring[last & mask].len - hdr_sz;
+}
+
+#define increment_last(last, packed, vring) \
+ do { \
+ last++; \
+ if (packed && last >= vring->size) \
+ { \
+ last = 0; \
+ vring->used_wrap_counter ^= 1; \
+ } \
+ } while (0)
+
static_always_inline uword
virtio_device_input_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
vlib_frame_t * frame, virtio_if_t * vif,
virtio_vring_t * vring, virtio_if_type_t type,
- int gso_enabled, int checksum_offload_enabled)
+ int gso_enabled, int checksum_offload_enabled,
+ int packed)
{
vnet_main_t *vnm = vnet_get_main ();
u32 thread_index = vm->thread_index;
@@ -274,7 +396,7 @@
u32 n_rx_bytes = 0;
u16 mask = vring->size - 1;
u16 last = vring->last_used_idx;
- u16 n_left = vring->used->idx - last;
+ u16 n_left = virtio_n_left_to_process (vring, packed);
vlib_buffer_t bt;
if (n_left == 0)
@@ -303,12 +425,24 @@
while (n_left && n_left_to_next)
{
+ if (packed)
+ {
+ vring_packed_desc_t *d = &vring->packed_desc[last];
+ u16 flags = d->flags;
+ if ((flags & VRING_DESC_F_AVAIL) !=
+ (vring->used_wrap_counter << 7)
+ || (flags & VRING_DESC_F_USED) !=
+ (vring->used_wrap_counter << 15))
+ {
+ n_left = 0;
+ break;
+ }
+ }
u8 l4_proto = 0, l4_hdr_sz = 0;
u16 num_buffers = 1;
- vring_used_elem_t *e = &vring->used->ring[last & mask];
virtio_net_hdr_v1_t *hdr;
- u16 slot = e->id;
- u16 len = e->len - hdr_sz;
+ u16 slot = virtio_get_slot_id (vring, packed, last, mask);
+ u16 len = virtio_get_len (vring, packed, hdr_sz, last, mask);
u32 bi0 = vring->buffers[slot];
vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0);
hdr = vlib_buffer_get_current (b0);
@@ -336,20 +470,22 @@
b0->total_length_not_including_first_buffer = 0;
while (num_buffers > 1)
{
- last++;
- e = &vring->used->ring[last & mask];
- u32 cbi = vring->buffers[e->id];
+ increment_last (last, packed, vring);
+ u16 cslot = virtio_get_slot_id (vring, packed, last, mask);
+ u16 clen =
+ virtio_get_len (vring, packed, hdr_sz, last, mask);
+ u32 cbi = vring->buffers[cslot];
cb = vlib_get_buffer (vm, cbi);
/* current buffer */
- cb->current_length = e->len;
+ cb->current_length = clen;
/* previous buffer */
pb->next_buffer = cbi;
pb->flags |= VLIB_BUFFER_NEXT_PRESENT;
/* first buffer */
- b0->total_length_not_including_first_buffer += e->len;
+ b0->total_length_not_including_first_buffer += clen;
pb = cb;
vring->desc_in_use--;
@@ -406,7 +542,7 @@
to_next += 1;
n_left_to_next--;
n_left--;
- last++;
+ increment_last (last, packed, vring);
/* only tun interfaces may have different next index */
if (type == VIRTIO_IF_TYPE_TUN)
@@ -451,21 +587,49 @@
clib_spinlock_unlock_if_init (&txq_vring->lockp);
}
- if ((vring->used->flags & VRING_USED_F_NO_NOTIFY) == 0 &&
- vring->last_kick_avail_idx != vring->avail->idx)
- virtio_kick (vm, vring, vif);
+ if (vif->is_packed)
+ {
+ if (vring->device_event->flags != VRING_EVENT_F_DISABLE)
+ virtio_kick (vm, vring, vif);
- if (vif->gso_enabled)
- rv = virtio_device_input_gso_inline (vm, node, frame, vif, vring, type,
- 1, 1);
- else if (vif->csum_offload_enabled)
- rv = virtio_device_input_gso_inline (vm, node, frame, vif, vring, type,
- 0, 1);
+ if (vif->gso_enabled)
+ rv =
+ virtio_device_input_gso_inline (vm, node, frame, vif, vring, type,
+ 1, 1, 1);
+ else if (vif->csum_offload_enabled)
+ rv =
+ virtio_device_input_gso_inline (vm, node, frame, vif, vring, type,
+ 0, 1, 1);
+ else
+ rv =
+ virtio_device_input_gso_inline (vm, node, frame, vif, vring, type,
+ 0, 0, 1);
+
+ virtio_refill_vring_packed (vm, vif, type, vring, hdr_sz,
+ node->node_index);
+ }
else
- rv = virtio_device_input_gso_inline (vm, node, frame, vif, vring, type,
- 0, 0);
+ {
+ if ((vring->used->flags & VRING_USED_F_NO_NOTIFY) == 0 &&
+ vring->last_kick_avail_idx != vring->avail->idx)
+ virtio_kick (vm, vring, vif);
- virtio_refill_vring (vm, vif, type, vring, hdr_sz, node->node_index);
+ if (vif->gso_enabled)
+ rv =
+ virtio_device_input_gso_inline (vm, node, frame, vif, vring, type,
+ 1, 1, 0);
+ else if (vif->csum_offload_enabled)
+ rv =
+ virtio_device_input_gso_inline (vm, node, frame, vif, vring, type,
+ 0, 1, 0);
+ else
+ rv =
+ virtio_device_input_gso_inline (vm, node, frame, vif, vring, type,
+ 0, 0, 0);
+
+ virtio_refill_vring_split (vm, vif, type, vring, hdr_sz,
+ node->node_index);
+ }
return rv;
}
diff --git a/src/vnet/devices/virtio/pci.c b/src/vnet/devices/virtio/pci.c
index 177f764..785ca48 100644
--- a/src/vnet/devices/virtio/pci.c
+++ b/src/vnet/devices/virtio/pci.c
@@ -195,8 +195,148 @@
}
static int
-virtio_pci_send_ctrl_msg (vlib_main_t * vm, virtio_if_t * vif,
- virtio_ctrl_msg_t * data, u32 len)
+virtio_pci_send_ctrl_msg_packed (vlib_main_t * vm, virtio_if_t * vif,
+ virtio_ctrl_msg_t * data, u32 len)
+{
+ virtio_vring_t *vring = vif->cxq_vring;
+ virtio_net_ctrl_ack_t status = VIRTIO_NET_ERR;
+ virtio_ctrl_msg_t result;
+ u32 buffer_index;
+ vlib_buffer_t *b;
+ u16 used, next;
+ u16 sz = vring->size;
+ u16 flags = 0, first_desc_flags = 0;
+
+ used = vring->desc_in_use;
+ next = vring->desc_next;
+ vring_packed_desc_t *d = &vring->packed_desc[next];
+
+ if (vlib_buffer_alloc (vm, &buffer_index, 1))
+ b = vlib_get_buffer (vm, buffer_index);
+ else
+ return VIRTIO_NET_ERR;
+ /*
+ * current_data may not be initialized with 0 and may contain
+ * previous offset.
+ */
+ b->current_data = 0;
+ clib_memcpy (vlib_buffer_get_current (b), data, sizeof (virtio_ctrl_msg_t));
+
+ first_desc_flags = VRING_DESC_F_NEXT;
+ if (vring->avail_wrap_counter)
+ {
+ first_desc_flags |= VRING_DESC_F_AVAIL;
+ first_desc_flags &= ~VRING_DESC_F_USED;
+ }
+ else
+ {
+ first_desc_flags &= ~VRING_DESC_F_AVAIL;
+ first_desc_flags |= VRING_DESC_F_USED;
+ }
+ d->addr = vlib_buffer_get_current_pa (vm, b);
+ d->len = sizeof (virtio_net_ctrl_hdr_t);
+ d->id = next;
+
+ next++;
+ if (next >= sz)
+ {
+ next = 0;
+ vring->avail_wrap_counter ^= 1;
+ }
+ used++;
+
+ d = &vring->packed_desc[next];
+ flags = VRING_DESC_F_NEXT;
+ if (vring->avail_wrap_counter)
+ {
+ flags |= VRING_DESC_F_AVAIL;
+ flags &= ~VRING_DESC_F_USED;
+ }
+ else
+ {
+ flags &= ~VRING_DESC_F_AVAIL;
+ flags |= VRING_DESC_F_USED;
+ }
+ d->addr = vlib_buffer_get_current_pa (vm, b) +
+ STRUCT_OFFSET_OF (virtio_ctrl_msg_t, data);
+ d->len = len;
+ d->id = next;
+ d->flags = flags;
+
+ next++;
+ if (next >= sz)
+ {
+ next = 0;
+ vring->avail_wrap_counter ^= 1;
+ }
+ used++;
+
+ d = &vring->packed_desc[next];
+ flags = VRING_DESC_F_WRITE;
+ if (vring->avail_wrap_counter)
+ {
+ flags |= VRING_DESC_F_AVAIL;
+ flags &= ~VRING_DESC_F_USED;
+ }
+ else
+ {
+ flags &= ~VRING_DESC_F_AVAIL;
+ flags |= VRING_DESC_F_USED;
+ }
+ d->addr = vlib_buffer_get_current_pa (vm, b) +
+ STRUCT_OFFSET_OF (virtio_ctrl_msg_t, status);
+ d->len = sizeof (data->status);
+ d->id = next;
+ d->flags = flags;
+
+ next++;
+ if (next >= sz)
+ {
+ next = 0;
+ vring->avail_wrap_counter ^= 1;
+ }
+ used++;
+
+ CLIB_MEMORY_STORE_BARRIER ();
+ vring->packed_desc[vring->desc_next].flags = first_desc_flags;
+ vring->desc_next = next;
+ vring->desc_in_use = used;
+ CLIB_MEMORY_BARRIER ();
+ if (vring->device_event->flags != VRING_EVENT_F_DISABLE)
+ {
+ virtio_kick (vm, vring, vif);
+ }
+
+ u16 last = vring->last_used_idx;
+ d = &vring->packed_desc[last];
+ do
+ {
+ flags = d->flags;
+ }
+ while ((flags & VRING_DESC_F_AVAIL) != (vring->used_wrap_counter << 7)
+ || (flags & VRING_DESC_F_USED) != (vring->used_wrap_counter << 15));
+
+ last += 3;
+ if (last >= vring->size)
+ {
+ last = last - vring->size;
+ vring->used_wrap_counter ^= 1;
+ }
+ vring->desc_in_use -= 3;
+ vring->last_used_idx = last;
+
+ CLIB_MEMORY_BARRIER ();
+ clib_memcpy (&result, vlib_buffer_get_current (b),
+ sizeof (virtio_ctrl_msg_t));
+ virtio_log_debug (vif, "ctrl-queue: status %u", result.status);
+ status = result.status;
+ vlib_buffer_free (vm, &buffer_index, 1);
+ return status;
+}
+
+static int
+virtio_pci_send_ctrl_msg_split (vlib_main_t * vm, virtio_if_t * vif,
+ virtio_ctrl_msg_t * data, u32 len)
{
virtio_vring_t *vring = vif->cxq_vring;
virtio_net_ctrl_ack_t status = VIRTIO_NET_ERR;
@@ -290,6 +430,16 @@
}
static int
+virtio_pci_send_ctrl_msg (vlib_main_t * vm, virtio_if_t * vif,
+ virtio_ctrl_msg_t * data, u32 len)
+{
+ if (vif->is_packed)
+ return virtio_pci_send_ctrl_msg_packed (vm, vif, data, len);
+ else
+ return virtio_pci_send_ctrl_msg_split (vm, vif, data, len);
+}
+
+static int
virtio_pci_disable_offload (vlib_main_t * vm, virtio_if_t * vif)
{
virtio_ctrl_msg_t offload_hdr;
@@ -440,8 +590,73 @@
}
clib_error_t *
-virtio_pci_control_vring_init (vlib_main_t * vm, virtio_if_t * vif,
- u16 queue_num)
+virtio_pci_control_vring_packed_init (vlib_main_t * vm, virtio_if_t * vif,
+ u16 queue_num)
+{
+ clib_error_t *error = 0;
+ u16 queue_size = 0;
+ virtio_vring_t *vring;
+ u32 i = 0;
+ void *ptr = NULL;
+
+ queue_size = vif->virtio_pci_func->get_queue_size (vm, vif, queue_num);
+
+ if (queue_size > 32768)
+ return clib_error_return (0, "ring size must be 32768 or lower");
+
+ if (queue_size == 0)
+ queue_size = 256;
+
+ vec_validate_aligned (vif->cxq_vring, 0, CLIB_CACHE_LINE_BYTES);
+ vring = vec_elt_at_index (vif->cxq_vring, 0);
+
+ i =
+ (((queue_size * sizeof (vring_packed_desc_t)) +
+ sizeof (vring_desc_event_t) + VIRTIO_PCI_VRING_ALIGN -
+ 1) & ~(VIRTIO_PCI_VRING_ALIGN - 1)) + sizeof (vring_desc_event_t);
+
+ ptr =
+ vlib_physmem_alloc_aligned_on_numa (vm, i, VIRTIO_PCI_VRING_ALIGN,
+ vif->numa_node);
+ if (!ptr)
+ return vlib_physmem_last_error (vm);
+ clib_memset (ptr, 0, i);
+
+ vring->packed_desc = ptr;
+
+ vring->driver_event = ptr + (queue_size * sizeof (vring_packed_desc_t));
+ vring->driver_event->off_wrap = 0;
+ vring->driver_event->flags = VRING_EVENT_F_DISABLE;
+
+ vring->device_event =
+ ptr +
+ (((queue_size * sizeof (vring_packed_desc_t)) +
+ sizeof (vring_desc_event_t) + VIRTIO_PCI_VRING_ALIGN -
+ 1) & ~(VIRTIO_PCI_VRING_ALIGN - 1));
+ vring->device_event->off_wrap = 0;
+ vring->device_event->flags = 0;
+
+ vring->queue_id = queue_num;
+ vring->size = queue_size;
+ vring->avail_wrap_counter = 1;
+ vring->used_wrap_counter = 1;
+
+ ASSERT (vring->buffers == 0);
+
+ virtio_log_debug (vif, "control-queue: number %u, size %u", queue_num,
+ queue_size);
+ vif->virtio_pci_func->setup_queue (vm, vif, queue_num, (void *) vring);
+ vring->queue_notify_offset =
+ vif->notify_off_multiplier *
+ vif->virtio_pci_func->get_queue_notify_off (vm, vif, queue_num);
+ virtio_log_debug (vif, "queue-notify-offset: number %u, offset %u",
+ queue_num, vring->queue_notify_offset);
+ return error;
+}
+
+clib_error_t *
+virtio_pci_control_vring_split_init (vlib_main_t * vm, virtio_if_t * vif,
+ u16 queue_num)
{
clib_error_t *error = 0;
u16 queue_size = 0;
@@ -496,7 +711,18 @@
}
clib_error_t *
-virtio_pci_vring_init (vlib_main_t * vm, virtio_if_t * vif, u16 queue_num)
+virtio_pci_control_vring_init (vlib_main_t * vm, virtio_if_t * vif,
+ u16 queue_num)
+{
+ if (vif->is_packed)
+ return virtio_pci_control_vring_packed_init (vm, vif, queue_num);
+ else
+ return virtio_pci_control_vring_split_init (vm, vif, queue_num);
+}
+
+clib_error_t *
+virtio_pci_vring_split_init (vlib_main_t * vm, virtio_if_t * vif,
+ u16 queue_num)
{
vlib_thread_main_t *vtm = vlib_get_thread_main ();
clib_error_t *error = 0;
@@ -574,6 +800,106 @@
return error;
}
+clib_error_t *
+virtio_pci_vring_packed_init (vlib_main_t * vm, virtio_if_t * vif,
+ u16 queue_num)
+{
+ vlib_thread_main_t *vtm = vlib_get_thread_main ();
+ clib_error_t *error = 0;
+ u16 queue_size = 0;
+ virtio_vring_t *vring;
+ u32 i = 0;
+ void *ptr = NULL;
+
+ queue_size = vif->virtio_pci_func->get_queue_size (vm, vif, queue_num);
+
+ if (queue_size > 32768)
+ return clib_error_return (0, "ring size must be 32768 or lower");
+
+ if (queue_size == 0)
+ queue_size = 256;
+
+ if (queue_num % 2)
+ {
+ vec_validate_aligned (vif->txq_vrings, TX_QUEUE_ACCESS (queue_num),
+ CLIB_CACHE_LINE_BYTES);
+ vring = vec_elt_at_index (vif->txq_vrings, TX_QUEUE_ACCESS (queue_num));
+ if (vif->max_queue_pairs < vtm->n_vlib_mains)
+ clib_spinlock_init (&vring->lockp);
+ }
+ else
+ {
+ vec_validate_aligned (vif->rxq_vrings, RX_QUEUE_ACCESS (queue_num),
+ CLIB_CACHE_LINE_BYTES);
+ vring = vec_elt_at_index (vif->rxq_vrings, RX_QUEUE_ACCESS (queue_num));
+ }
+
+ i =
+ (((queue_size * sizeof (vring_packed_desc_t)) +
+ sizeof (vring_desc_event_t) + VIRTIO_PCI_VRING_ALIGN -
+ 1) & ~(VIRTIO_PCI_VRING_ALIGN - 1)) + sizeof (vring_desc_event_t);
+
+ ptr =
+ vlib_physmem_alloc_aligned_on_numa (vm, i, VIRTIO_PCI_VRING_ALIGN,
+ vif->numa_node);
+ if (!ptr)
+ return vlib_physmem_last_error (vm);
+
+ clib_memset (ptr, 0, i);
+ vring->packed_desc = ptr;
+
+ vring->driver_event = ptr + (queue_size * sizeof (vring_packed_desc_t));
+ vring->driver_event->off_wrap = 0;
+ vring->driver_event->flags = VRING_EVENT_F_DISABLE;
+
+ vring->device_event =
+ ptr +
+ (((queue_size * sizeof (vring_packed_desc_t)) +
+ sizeof (vring_desc_event_t) + VIRTIO_PCI_VRING_ALIGN -
+ 1) & ~(VIRTIO_PCI_VRING_ALIGN - 1));
+ vring->device_event->off_wrap = 0;
+ vring->device_event->flags = 0;
+
+ vring->queue_id = queue_num;
+
+ vring->avail_wrap_counter = 1;
+ vring->used_wrap_counter = 1;
+
+ ASSERT (vring->buffers == 0);
+ vec_validate_aligned (vring->buffers, queue_size, CLIB_CACHE_LINE_BYTES);
+ if (queue_num % 2)
+ {
+ virtio_log_debug (vif, "tx-queue: number %u, size %u", queue_num,
+ queue_size);
+ clib_memset_u32 (vring->buffers, ~0, queue_size);
+ }
+ else
+ {
+ virtio_log_debug (vif, "rx-queue: number %u, size %u", queue_num,
+ queue_size);
+ }
+ vring->size = queue_size;
+ if (vif->virtio_pci_func->setup_queue (vm, vif, queue_num, (void *) vring))
+ return clib_error_return (0, "error in queue address setup");
+
+ vring->queue_notify_offset =
+ vif->notify_off_multiplier *
+ vif->virtio_pci_func->get_queue_notify_off (vm, vif, queue_num);
+ virtio_log_debug (vif, "queue-notify-offset: number %u, offset %u",
+ queue_num, vring->queue_notify_offset);
+
+ return error;
+}
+
+clib_error_t *
+virtio_pci_vring_init (vlib_main_t * vm, virtio_if_t * vif, u16 queue_num)
+{
+ if (vif->is_packed)
+ return virtio_pci_vring_packed_init (vm, vif, queue_num);
+ else
+ return virtio_pci_vring_split_init (vm, vif, queue_num);
+}
+
static void
virtio_negotiate_features (vlib_main_t * vm, virtio_if_t * vif,
u64 req_features)
@@ -605,6 +931,13 @@
if (vif->is_modern)
supported_features |= VIRTIO_FEATURE (VIRTIO_F_VERSION_1);
+ if (vif->is_packed)
+ {
+ supported_features |=
+ (VIRTIO_FEATURE (VIRTIO_F_RING_PACKED) |
+ VIRTIO_FEATURE (VIRTIO_F_IN_ORDER));
+ }
+
if (req_features == 0)
{
req_features = supported_features;
@@ -621,6 +954,9 @@
vif->features &= ~VIRTIO_FEATURE (VIRTIO_NET_F_MTU);
}
+ if ((vif->features & (VIRTIO_FEATURE (VIRTIO_F_RING_PACKED))) == 0)
+ vif->is_packed = 0;
+
vif->virtio_pci_func->set_driver_features (vm, vif, vif->features);
vif->features = vif->virtio_pci_func->get_driver_features (vm, vif);
}
@@ -1034,6 +1370,8 @@
vif->dev_instance = vif - vim->interfaces;
vif->per_interface_next_index = ~0;
vif->pci_addr.as_u32 = args->addr;
+ if (args->virtio_flags & VIRTIO_FLAG_PACKED)
+ vif->is_packed = 1;
if ((error =
vlib_pci_device_open (vm, (vlib_pci_addr_t *) & vif->pci_addr,
@@ -1260,7 +1598,7 @@
virtio_vring_t *vring = vec_elt_at_index (vif->rxq_vrings, i);
if (vring->used)
{
- virtio_free_rx_buffers (vm, vring);
+ virtio_free_buffers (vm, vring);
}
vec_free (vring->buffers);
vlib_physmem_free (vm, vring->desc);
@@ -1271,9 +1609,11 @@
virtio_vring_t *vring = vec_elt_at_index (vif->txq_vrings, i);
if (vring->used)
{
- virtio_free_used_desc (vm, vring);
+ virtio_free_buffers (vm, vring);
}
vec_free (vring->buffers);
+ gro_flow_table_free (vring->flow_table);
+ virtio_vring_buffering_free (vm, vring->buffering);
clib_spinlock_free (&vring->lockp);
vlib_physmem_free (vm, vring->desc);
}
diff --git a/src/vnet/devices/virtio/virtio.c b/src/vnet/devices/virtio/virtio.c
index 925ad09..99f581a 100644
--- a/src/vnet/devices/virtio/virtio.c
+++ b/src/vnet/devices/virtio/virtio.c
@@ -134,7 +134,7 @@
}
inline void
-virtio_free_rx_buffers (vlib_main_t * vm, virtio_vring_t * vring)
+virtio_free_buffers (vlib_main_t * vm, virtio_vring_t * vring)
{
u16 used = vring->desc_in_use;
u16 last = vring->last_used_idx;
@@ -159,7 +159,7 @@
close (vring->call_fd);
if (vring->used)
{
- virtio_free_rx_buffers (vm, vring);
+ virtio_free_buffers (vm, vring);
clib_mem_free (vring->used);
}
if (vring->desc)
@@ -170,32 +170,6 @@
return 0;
}
-inline void
-virtio_free_used_desc (vlib_main_t * vm, virtio_vring_t * vring)
-{
- u16 used = vring->desc_in_use;
- u16 sz = vring->size;
- u16 mask = sz - 1;
- u16 last = vring->last_used_idx;
- u16 n_left = vring->used->idx - last;
-
- if (n_left == 0)
- return;
-
- while (n_left)
- {
- vring_used_elem_t *e = &vring->used->ring[last & mask];
- u16 slot = e->id;
-
- vlib_buffer_free (vm, &vring->buffers[slot], 1);
- used--;
- last++;
- n_left--;
- }
- vring->desc_in_use = used;
- vring->last_used_idx = last;
-}
-
clib_error_t *
virtio_vring_free_tx (vlib_main_t * vm, virtio_if_t * vif, u32 idx)
{
@@ -207,7 +181,7 @@
close (vring->call_fd);
if (vring->used)
{
- virtio_free_used_desc (vm, vring);
+ virtio_free_buffers (vm, vring);
clib_mem_free (vring->used);
}
if (vring->desc)
@@ -408,10 +382,24 @@
" qsz %d, last_used_idx %d, desc_next %d, desc_in_use %d",
vring->size, vring->last_used_idx, vring->desc_next,
vring->desc_in_use);
- vlib_cli_output (vm,
- " avail.flags 0x%x avail.idx %d used.flags 0x%x used.idx %d",
- vring->avail->flags, vring->avail->idx,
- vring->used->flags, vring->used->idx);
+ if (vif->is_packed)
+ {
+ vlib_cli_output (vm,
+ " driver_event.flags 0x%x driver_event.off_wrap %d device_event.flags 0x%x device_event.off_wrap %d",
+ vring->driver_event->flags,
+ vring->driver_event->off_wrap,
+ vring->device_event->flags,
+ vring->device_event->off_wrap);
+ vlib_cli_output (vm,
+ " avail wrap counter %d, used wrap counter %d",
+ vring->avail_wrap_counter,
+ vring->used_wrap_counter);
+ }
+ else
+ vlib_cli_output (vm,
+ " avail.flags 0x%x avail.idx %d used.flags 0x%x used.idx %d",
+ vring->avail->flags, vring->avail->idx,
+ vring->used->flags, vring->used->idx);
if (type & (VIRTIO_IF_TYPE_TAP | VIRTIO_IF_TYPE_TUN))
{
vlib_cli_output (vm, " kickfd %d, callfd %d", vring->kick_fd,
@@ -421,17 +409,29 @@
{
vlib_cli_output (vm, "\n descriptor table:\n");
vlib_cli_output (vm,
- " id addr len flags next user_addr\n");
+ " id addr len flags next/id user_addr\n");
vlib_cli_output (vm,
- " ===== ================== ===== ====== ===== ==================\n");
+ " ===== ================== ===== ====== ======= ==================\n");
for (j = 0; j < vring->size; j++)
{
- vring_desc_t *desc = &vring->desc[j];
- vlib_cli_output (vm,
- " %-5d 0x%016lx %-5d 0x%04x %-5d 0x%016lx\n",
- j, desc->addr,
- desc->len,
- desc->flags, desc->next, desc->addr);
+ if (vif->is_packed)
+ {
+ vring_packed_desc_t *desc = &vring->packed_desc[j];
+ vlib_cli_output (vm,
+ " %-5d 0x%016lx %-5d 0x%04x %-8d 0x%016lx\n",
+ j, desc->addr,
+ desc->len,
+ desc->flags, desc->id, desc->addr);
+ }
+ else
+ {
+ vring_desc_t *desc = &vring->desc[j];
+ vlib_cli_output (vm,
+ " %-5d 0x%016lx %-5d 0x%04x %-8d 0x%016lx\n",
+ j, desc->addr,
+ desc->len,
+ desc->flags, desc->next, desc->addr);
+ }
}
}
}
@@ -443,10 +443,24 @@
" qsz %d, last_used_idx %d, desc_next %d, desc_in_use %d",
vring->size, vring->last_used_idx, vring->desc_next,
vring->desc_in_use);
- vlib_cli_output (vm,
- " avail.flags 0x%x avail.idx %d used.flags 0x%x used.idx %d",
- vring->avail->flags, vring->avail->idx,
- vring->used->flags, vring->used->idx);
+ if (vif->is_packed)
+ {
+ vlib_cli_output (vm,
+ " driver_event.flags 0x%x driver_event.off_wrap %d device_event.flags 0x%x device_event.off_wrap %d",
+ vring->driver_event->flags,
+ vring->driver_event->off_wrap,
+ vring->device_event->flags,
+ vring->device_event->off_wrap);
+ vlib_cli_output (vm,
+ " avail wrap counter %d, used wrap counter %d",
+ vring->avail_wrap_counter,
+ vring->used_wrap_counter);
+ }
+ else
+ vlib_cli_output (vm,
+ " avail.flags 0x%x avail.idx %d used.flags 0x%x used.idx %d",
+ vring->avail->flags, vring->avail->idx,
+ vring->used->flags, vring->used->idx);
if (type & (VIRTIO_IF_TYPE_TAP | VIRTIO_IF_TYPE_TUN))
{
vlib_cli_output (vm, " kickfd %d, callfd %d", vring->kick_fd,
@@ -466,17 +480,29 @@
{
vlib_cli_output (vm, "\n descriptor table:\n");
vlib_cli_output (vm,
- " id addr len flags next user_addr\n");
+ " id addr len flags next/id user_addr\n");
vlib_cli_output (vm,
- " ===== ================== ===== ====== ===== ==================\n");
+ " ===== ================== ===== ====== ======== ==================\n");
for (j = 0; j < vring->size; j++)
{
- vring_desc_t *desc = &vring->desc[j];
- vlib_cli_output (vm,
- " %-5d 0x%016lx %-5d 0x%04x %-5d 0x%016lx\n",
- j, desc->addr,
- desc->len,
- desc->flags, desc->next, desc->addr);
+ if (vif->is_packed)
+ {
+ vring_packed_desc_t *desc = &vring->packed_desc[j];
+ vlib_cli_output (vm,
+ " %-5d 0x%016lx %-5d 0x%04x %-8d 0x%016lx\n",
+ j, desc->addr,
+ desc->len,
+ desc->flags, desc->id, desc->addr);
+ }
+ else
+ {
+ vring_desc_t *desc = &vring->desc[j];
+ vlib_cli_output (vm,
+ " %-5d 0x%016lx %-5d 0x%04x %-8d 0x%016lx\n",
+ j, desc->addr,
+ desc->len,
+ desc->flags, desc->next, desc->addr);
+ }
}
}
}
@@ -489,10 +515,26 @@
" qsz %d, last_used_idx %d, desc_next %d, desc_in_use %d",
vring->size, vring->last_used_idx,
vring->desc_next, vring->desc_in_use);
- vlib_cli_output (vm,
- " avail.flags 0x%x avail.idx %d used.flags 0x%x used.idx %d",
- vring->avail->flags, vring->avail->idx,
- vring->used->flags, vring->used->idx);
+ if (vif->is_packed)
+ {
+ vlib_cli_output (vm,
+ " driver_event.flags 0x%x driver_event.off_wrap %d device_event.flags 0x%x device_event.off_wrap %d",
+ vring->driver_event->flags,
+ vring->driver_event->off_wrap,
+ vring->device_event->flags,
+ vring->device_event->off_wrap);
+ vlib_cli_output (vm,
+ " avail wrap counter %d, used wrap counter %d",
+ vring->avail_wrap_counter,
+ vring->used_wrap_counter);
+ }
+ else
+ {
+ vlib_cli_output (vm,
+ " avail.flags 0x%x avail.idx %d used.flags 0x%x used.idx %d",
+ vring->avail->flags, vring->avail->idx,
+ vring->used->flags, vring->used->idx);
+ }
if (type & (VIRTIO_IF_TYPE_TAP | VIRTIO_IF_TYPE_TUN))
{
vlib_cli_output (vm, " kickfd %d, callfd %d", vring->kick_fd,
@@ -502,17 +544,29 @@
{
vlib_cli_output (vm, "\n descriptor table:\n");
vlib_cli_output (vm,
- " id addr len flags next user_addr\n");
+ " id addr len flags next/id user_addr\n");
vlib_cli_output (vm,
- " ===== ================== ===== ====== ===== ==================\n");
+ " ===== ================== ===== ====== ======== ==================\n");
for (j = 0; j < vring->size; j++)
{
- vring_desc_t *desc = &vring->desc[j];
- vlib_cli_output (vm,
- " %-5d 0x%016lx %-5d 0x%04x %-5d 0x%016lx\n",
- j, desc->addr,
- desc->len,
- desc->flags, desc->next, desc->addr);
+ if (vif->is_packed)
+ {
+ vring_packed_desc_t *desc = &vring->packed_desc[j];
+ vlib_cli_output (vm,
+ " %-5d 0x%016lx %-5d 0x%04x %-8d 0x%016lx\n",
+ j, desc->addr,
+ desc->len,
+ desc->flags, desc->id, desc->addr);
+ }
+ else
+ {
+ vring_desc_t *desc = &vring->desc[j];
+ vlib_cli_output (vm,
+ " %-5d 0x%016lx %-5d 0x%04x %-8d 0x%016lx\n",
+ j, desc->addr,
+ desc->len,
+ desc->flags, desc->next, desc->addr);
+ }
}
}
}
diff --git a/src/vnet/devices/virtio/virtio.h b/src/vnet/devices/virtio/virtio.h
index fda7236..035dc9c 100644
--- a/src/vnet/devices/virtio/virtio.h
+++ b/src/vnet/devices/virtio/virtio.h
@@ -64,29 +64,46 @@
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
clib_spinlock_t lockp;
- vring_desc_t *desc;
- vring_used_t *used;
- vring_avail_t *avail;
+ union
+ {
+ struct
+ {
+ vring_desc_t *desc;
+ vring_used_t *used;
+ vring_avail_t *avail;
+ };
+ struct
+ {
+ vring_packed_desc_t *packed_desc;
+ vring_desc_event_t *driver_event;
+ vring_desc_event_t *device_event;
+ };
+ };
+ u32 *buffers;
+ u16 size;
+ u16 queue_id;
u16 desc_in_use;
u16 desc_next;
+ u16 last_used_idx;
+ u16 last_kick_avail_idx;
union
{
struct
{
int kick_fd;
int call_fd;
+ u32 call_file_index;
};
- u16 queue_notify_offset;
+ struct
+ {
+ u16 avail_wrap_counter;
+ u16 used_wrap_counter;
+ u16 queue_notify_offset;
+ };
};
- u8 buffer_pool_index;
- u16 size;
- u16 queue_id;
#define VRING_TX_OUT_OF_ORDER 1
u16 flags;
- u32 *buffers;
- u16 last_used_idx;
- u16 last_kick_avail_idx;
- u32 call_file_index;
+ u8 buffer_pool_index;
vnet_hw_if_rx_mode mode;
virtio_vring_buffering_t *buffering;
gro_flow_table_t *flow_table;
@@ -190,6 +207,7 @@
};
};
const virtio_pci_func_t *virtio_pci_func;
+ int is_packed;
} virtio_if_t;
typedef struct
@@ -214,8 +232,7 @@
u32 idx);
void virtio_vring_set_numa_node (vlib_main_t * vm, virtio_if_t * vif,
u32 idx);
-extern void virtio_free_used_desc (vlib_main_t * vm, virtio_vring_t * vring);
-extern void virtio_free_rx_buffers (vlib_main_t * vm, virtio_vring_t * vring);
+extern void virtio_free_buffers (vlib_main_t * vm, virtio_vring_t * vring);
extern void virtio_set_net_hdr_size (virtio_if_t * vif);
extern void virtio_show (vlib_main_t * vm, u32 * hw_if_indices, u8 show_descr,
u32 type);
diff --git a/src/vnet/devices/virtio/virtio_pci_modern.c b/src/vnet/devices/virtio/virtio_pci_modern.c
index 1934f98..8e090ff 100644
--- a/src/vnet/devices/virtio/virtio_pci_modern.c
+++ b/src/vnet/devices/virtio/virtio_pci_modern.c
@@ -268,24 +268,39 @@
virtio_pci_modern_setup_queue (vlib_main_t * vm, virtio_if_t * vif,
u16 queue_id, void *p)
{
- vring_t vr;
+ u64 desc, avail, used;
u16 queue_size = 0;
virtio_pci_modern_set_queue_select (vif, queue_id);
queue_size = virtio_pci_modern_get_queue_size (vm, vif, queue_id);
- vring_init (&vr, queue_size, p, VIRTIO_PCI_VRING_ALIGN);
- u64 desc = vlib_physmem_get_pa (vm, vr.desc);
+ if (vif->is_packed)
+ {
+ virtio_vring_t *vring = (virtio_vring_t *) p;
+
+ desc = vlib_physmem_get_pa (vm, vring->packed_desc);
+ avail = vlib_physmem_get_pa (vm, vring->driver_event);
+ used = vlib_physmem_get_pa (vm, vring->device_event);
+ }
+ else
+ {
+ vring_t vr;
+
+ vring_init (&vr, queue_size, p, VIRTIO_PCI_VRING_ALIGN);
+
+ desc = vlib_physmem_get_pa (vm, vr.desc);
+ avail = vlib_physmem_get_pa (vm, vr.avail);
+ used = vlib_physmem_get_pa (vm, vr.used);
+ }
+
virtio_pci_modern_set_queue_desc (vif, desc);
if (desc != virtio_pci_modern_get_queue_desc (vif))
return 1;
- u64 avail = vlib_physmem_get_pa (vm, vr.avail);
virtio_pci_modern_set_queue_driver (vif, avail);
if (avail != virtio_pci_modern_get_queue_driver (vif))
return 1;
- u64 used = vlib_physmem_get_pa (vm, vr.used);
virtio_pci_modern_set_queue_device (vif, used);
if (used != virtio_pci_modern_get_queue_device (vif))
return 1;