commit 37623706014a591186585613e6de19686abe6104
author Damjan Marion <damarion@cisco.com> Tue Sep 20 11:25:27 2016 +0200
committer Dave Barach <openvpp@barachs.net> Thu Sep 22 21:01:19 2016 +0000
tree d25df4b4a39acc5eeb9d70de1d6a2e4aa01aae19
parent c9e0843d5e37f3360364b3545e4c1df8630b10e8

vhost-user: branchless SSE4.2 implementation of map_guest_mem()

Change-Id: I64349830c1f7534a8d090572e9473c51c0818e51
Signed-off-by: Damjan Marion <damarion@cisco.com>

vnet/vnet/devices/virtio/vhost-user.c

diff --git a/vnet/vnet/devices/virtio/vhost-user.c b/vnet/vnet/devices/virtio/vhost-user.c
index 046a02e..6182ffa 100644
--- a/vnet/vnet/devices/virtio/vhost-user.c
+++ b/vnet/vnet/devices/virtio/vhost-user.c
@@ -153,6 +153,45 @@
 map_guest_mem (vhost_user_intf_t * vui, uword addr)
 {
   int i;
+#if __SSE4_2__
+  __m128i rl, rh, al, ah, r;
+  al = _mm_set1_epi64x (addr + 1);
+  ah = _mm_set1_epi64x (addr);
+
+  rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[0]);
+  rl = _mm_cmpgt_epi64 (al, rl);
+  rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[0]);
+  rh = _mm_cmpgt_epi64 (rh, ah);
+  r = _mm_and_si128 (rl, rh);
+
+  rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[2]);
+  rl = _mm_cmpgt_epi64 (al, rl);
+  rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[2]);
+  rh = _mm_cmpgt_epi64 (rh, ah);
+  r = _mm_blend_epi16 (r, _mm_and_si128 (rl, rh), 0x22);
+
+  rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[4]);
+  rl = _mm_cmpgt_epi64 (al, rl);
+  rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[4]);
+  rh = _mm_cmpgt_epi64 (rh, ah);
+  r = _mm_blend_epi16 (r, _mm_and_si128 (rl, rh), 0x44);
+
+  rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[6]);
+  rl = _mm_cmpgt_epi64 (al, rl);
+  rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[6]);
+  rh = _mm_cmpgt_epi64 (rh, ah);
+  r = _mm_blend_epi16 (r, _mm_and_si128 (rl, rh), 0x88);
+
+  r = _mm_shuffle_epi8 (r, _mm_set_epi64x (0, 0x0e060c040a020800));
+  i = __builtin_ctzll (_mm_movemask_epi8 (r));
+
+  if (i < vui->nregions)
+    {
+      return (void *) (vui->region_mmap_addr[i] + addr -
+		       vui->regions[i].guest_phys_addr);
+    }
+
+#else
   for (i = 0; i < vui->nregions; i++)
     {
       if ((vui->regions[i].guest_phys_addr <= addr) &&
@@ -163,6 +202,7 @@
 			   vui->regions[i].guest_phys_addr);
 	}
     }
+#endif
   DBG_VQ ("failed to map guest mem addr %llx", addr);
   return 0;
 }
@@ -463,6 +503,9 @@
 
 	  vui->region_mmap_addr[i] = mmap (0, map_sz, PROT_READ | PROT_WRITE,
 					   MAP_SHARED, fds[i], 0);
+	  vui->region_guest_addr_lo[i] = vui->regions[i].guest_phys_addr;
+	  vui->region_guest_addr_hi[i] = vui->regions[i].guest_phys_addr +
+	    vui->regions[i].memory_size;
 
 	  DBG_SOCK
 	    ("map memory region %d addr 0 len 0x%lx fd %d mapped 0x%lx "

vnet/vnet/devices/virtio/vhost-user.h

diff --git a/vnet/vnet/devices/virtio/vhost-user.h b/vnet/vnet/devices/virtio/vhost-user.h
index 91369c3..af4eb64 100644
--- a/vnet/vnet/devices/virtio/vhost-user.h
+++ b/vnet/vnet/devices/virtio/vhost-user.h
@@ -232,6 +232,8 @@
   u32 num_vrings;
   vhost_user_memory_region_t regions[VHOST_MEMORY_MAX_NREGIONS];
   void *region_mmap_addr[VHOST_MEMORY_MAX_NREGIONS];
+  u64 region_guest_addr_lo[VHOST_MEMORY_MAX_NREGIONS];
+  u64 region_guest_addr_hi[VHOST_MEMORY_MAX_NREGIONS];
   u32 region_mmap_fd[VHOST_MEMORY_MAX_NREGIONS];
   vhost_user_vring_t vrings[2];
   int virtio_net_hdr_sz;