Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1 | /* |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2 | *------------------------------------------------------------------ |
| 3 | * vhost.c - vhost-user |
| 4 | * |
| 5 | * Copyright (c) 2014 Cisco and/or its affiliates. |
| 6 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 7 | * you may not use this file except in compliance with the License. |
| 8 | * You may obtain a copy of the License at: |
| 9 | * |
| 10 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | * |
| 12 | * Unless required by applicable law or agreed to in writing, software |
| 13 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 15 | * See the License for the specific language governing permissions and |
| 16 | * limitations under the License. |
| 17 | *------------------------------------------------------------------ |
| 18 | */ |
| 19 | |
| 20 | #include <fcntl.h> /* for open */ |
| 21 | #include <sys/ioctl.h> |
| 22 | #include <sys/socket.h> |
| 23 | #include <sys/un.h> |
| 24 | #include <sys/stat.h> |
| 25 | #include <sys/types.h> |
| 26 | #include <sys/uio.h> /* for iovec */ |
| 27 | #include <netinet/in.h> |
| 28 | #include <sys/vfs.h> |
| 29 | |
| 30 | #include <linux/if_arp.h> |
| 31 | #include <linux/if_tun.h> |
| 32 | |
| 33 | #include <vlib/vlib.h> |
| 34 | #include <vlib/unix/unix.h> |
| 35 | |
| 36 | #include <vnet/ip/ip.h> |
| 37 | |
| 38 | #include <vnet/ethernet/ethernet.h> |
Damjan Marion | 8bdc63b | 2016-11-02 14:48:21 +0100 | [diff] [blame] | 39 | #include <vnet/devices/devices.h> |
Damjan Marion | 2231150 | 2016-10-28 20:30:15 +0200 | [diff] [blame] | 40 | #include <vnet/feature/feature.h> |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 41 | |
| 42 | #include <vnet/devices/virtio/vhost-user.h> |
| 43 | |
Billy McFall | a92501a | 2016-11-23 12:45:29 -0500 | [diff] [blame] | 44 | /** |
| 45 | * @file |
| 46 | * @brief vHost User Device Driver. |
| 47 | * |
| 48 | * This file contains the source code for vHost User interface. |
| 49 | */ |
| 50 | |
| 51 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 52 | #define VHOST_USER_DEBUG_SOCKET 0 |
Pierre Pfister | 116ea4b | 2016-11-08 15:49:28 +0000 | [diff] [blame] | 53 | #define VHOST_DEBUG_VQ 0 |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 54 | |
| 55 | #if VHOST_USER_DEBUG_SOCKET == 1 |
| 56 | #define DBG_SOCK(args...) clib_warning(args); |
| 57 | #else |
| 58 | #define DBG_SOCK(args...) |
| 59 | #endif |
| 60 | |
Pierre Pfister | 116ea4b | 2016-11-08 15:49:28 +0000 | [diff] [blame] | 61 | #if VHOST_DEBUG_VQ == 1 |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 62 | #define DBG_VQ(args...) clib_warning(args); |
| 63 | #else |
| 64 | #define DBG_VQ(args...) |
| 65 | #endif |
| 66 | |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 67 | /* |
| 68 | * When an RX queue is down but active, received packets |
| 69 | * must be discarded. This value controls up to how many |
| 70 | * packets will be discarded during each round. |
| 71 | */ |
| 72 | #define VHOST_USER_DOWN_DISCARD_COUNT 256 |
| 73 | |
| 74 | /* |
| 75 | * When the number of available buffers gets under this threshold, |
| 76 | * RX node will start discarding packets. |
| 77 | */ |
| 78 | #define VHOST_USER_RX_BUFFER_STARVATION 32 |
| 79 | |
| 80 | /* |
| 81 | * On the receive side, the host should free descriptors as soon |
| 82 | * as possible in order to avoid TX drop in the VM. |
| 83 | * This value controls the number of copy operations that are stacked |
| 84 | * before copy is done for all and descriptors are given back to |
| 85 | * the guest. |
| 86 | * The value 64 was obtained by testing (48 and 128 were not as good). |
| 87 | */ |
| 88 | #define VHOST_USER_RX_COPY_THRESHOLD 64 |
| 89 | |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 90 | #define UNIX_GET_FD(unixfd_idx) \ |
| 91 | (unixfd_idx != ~0) ? \ |
| 92 | pool_elt_at_index (unix_main.file_pool, \ |
| 93 | unixfd_idx)->file_descriptor : -1; |
| 94 | |
Pierre Pfister | 116ea4b | 2016-11-08 15:49:28 +0000 | [diff] [blame] | 95 | #define foreach_virtio_trace_flags \ |
| 96 | _ (SIMPLE_CHAINED, 0, "Simple descriptor chaining") \ |
| 97 | _ (SINGLE_DESC, 1, "Single descriptor packet") \ |
| 98 | _ (INDIRECT, 2, "Indirect descriptor") \ |
| 99 | _ (MAP_ERROR, 4, "Memory mapping error") |
| 100 | |
| 101 | typedef enum |
| 102 | { |
| 103 | #define _(n,i,s) VIRTIO_TRACE_F_##n, |
| 104 | foreach_virtio_trace_flags |
| 105 | #undef _ |
| 106 | } virtio_trace_flag_t; |
| 107 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 108 | vlib_node_registration_t vhost_user_input_node; |
| 109 | |
| 110 | #define foreach_vhost_user_tx_func_error \ |
Pierre Pfister | 328e99b | 2016-02-12 13:18:42 +0000 | [diff] [blame] | 111 | _(NONE, "no error") \ |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 112 | _(NOT_READY, "vhost vring not ready") \ |
| 113 | _(DOWN, "vhost interface is down") \ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 114 | _(PKT_DROP_NOBUF, "tx packet drops (no available descriptors)") \ |
Pierre Pfister | ba1d046 | 2016-07-27 16:38:20 +0100 | [diff] [blame] | 115 | _(PKT_DROP_NOMRG, "tx packet drops (cannot merge descriptors)") \ |
| 116 | _(MMAP_FAIL, "mmap failure") \ |
| 117 | _(INDIRECT_OVERFLOW, "indirect descriptor table overflow") |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 118 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 119 | typedef enum |
| 120 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 121 | #define _(f,s) VHOST_USER_TX_FUNC_ERROR_##f, |
| 122 | foreach_vhost_user_tx_func_error |
| 123 | #undef _ |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 124 | VHOST_USER_TX_FUNC_N_ERROR, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 125 | } vhost_user_tx_func_error_t; |
| 126 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 127 | static char *vhost_user_tx_func_error_strings[] = { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 128 | #define _(n,s) s, |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 129 | foreach_vhost_user_tx_func_error |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 130 | #undef _ |
| 131 | }; |
| 132 | |
| 133 | #define foreach_vhost_user_input_func_error \ |
| 134 | _(NO_ERROR, "no error") \ |
Pierre Pfister | 328e99b | 2016-02-12 13:18:42 +0000 | [diff] [blame] | 135 | _(NO_BUFFER, "no available buffer") \ |
| 136 | _(MMAP_FAIL, "mmap failure") \ |
Pierre Pfister | ba1d046 | 2016-07-27 16:38:20 +0100 | [diff] [blame] | 137 | _(INDIRECT_OVERFLOW, "indirect descriptor overflows table") \ |
| 138 | _(UNDERSIZED_FRAME, "undersized ethernet frame received (< 14 bytes)") \ |
| 139 | _(FULL_RX_QUEUE, "full rx queue (possible driver tx drop)") |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 140 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 141 | typedef enum |
| 142 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 143 | #define _(f,s) VHOST_USER_INPUT_FUNC_ERROR_##f, |
| 144 | foreach_vhost_user_input_func_error |
| 145 | #undef _ |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 146 | VHOST_USER_INPUT_FUNC_N_ERROR, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 147 | } vhost_user_input_func_error_t; |
| 148 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 149 | static char *vhost_user_input_func_error_strings[] = { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 150 | #define _(n,s) s, |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 151 | foreach_vhost_user_input_func_error |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 152 | #undef _ |
| 153 | }; |
| 154 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 155 | /* *INDENT-OFF* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 156 | static vhost_user_main_t vhost_user_main = { |
| 157 | .mtu_bytes = 1518, |
| 158 | }; |
| 159 | |
| 160 | VNET_HW_INTERFACE_CLASS (vhost_interface_class, static) = { |
| 161 | .name = "vhost-user", |
| 162 | }; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 163 | /* *INDENT-ON* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 164 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 165 | static u8 * |
| 166 | format_vhost_user_interface_name (u8 * s, va_list * args) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 167 | { |
| 168 | u32 i = va_arg (*args, u32); |
| 169 | u32 show_dev_instance = ~0; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 170 | vhost_user_main_t *vum = &vhost_user_main; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 171 | |
| 172 | if (i < vec_len (vum->show_dev_instance_by_real_dev_instance)) |
| 173 | show_dev_instance = vum->show_dev_instance_by_real_dev_instance[i]; |
| 174 | |
| 175 | if (show_dev_instance != ~0) |
| 176 | i = show_dev_instance; |
| 177 | |
| 178 | s = format (s, "VirtualEthernet0/0/%d", i); |
| 179 | return s; |
| 180 | } |
| 181 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 182 | static int |
| 183 | vhost_user_name_renumber (vnet_hw_interface_t * hi, u32 new_dev_instance) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 184 | { |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 185 | // FIXME: check if the new dev instance is already used |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 186 | vhost_user_main_t *vum = &vhost_user_main; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 187 | vec_validate_init_empty (vum->show_dev_instance_by_real_dev_instance, |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 188 | hi->dev_instance, ~0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 189 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 190 | vum->show_dev_instance_by_real_dev_instance[hi->dev_instance] = |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 191 | new_dev_instance; |
| 192 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 193 | DBG_SOCK ("renumbered vhost-user interface dev_instance %d to %d", |
| 194 | hi->dev_instance, new_dev_instance); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 195 | |
| 196 | return 0; |
| 197 | } |
| 198 | |
Pierre Pfister | 11f9205 | 2016-09-21 08:08:55 +0100 | [diff] [blame] | 199 | static_always_inline void * |
| 200 | map_guest_mem (vhost_user_intf_t * vui, uword addr, u32 * hint) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 201 | { |
Pierre Pfister | 11f9205 | 2016-09-21 08:08:55 +0100 | [diff] [blame] | 202 | int i = *hint; |
| 203 | if (PREDICT_TRUE ((vui->regions[i].guest_phys_addr <= addr) && |
| 204 | ((vui->regions[i].guest_phys_addr + |
| 205 | vui->regions[i].memory_size) > addr))) |
| 206 | { |
| 207 | return (void *) (vui->region_mmap_addr[i] + addr - |
| 208 | vui->regions[i].guest_phys_addr); |
| 209 | } |
Damjan Marion | 3762370 | 2016-09-20 11:25:27 +0200 | [diff] [blame] | 210 | #if __SSE4_2__ |
| 211 | __m128i rl, rh, al, ah, r; |
| 212 | al = _mm_set1_epi64x (addr + 1); |
| 213 | ah = _mm_set1_epi64x (addr); |
| 214 | |
| 215 | rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[0]); |
| 216 | rl = _mm_cmpgt_epi64 (al, rl); |
| 217 | rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[0]); |
| 218 | rh = _mm_cmpgt_epi64 (rh, ah); |
| 219 | r = _mm_and_si128 (rl, rh); |
| 220 | |
| 221 | rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[2]); |
| 222 | rl = _mm_cmpgt_epi64 (al, rl); |
| 223 | rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[2]); |
| 224 | rh = _mm_cmpgt_epi64 (rh, ah); |
| 225 | r = _mm_blend_epi16 (r, _mm_and_si128 (rl, rh), 0x22); |
| 226 | |
| 227 | rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[4]); |
| 228 | rl = _mm_cmpgt_epi64 (al, rl); |
| 229 | rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[4]); |
| 230 | rh = _mm_cmpgt_epi64 (rh, ah); |
| 231 | r = _mm_blend_epi16 (r, _mm_and_si128 (rl, rh), 0x44); |
| 232 | |
| 233 | rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[6]); |
| 234 | rl = _mm_cmpgt_epi64 (al, rl); |
| 235 | rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[6]); |
| 236 | rh = _mm_cmpgt_epi64 (rh, ah); |
| 237 | r = _mm_blend_epi16 (r, _mm_and_si128 (rl, rh), 0x88); |
| 238 | |
| 239 | r = _mm_shuffle_epi8 (r, _mm_set_epi64x (0, 0x0e060c040a020800)); |
| 240 | i = __builtin_ctzll (_mm_movemask_epi8 (r)); |
| 241 | |
| 242 | if (i < vui->nregions) |
| 243 | { |
Pierre Pfister | 11f9205 | 2016-09-21 08:08:55 +0100 | [diff] [blame] | 244 | *hint = i; |
Damjan Marion | 3762370 | 2016-09-20 11:25:27 +0200 | [diff] [blame] | 245 | return (void *) (vui->region_mmap_addr[i] + addr - |
| 246 | vui->regions[i].guest_phys_addr); |
| 247 | } |
| 248 | |
| 249 | #else |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 250 | for (i = 0; i < vui->nregions; i++) |
| 251 | { |
| 252 | if ((vui->regions[i].guest_phys_addr <= addr) && |
| 253 | ((vui->regions[i].guest_phys_addr + vui->regions[i].memory_size) > |
| 254 | addr)) |
| 255 | { |
Pierre Pfister | 11f9205 | 2016-09-21 08:08:55 +0100 | [diff] [blame] | 256 | *hint = i; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 257 | return (void *) (vui->region_mmap_addr[i] + addr - |
| 258 | vui->regions[i].guest_phys_addr); |
| 259 | } |
| 260 | } |
Damjan Marion | 3762370 | 2016-09-20 11:25:27 +0200 | [diff] [blame] | 261 | #endif |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 262 | DBG_VQ ("failed to map guest mem addr %llx", addr); |
Pierre Pfister | 11f9205 | 2016-09-21 08:08:55 +0100 | [diff] [blame] | 263 | *hint = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 264 | return 0; |
| 265 | } |
| 266 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 267 | static inline void * |
| 268 | map_user_mem (vhost_user_intf_t * vui, uword addr) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 269 | { |
| 270 | int i; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 271 | for (i = 0; i < vui->nregions; i++) |
| 272 | { |
| 273 | if ((vui->regions[i].userspace_addr <= addr) && |
| 274 | ((vui->regions[i].userspace_addr + vui->regions[i].memory_size) > |
| 275 | addr)) |
| 276 | { |
| 277 | return (void *) (vui->region_mmap_addr[i] + addr - |
| 278 | vui->regions[i].userspace_addr); |
| 279 | } |
| 280 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 281 | return 0; |
| 282 | } |
| 283 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 284 | static long |
| 285 | get_huge_page_size (int fd) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 286 | { |
| 287 | struct statfs s; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 288 | fstatfs (fd, &s); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 289 | return s.f_bsize; |
| 290 | } |
| 291 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 292 | static void |
| 293 | unmap_all_mem_regions (vhost_user_intf_t * vui) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 294 | { |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 295 | int i, r; |
| 296 | for (i = 0; i < vui->nregions; i++) |
| 297 | { |
| 298 | if (vui->region_mmap_addr[i] != (void *) -1) |
| 299 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 300 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 301 | long page_sz = get_huge_page_size (vui->region_mmap_fd[i]); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 302 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 303 | ssize_t map_sz = (vui->regions[i].memory_size + |
| 304 | vui->regions[i].mmap_offset + |
| 305 | page_sz) & ~(page_sz - 1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 306 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 307 | r = |
| 308 | munmap (vui->region_mmap_addr[i] - vui->regions[i].mmap_offset, |
| 309 | map_sz); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 310 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 311 | DBG_SOCK |
| 312 | ("unmap memory region %d addr 0x%lx len 0x%lx page_sz 0x%x", i, |
| 313 | vui->region_mmap_addr[i], map_sz, page_sz); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 314 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 315 | vui->region_mmap_addr[i] = (void *) -1; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 316 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 317 | if (r == -1) |
| 318 | { |
| 319 | clib_warning ("failed to unmap memory region (errno %d)", |
| 320 | errno); |
| 321 | } |
| 322 | close (vui->region_mmap_fd[i]); |
| 323 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 324 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 325 | vui->nregions = 0; |
| 326 | } |
| 327 | |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 328 | static void |
| 329 | vhost_user_tx_thread_placement (vhost_user_intf_t * vui) |
| 330 | { |
| 331 | //Let's try to assign one queue to each thread |
| 332 | u32 qid = 0; |
| 333 | u32 cpu_index = 0; |
| 334 | vui->use_tx_spinlock = 0; |
| 335 | while (1) |
| 336 | { |
| 337 | for (qid = 0; qid < VHOST_VRING_MAX_N / 2; qid++) |
| 338 | { |
| 339 | vhost_user_vring_t *rxvq = &vui->vrings[VHOST_VRING_IDX_RX (qid)]; |
| 340 | if (!rxvq->started || !rxvq->enabled) |
| 341 | continue; |
| 342 | |
| 343 | vui->per_cpu_tx_qid[cpu_index] = qid; |
| 344 | cpu_index++; |
| 345 | if (cpu_index == vlib_get_thread_main ()->n_vlib_mains) |
| 346 | return; |
| 347 | } |
| 348 | //We need to loop, meaning the spinlock has to be used |
| 349 | vui->use_tx_spinlock = 1; |
| 350 | if (cpu_index == 0) |
| 351 | { |
| 352 | //Could not find a single valid one |
| 353 | for (cpu_index = 0; |
| 354 | cpu_index < vlib_get_thread_main ()->n_vlib_mains; cpu_index++) |
| 355 | { |
| 356 | vui->per_cpu_tx_qid[cpu_index] = 0; |
| 357 | } |
| 358 | return; |
| 359 | } |
| 360 | } |
| 361 | } |
| 362 | |
| 363 | static void |
| 364 | vhost_user_rx_thread_placement () |
| 365 | { |
| 366 | vhost_user_main_t *vum = &vhost_user_main; |
| 367 | vhost_user_intf_t *vui; |
| 368 | vhost_cpu_t *vhc; |
| 369 | u32 *workers = 0; |
| 370 | |
| 371 | //Let's list all workers cpu indexes |
| 372 | u32 i; |
| 373 | for (i = vum->input_cpu_first_index; |
| 374 | i < vum->input_cpu_first_index + vum->input_cpu_count; i++) |
| 375 | { |
| 376 | vlib_node_set_state (vlib_mains ? vlib_mains[i] : &vlib_global_main, |
| 377 | vhost_user_input_node.index, |
| 378 | VLIB_NODE_STATE_DISABLED); |
| 379 | vec_add1 (workers, i); |
| 380 | } |
| 381 | |
| 382 | vec_foreach (vhc, vum->cpus) |
| 383 | { |
| 384 | vec_reset_length (vhc->rx_queues); |
| 385 | } |
| 386 | |
| 387 | i = 0; |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 388 | vhost_iface_and_queue_t iaq; |
| 389 | /* *INDENT-OFF* */ |
| 390 | pool_foreach (vui, vum->vhost_user_interfaces, { |
| 391 | u32 *vui_workers = vec_len (vui->workers) ? vui->workers : workers; |
| 392 | u32 qid; |
| 393 | for (qid = 0; qid < VHOST_VRING_MAX_N / 2; qid++) |
| 394 | { |
| 395 | vhost_user_vring_t *txvq = |
| 396 | &vui->vrings[VHOST_VRING_IDX_TX (qid)]; |
| 397 | if (!txvq->started) |
| 398 | continue; |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 399 | |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 400 | i %= vec_len (vui_workers); |
| 401 | u32 cpu_index = vui_workers[i]; |
| 402 | i++; |
| 403 | vhc = &vum->cpus[cpu_index]; |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 404 | |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 405 | iaq.qid = qid; |
| 406 | iaq.vhost_iface_index = vui - vum->vhost_user_interfaces; |
| 407 | vec_add1 (vhc->rx_queues, iaq); |
| 408 | vlib_node_set_state (vlib_mains ? vlib_mains[cpu_index] : |
| 409 | &vlib_global_main, vhost_user_input_node.index, |
| 410 | VLIB_NODE_STATE_POLLING); |
| 411 | } |
| 412 | }); |
| 413 | /* *INDENT-ON* */ |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 414 | } |
| 415 | |
| 416 | static int |
| 417 | vhost_user_thread_placement (u32 sw_if_index, u32 worker_thread_index, u8 del) |
| 418 | { |
| 419 | vhost_user_main_t *vum = &vhost_user_main; |
| 420 | vhost_user_intf_t *vui; |
| 421 | vnet_hw_interface_t *hw; |
| 422 | |
| 423 | if (worker_thread_index < vum->input_cpu_first_index || |
| 424 | worker_thread_index >= |
| 425 | vum->input_cpu_first_index + vum->input_cpu_count) |
| 426 | return -1; |
| 427 | |
| 428 | if (!(hw = vnet_get_sup_hw_interface (vnet_get_main (), sw_if_index))) |
| 429 | return -2; |
| 430 | |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 431 | vui = pool_elt_at_index (vum->vhost_user_interfaces, hw->dev_instance); |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 432 | u32 found = ~0, *w; |
| 433 | vec_foreach (w, vui->workers) |
| 434 | { |
| 435 | if (*w == worker_thread_index) |
| 436 | { |
| 437 | found = w - vui->workers; |
| 438 | break; |
| 439 | } |
| 440 | } |
| 441 | |
| 442 | if (del) |
| 443 | { |
| 444 | if (found == ~0) |
| 445 | return -3; |
| 446 | vec_del1 (vui->workers, found); |
| 447 | } |
| 448 | else if (found == ~0) |
| 449 | { |
| 450 | vec_add1 (vui->workers, worker_thread_index); |
| 451 | } |
| 452 | |
| 453 | vhost_user_rx_thread_placement (); |
| 454 | return 0; |
| 455 | } |
| 456 | |
| 457 | /** @brief Returns whether at least one TX and one RX vring are enabled */ |
| 458 | int |
| 459 | vhost_user_intf_ready (vhost_user_intf_t * vui) |
| 460 | { |
| 461 | int i, found[2] = { }; //RX + TX |
| 462 | |
| 463 | for (i = 0; i < VHOST_VRING_MAX_N; i++) |
| 464 | if (vui->vrings[i].started && vui->vrings[i].enabled) |
| 465 | found[i & 1] = 1; |
| 466 | |
| 467 | return found[0] && found[1]; |
| 468 | } |
| 469 | |
| 470 | static void |
| 471 | vhost_user_update_iface_state (vhost_user_intf_t * vui) |
| 472 | { |
| 473 | /* if we have pointers to descriptor table, go up */ |
| 474 | int is_up = vhost_user_intf_ready (vui); |
| 475 | if (is_up != vui->is_up) |
| 476 | { |
| 477 | DBG_SOCK ("interface %d %s", vui->sw_if_index, |
| 478 | is_up ? "ready" : "down"); |
| 479 | vnet_hw_interface_set_flags (vnet_get_main (), vui->hw_if_index, |
| 480 | is_up ? VNET_HW_INTERFACE_FLAG_LINK_UP : |
| 481 | 0); |
| 482 | vui->is_up = is_up; |
| 483 | } |
| 484 | vhost_user_rx_thread_placement (); |
| 485 | vhost_user_tx_thread_placement (vui); |
| 486 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 487 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 488 | static clib_error_t * |
| 489 | vhost_user_callfd_read_ready (unix_file_t * uf) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 490 | { |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 491 | __attribute__ ((unused)) int n; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 492 | u8 buff[8]; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 493 | n = read (uf->file_descriptor, ((char *) &buff), 8); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 494 | return 0; |
| 495 | } |
| 496 | |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 497 | static clib_error_t * |
| 498 | vhost_user_kickfd_read_ready (unix_file_t * uf) |
| 499 | { |
| 500 | __attribute__ ((unused)) int n; |
| 501 | u8 buff[8]; |
| 502 | vhost_user_intf_t *vui = |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 503 | pool_elt_at_index (vhost_user_main.vhost_user_interfaces, |
| 504 | uf->private_data >> 8); |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 505 | u32 qid = uf->private_data & 0xff; |
| 506 | n = read (uf->file_descriptor, ((char *) &buff), 8); |
| 507 | DBG_SOCK ("if %d KICK queue %d", uf->private_data >> 8, qid); |
| 508 | |
| 509 | vlib_worker_thread_barrier_sync (vlib_get_main ()); |
| 510 | vui->vrings[qid].started = 1; |
| 511 | vhost_user_update_iface_state (vui); |
| 512 | vlib_worker_thread_barrier_release (vlib_get_main ()); |
| 513 | return 0; |
| 514 | } |
| 515 | |
| 516 | /** |
| 517 | * @brief Try once to lock the vring |
| 518 | * @return 0 on success, non-zero on failure. |
| 519 | */ |
| 520 | static inline int |
| 521 | vhost_user_vring_try_lock (vhost_user_intf_t * vui, u32 qid) |
| 522 | { |
| 523 | return __sync_lock_test_and_set (vui->vring_locks[qid], 1); |
| 524 | } |
| 525 | |
| 526 | /** |
| 527 | * @brief Spin until the vring is successfully locked |
| 528 | */ |
| 529 | static inline void |
| 530 | vhost_user_vring_lock (vhost_user_intf_t * vui, u32 qid) |
| 531 | { |
| 532 | while (vhost_user_vring_try_lock (vui, qid)) |
| 533 | ; |
| 534 | } |
| 535 | |
| 536 | /** |
| 537 | * @brief Unlock the vring lock |
| 538 | */ |
| 539 | static inline void |
| 540 | vhost_user_vring_unlock (vhost_user_intf_t * vui, u32 qid) |
| 541 | { |
| 542 | *vui->vring_locks[qid] = 0; |
| 543 | } |
| 544 | |
| 545 | static inline void |
| 546 | vhost_user_vring_init (vhost_user_intf_t * vui, u32 qid) |
| 547 | { |
| 548 | vhost_user_vring_t *vring = &vui->vrings[qid]; |
| 549 | memset (vring, 0, sizeof (*vring)); |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 550 | vring->kickfd_idx = ~0; |
| 551 | vring->callfd_idx = ~0; |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 552 | vring->errfd = -1; |
| 553 | |
| 554 | /* |
| 555 | * We have a bug with some qemu 2.5, and this may be a fix. |
| 556 | * Feel like interpretation holy text, but this is from vhost-user.txt. |
| 557 | * " |
| 558 | * One queue pair is enabled initially. More queues are enabled |
| 559 | * dynamically, by sending message VHOST_USER_SET_VRING_ENABLE. |
| 560 | * " |
| 561 | * Don't know who's right, but this is what DPDK does. |
| 562 | */ |
| 563 | if (qid == 0 || qid == 1) |
| 564 | vring->enabled = 1; |
| 565 | } |
| 566 | |
| 567 | static inline void |
| 568 | vhost_user_vring_close (vhost_user_intf_t * vui, u32 qid) |
| 569 | { |
| 570 | vhost_user_vring_t *vring = &vui->vrings[qid]; |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 571 | if (vring->kickfd_idx != ~0) |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 572 | { |
| 573 | unix_file_t *uf = pool_elt_at_index (unix_main.file_pool, |
| 574 | vring->kickfd_idx); |
| 575 | unix_file_del (&unix_main, uf); |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 576 | vring->kickfd_idx = ~0; |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 577 | } |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 578 | if (vring->callfd_idx != ~0) |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 579 | { |
| 580 | unix_file_t *uf = pool_elt_at_index (unix_main.file_pool, |
| 581 | vring->callfd_idx); |
| 582 | unix_file_del (&unix_main, uf); |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 583 | vring->callfd_idx = ~0; |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 584 | } |
| 585 | if (vring->errfd != -1) |
| 586 | close (vring->errfd); |
| 587 | vhost_user_vring_init (vui, qid); |
| 588 | } |
| 589 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 590 | static inline void |
| 591 | vhost_user_if_disconnect (vhost_user_intf_t * vui) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 592 | { |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 593 | vnet_main_t *vnm = vnet_get_main (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 594 | int q; |
| 595 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 596 | vnet_hw_interface_set_flags (vnm, vui->hw_if_index, 0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 597 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 598 | if (vui->unix_file_index != ~0) |
| 599 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 600 | unix_file_del (&unix_main, unix_main.file_pool + vui->unix_file_index); |
| 601 | vui->unix_file_index = ~0; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 602 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 603 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 604 | vui->is_up = 0; |
Steve Shin | 4448957 | 2016-09-22 12:08:55 -0700 | [diff] [blame] | 605 | |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 606 | for (q = 0; q < VHOST_VRING_MAX_N; q++) |
| 607 | vhost_user_vring_close (vui, q); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 608 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 609 | unmap_all_mem_regions (vui); |
| 610 | DBG_SOCK ("interface ifindex %d disconnected", vui->sw_if_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 611 | } |
| 612 | |
Yoann Desmouceaux | 4667c22 | 2016-02-24 22:51:00 +0100 | [diff] [blame] | 613 | #define VHOST_LOG_PAGE 0x1000 |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 614 | static_always_inline void |
| 615 | vhost_user_log_dirty_pages_2 (vhost_user_intf_t * vui, |
| 616 | u64 addr, u64 len, u8 is_host_address) |
Yoann Desmouceaux | 4667c22 | 2016-02-24 22:51:00 +0100 | [diff] [blame] | 617 | { |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 618 | if (PREDICT_TRUE (vui->log_base_addr == 0 |
| 619 | || !(vui->features & (1 << FEAT_VHOST_F_LOG_ALL)))) |
| 620 | { |
| 621 | return; |
| 622 | } |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 623 | if (is_host_address) |
| 624 | { |
| 625 | addr = (u64) map_user_mem (vui, (uword) addr); |
| 626 | } |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 627 | if (PREDICT_FALSE ((addr + len - 1) / VHOST_LOG_PAGE / 8 >= vui->log_size)) |
| 628 | { |
| 629 | DBG_SOCK ("vhost_user_log_dirty_pages(): out of range\n"); |
| 630 | return; |
| 631 | } |
Yoann Desmouceaux | 4667c22 | 2016-02-24 22:51:00 +0100 | [diff] [blame] | 632 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 633 | CLIB_MEMORY_BARRIER (); |
Yoann Desmouceaux | 4667c22 | 2016-02-24 22:51:00 +0100 | [diff] [blame] | 634 | u64 page = addr / VHOST_LOG_PAGE; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 635 | while (page * VHOST_LOG_PAGE < addr + len) |
| 636 | { |
| 637 | ((u8 *) vui->log_base_addr)[page / 8] |= 1 << page % 8; |
| 638 | page++; |
| 639 | } |
Yoann Desmouceaux | 4667c22 | 2016-02-24 22:51:00 +0100 | [diff] [blame] | 640 | } |
| 641 | |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 642 | static_always_inline void |
| 643 | vhost_user_log_dirty_pages (vhost_user_intf_t * vui, u64 addr, u64 len) |
| 644 | { |
| 645 | vhost_user_log_dirty_pages_2 (vui, addr, len, 0); |
| 646 | } |
| 647 | |
Yoann Desmouceaux | 4667c22 | 2016-02-24 22:51:00 +0100 | [diff] [blame] | 648 | #define vhost_user_log_dirty_ring(vui, vq, member) \ |
Yoann Desmouceaux | fe2da0e | 2016-03-08 14:54:28 +0100 | [diff] [blame] | 649 | if (PREDICT_FALSE(vq->log_used)) { \ |
Damjan Marion | 8d281b3 | 2016-08-24 14:32:39 +0200 | [diff] [blame] | 650 | vhost_user_log_dirty_pages(vui, vq->log_guest_addr + STRUCT_OFFSET_OF(vring_used_t, member), \ |
Yoann Desmouceaux | fe2da0e | 2016-03-08 14:54:28 +0100 | [diff] [blame] | 651 | sizeof(vq->used->member)); \ |
| 652 | } |
Yoann Desmouceaux | 4667c22 | 2016-02-24 22:51:00 +0100 | [diff] [blame] | 653 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 654 | static clib_error_t * |
| 655 | vhost_user_socket_read (unix_file_t * uf) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 656 | { |
| 657 | int n, i; |
| 658 | int fd, number_of_fds = 0; |
| 659 | int fds[VHOST_MEMORY_MAX_NREGIONS]; |
| 660 | vhost_user_msg_t msg; |
| 661 | struct msghdr mh; |
| 662 | struct iovec iov[1]; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 663 | vhost_user_main_t *vum = &vhost_user_main; |
| 664 | vhost_user_intf_t *vui; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 665 | struct cmsghdr *cmsg; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 666 | u8 q; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 667 | unix_file_t template = { 0 }; |
| 668 | vnet_main_t *vnm = vnet_get_main (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 669 | |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 670 | vui = pool_elt_at_index (vum->vhost_user_interfaces, uf->private_data); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 671 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 672 | char control[CMSG_SPACE (VHOST_MEMORY_MAX_NREGIONS * sizeof (int))]; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 673 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 674 | memset (&mh, 0, sizeof (mh)); |
| 675 | memset (control, 0, sizeof (control)); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 676 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 677 | for (i = 0; i < VHOST_MEMORY_MAX_NREGIONS; i++) |
Damjan Marion | a290d7c | 2016-08-16 12:37:24 +0200 | [diff] [blame] | 678 | fds[i] = -1; |
| 679 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 680 | /* set the payload */ |
| 681 | iov[0].iov_base = (void *) &msg; |
| 682 | iov[0].iov_len = VHOST_USER_MSG_HDR_SZ; |
| 683 | |
| 684 | mh.msg_iov = iov; |
| 685 | mh.msg_iovlen = 1; |
| 686 | mh.msg_control = control; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 687 | mh.msg_controllen = sizeof (control); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 688 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 689 | n = recvmsg (uf->file_descriptor, &mh, 0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 690 | |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 691 | /* Stop workers to avoid end of the world */ |
| 692 | vlib_worker_thread_barrier_sync (vlib_get_main ()); |
| 693 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 694 | if (n != VHOST_USER_MSG_HDR_SZ) |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 695 | { |
| 696 | if (n == -1) |
| 697 | { |
| 698 | DBG_SOCK ("recvmsg returned error %d %s", errno, strerror (errno)); |
| 699 | } |
| 700 | else |
| 701 | { |
| 702 | DBG_SOCK ("n (%d) != VHOST_USER_MSG_HDR_SZ (%d)", |
| 703 | n, VHOST_USER_MSG_HDR_SZ); |
| 704 | } |
| 705 | goto close_socket; |
| 706 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 707 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 708 | if (mh.msg_flags & MSG_CTRUNC) |
| 709 | { |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 710 | DBG_SOCK ("MSG_CTRUNC is set"); |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 711 | goto close_socket; |
| 712 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 713 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 714 | cmsg = CMSG_FIRSTHDR (&mh); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 715 | |
| 716 | if (cmsg && (cmsg->cmsg_len > 0) && (cmsg->cmsg_level == SOL_SOCKET) && |
| 717 | (cmsg->cmsg_type == SCM_RIGHTS) && |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 718 | (cmsg->cmsg_len - CMSG_LEN (0) <= |
| 719 | VHOST_MEMORY_MAX_NREGIONS * sizeof (int))) |
| 720 | { |
| 721 | number_of_fds = (cmsg->cmsg_len - CMSG_LEN (0)) / sizeof (int); |
| 722 | clib_memcpy (fds, CMSG_DATA (cmsg), number_of_fds * sizeof (int)); |
| 723 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 724 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 725 | /* version 1, no reply bit set */ |
| 726 | if ((msg.flags & 7) != 1) |
| 727 | { |
| 728 | DBG_SOCK ("malformed message received. closing socket"); |
| 729 | goto close_socket; |
| 730 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 731 | |
| 732 | { |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 733 | int rv; |
| 734 | rv = |
| 735 | read (uf->file_descriptor, ((char *) &msg) + VHOST_USER_MSG_HDR_SZ, |
| 736 | msg.size); |
| 737 | if (rv < 0) |
| 738 | { |
| 739 | DBG_SOCK ("read failed %s", strerror (errno)); |
| 740 | goto close_socket; |
| 741 | } |
| 742 | else if (rv != msg.size) |
| 743 | { |
| 744 | DBG_SOCK ("message too short (read %dB should be %dB)", rv, msg.size); |
| 745 | goto close_socket; |
| 746 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 747 | } |
| 748 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 749 | switch (msg.request) |
| 750 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 751 | case VHOST_USER_GET_FEATURES: |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 752 | msg.flags |= 4; |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 753 | msg.u64 = (1ULL << FEAT_VIRTIO_NET_F_MRG_RXBUF) | |
| 754 | (1ULL << FEAT_VIRTIO_NET_F_CTRL_VQ) | |
| 755 | (1ULL << FEAT_VIRTIO_F_ANY_LAYOUT) | |
| 756 | (1ULL << FEAT_VIRTIO_F_INDIRECT_DESC) | |
| 757 | (1ULL << FEAT_VHOST_F_LOG_ALL) | |
| 758 | (1ULL << FEAT_VIRTIO_NET_F_GUEST_ANNOUNCE) | |
| 759 | (1ULL << FEAT_VIRTIO_NET_F_MQ) | |
| 760 | (1ULL << FEAT_VHOST_USER_F_PROTOCOL_FEATURES) | |
| 761 | (1ULL << FEAT_VIRTIO_F_VERSION_1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 762 | msg.u64 &= vui->feature_mask; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 763 | msg.size = sizeof (msg.u64); |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 764 | DBG_SOCK ("if %d msg VHOST_USER_GET_FEATURES - reply 0x%016llx", |
| 765 | vui->hw_if_index, msg.u64); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 766 | break; |
| 767 | |
| 768 | case VHOST_USER_SET_FEATURES: |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 769 | DBG_SOCK ("if %d msg VHOST_USER_SET_FEATURES features 0x%016llx", |
| 770 | vui->hw_if_index, msg.u64); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 771 | |
| 772 | vui->features = msg.u64; |
Yoann Desmouceaux | 4667c22 | 2016-02-24 22:51:00 +0100 | [diff] [blame] | 773 | |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 774 | if (vui->features & |
| 775 | ((1 << FEAT_VIRTIO_NET_F_MRG_RXBUF) | |
| 776 | (1ULL << FEAT_VIRTIO_F_VERSION_1))) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 777 | vui->virtio_net_hdr_sz = 12; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 778 | else |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 779 | vui->virtio_net_hdr_sz = 10; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 780 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 781 | vui->is_any_layout = |
| 782 | (vui->features & (1 << FEAT_VIRTIO_F_ANY_LAYOUT)) ? 1 : 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 783 | |
| 784 | ASSERT (vui->virtio_net_hdr_sz < VLIB_BUFFER_PRE_DATA_SIZE); |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 785 | vnet_hw_interface_set_flags (vnm, vui->hw_if_index, 0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 786 | vui->is_up = 0; |
| 787 | |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 788 | /*for (q = 0; q < VHOST_VRING_MAX_N; q++) |
| 789 | vhost_user_vring_close(&vui->vrings[q]); */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 790 | |
| 791 | break; |
| 792 | |
| 793 | case VHOST_USER_SET_MEM_TABLE: |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 794 | DBG_SOCK ("if %d msg VHOST_USER_SET_MEM_TABLE nregions %d", |
| 795 | vui->hw_if_index, msg.memory.nregions); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 796 | |
| 797 | if ((msg.memory.nregions < 1) || |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 798 | (msg.memory.nregions > VHOST_MEMORY_MAX_NREGIONS)) |
| 799 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 800 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 801 | DBG_SOCK ("number of mem regions must be between 1 and %i", |
| 802 | VHOST_MEMORY_MAX_NREGIONS); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 803 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 804 | goto close_socket; |
| 805 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 806 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 807 | if (msg.memory.nregions != number_of_fds) |
| 808 | { |
| 809 | DBG_SOCK ("each memory region must have FD"); |
| 810 | goto close_socket; |
| 811 | } |
| 812 | unmap_all_mem_regions (vui); |
| 813 | for (i = 0; i < msg.memory.nregions; i++) |
| 814 | { |
| 815 | clib_memcpy (&(vui->regions[i]), &msg.memory.regions[i], |
| 816 | sizeof (vhost_user_memory_region_t)); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 817 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 818 | long page_sz = get_huge_page_size (fds[i]); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 819 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 820 | /* align size to 2M page */ |
| 821 | ssize_t map_sz = (vui->regions[i].memory_size + |
| 822 | vui->regions[i].mmap_offset + |
| 823 | page_sz) & ~(page_sz - 1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 824 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 825 | vui->region_mmap_addr[i] = mmap (0, map_sz, PROT_READ | PROT_WRITE, |
| 826 | MAP_SHARED, fds[i], 0); |
Damjan Marion | 3762370 | 2016-09-20 11:25:27 +0200 | [diff] [blame] | 827 | vui->region_guest_addr_lo[i] = vui->regions[i].guest_phys_addr; |
| 828 | vui->region_guest_addr_hi[i] = vui->regions[i].guest_phys_addr + |
| 829 | vui->regions[i].memory_size; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 830 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 831 | DBG_SOCK |
| 832 | ("map memory region %d addr 0 len 0x%lx fd %d mapped 0x%lx " |
| 833 | "page_sz 0x%x", i, map_sz, fds[i], vui->region_mmap_addr[i], |
| 834 | page_sz); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 835 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 836 | if (vui->region_mmap_addr[i] == MAP_FAILED) |
| 837 | { |
| 838 | clib_warning ("failed to map memory. errno is %d", errno); |
| 839 | goto close_socket; |
| 840 | } |
| 841 | vui->region_mmap_addr[i] += vui->regions[i].mmap_offset; |
| 842 | vui->region_mmap_fd[i] = fds[i]; |
| 843 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 844 | vui->nregions = msg.memory.nregions; |
| 845 | break; |
| 846 | |
| 847 | case VHOST_USER_SET_VRING_NUM: |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 848 | DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_NUM idx %d num %d", |
| 849 | vui->hw_if_index, msg.state.index, msg.state.num); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 850 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 851 | if ((msg.state.num > 32768) || /* maximum ring size is 32768 */ |
| 852 | (msg.state.num == 0) || /* it cannot be zero */ |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 853 | ((msg.state.num - 1) & msg.state.num)) /* must be power of 2 */ |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 854 | goto close_socket; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 855 | vui->vrings[msg.state.index].qsz = msg.state.num; |
| 856 | break; |
| 857 | |
| 858 | case VHOST_USER_SET_VRING_ADDR: |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 859 | DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_ADDR idx %d", |
| 860 | vui->hw_if_index, msg.state.index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 861 | |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 862 | if (msg.state.index >= VHOST_VRING_MAX_N) |
| 863 | { |
| 864 | DBG_SOCK ("invalid vring index VHOST_USER_SET_VRING_ADDR:" |
| 865 | " %d >= %d", msg.state.index, VHOST_VRING_MAX_N); |
| 866 | goto close_socket; |
| 867 | } |
| 868 | |
| 869 | if (msg.size < sizeof (msg.addr)) |
| 870 | { |
| 871 | DBG_SOCK ("vhost message is too short (%d < %d)", |
| 872 | msg.size, sizeof (msg.addr)); |
| 873 | goto close_socket; |
| 874 | } |
| 875 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 876 | vui->vrings[msg.state.index].desc = (vring_desc_t *) |
| 877 | map_user_mem (vui, msg.addr.desc_user_addr); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 878 | vui->vrings[msg.state.index].used = (vring_used_t *) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 879 | map_user_mem (vui, msg.addr.used_user_addr); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 880 | vui->vrings[msg.state.index].avail = (vring_avail_t *) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 881 | map_user_mem (vui, msg.addr.avail_user_addr); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 882 | |
| 883 | if ((vui->vrings[msg.state.index].desc == NULL) || |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 884 | (vui->vrings[msg.state.index].used == NULL) || |
| 885 | (vui->vrings[msg.state.index].avail == NULL)) |
| 886 | { |
| 887 | DBG_SOCK ("failed to map user memory for hw_if_index %d", |
| 888 | vui->hw_if_index); |
| 889 | goto close_socket; |
| 890 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 891 | |
Yoann Desmouceaux | 4667c22 | 2016-02-24 22:51:00 +0100 | [diff] [blame] | 892 | vui->vrings[msg.state.index].log_guest_addr = msg.addr.log_guest_addr; |
Yoann Desmouceaux | fe2da0e | 2016-03-08 14:54:28 +0100 | [diff] [blame] | 893 | vui->vrings[msg.state.index].log_used = |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 894 | (msg.addr.flags & (1 << VHOST_VRING_F_LOG)) ? 1 : 0; |
Yoann Desmouceaux | 4667c22 | 2016-02-24 22:51:00 +0100 | [diff] [blame] | 895 | |
| 896 | /* Spec says: If VHOST_USER_F_PROTOCOL_FEATURES has not been negotiated, |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 897 | the ring is initialized in an enabled state. */ |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 898 | if (!(vui->features & (1 << FEAT_VHOST_USER_F_PROTOCOL_FEATURES))) |
| 899 | { |
| 900 | vui->vrings[msg.state.index].enabled = 1; |
| 901 | } |
Yoann Desmouceaux | 4667c22 | 2016-02-24 22:51:00 +0100 | [diff] [blame] | 902 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 903 | vui->vrings[msg.state.index].last_used_idx = |
Damjan Marion | 10eb1ea | 2016-10-13 10:02:19 +0200 | [diff] [blame] | 904 | vui->vrings[msg.state.index].last_avail_idx = |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 905 | vui->vrings[msg.state.index].used->idx; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 906 | |
| 907 | /* tell driver that we don't want interrupts */ |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 908 | vui->vrings[msg.state.index].used->flags = VRING_USED_F_NO_NOTIFY; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 909 | break; |
| 910 | |
| 911 | case VHOST_USER_SET_OWNER: |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 912 | DBG_SOCK ("if %d msg VHOST_USER_SET_OWNER", vui->hw_if_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 913 | break; |
| 914 | |
| 915 | case VHOST_USER_RESET_OWNER: |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 916 | DBG_SOCK ("if %d msg VHOST_USER_RESET_OWNER", vui->hw_if_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 917 | break; |
| 918 | |
| 919 | case VHOST_USER_SET_VRING_CALL: |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 920 | DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_CALL u64 %d", |
| 921 | vui->hw_if_index, msg.u64); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 922 | |
| 923 | q = (u8) (msg.u64 & 0xFF); |
| 924 | |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 925 | /* if there is old fd, delete and close it */ |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 926 | if (vui->vrings[q].callfd_idx != ~0) |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 927 | { |
| 928 | unix_file_t *uf = pool_elt_at_index (unix_main.file_pool, |
| 929 | vui->vrings[q].callfd_idx); |
| 930 | unix_file_del (&unix_main, uf); |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 931 | vui->vrings[q].callfd_idx = ~0; |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 932 | } |
| 933 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 934 | if (!(msg.u64 & 0x100)) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 935 | { |
| 936 | if (number_of_fds != 1) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 937 | { |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 938 | DBG_SOCK ("More than one fd received !"); |
| 939 | goto close_socket; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 940 | } |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 941 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 942 | template.read_function = vhost_user_callfd_read_ready; |
| 943 | template.file_descriptor = fds[0]; |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 944 | template.private_data = |
| 945 | ((vui - vhost_user_main.vhost_user_interfaces) << 8) + q; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 946 | vui->vrings[q].callfd_idx = unix_file_add (&unix_main, &template); |
| 947 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 948 | else |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 949 | vui->vrings[q].callfd_idx = ~0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 950 | break; |
| 951 | |
| 952 | case VHOST_USER_SET_VRING_KICK: |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 953 | DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_KICK u64 %d", |
| 954 | vui->hw_if_index, msg.u64); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 955 | |
| 956 | q = (u8) (msg.u64 & 0xFF); |
| 957 | |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 958 | if (vui->vrings[q].kickfd_idx != ~0) |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 959 | { |
| 960 | unix_file_t *uf = pool_elt_at_index (unix_main.file_pool, |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 961 | vui->vrings[q].kickfd_idx); |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 962 | unix_file_del (&unix_main, uf); |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 963 | vui->vrings[q].kickfd_idx = ~0; |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 964 | } |
| 965 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 966 | if (!(msg.u64 & 0x100)) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 967 | { |
| 968 | if (number_of_fds != 1) |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 969 | { |
| 970 | DBG_SOCK ("More than one fd received !"); |
| 971 | goto close_socket; |
| 972 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 973 | |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 974 | template.read_function = vhost_user_kickfd_read_ready; |
| 975 | template.file_descriptor = fds[0]; |
| 976 | template.private_data = |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 977 | (((uword) (vui - vhost_user_main.vhost_user_interfaces)) << 8) + |
| 978 | q; |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 979 | vui->vrings[q].kickfd_idx = unix_file_add (&unix_main, &template); |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 980 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 981 | else |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 982 | { |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 983 | //When no kickfd is set, the queue is initialized as started |
| 984 | vui->vrings[q].kickfd_idx = ~0; |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 985 | vui->vrings[q].started = 1; |
| 986 | } |
| 987 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 988 | break; |
| 989 | |
| 990 | case VHOST_USER_SET_VRING_ERR: |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 991 | DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_ERR u64 %d", |
| 992 | vui->hw_if_index, msg.u64); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 993 | |
| 994 | q = (u8) (msg.u64 & 0xFF); |
| 995 | |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 996 | if (vui->vrings[q].errfd != -1) |
| 997 | close (vui->vrings[q].errfd); |
| 998 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 999 | if (!(msg.u64 & 0x100)) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1000 | { |
| 1001 | if (number_of_fds != 1) |
| 1002 | goto close_socket; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1003 | |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1004 | vui->vrings[q].errfd = fds[0]; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1005 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1006 | else |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1007 | vui->vrings[q].errfd = -1; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1008 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1009 | break; |
| 1010 | |
| 1011 | case VHOST_USER_SET_VRING_BASE: |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1012 | DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_BASE idx %d num %d", |
| 1013 | vui->hw_if_index, msg.state.index, msg.state.num); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1014 | |
| 1015 | vui->vrings[msg.state.index].last_avail_idx = msg.state.num; |
| 1016 | break; |
| 1017 | |
| 1018 | case VHOST_USER_GET_VRING_BASE: |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1019 | DBG_SOCK ("if %d msg VHOST_USER_GET_VRING_BASE idx %d num %d", |
| 1020 | vui->hw_if_index, msg.state.index, msg.state.num); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1021 | |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1022 | if (msg.state.index >= VHOST_VRING_MAX_N) |
| 1023 | { |
| 1024 | DBG_SOCK ("invalid vring index VHOST_USER_GET_VRING_BASE:" |
| 1025 | " %d >= %d", msg.state.index, VHOST_VRING_MAX_N); |
| 1026 | goto close_socket; |
| 1027 | } |
| 1028 | |
Yoann Desmouceaux | 4667c22 | 2016-02-24 22:51:00 +0100 | [diff] [blame] | 1029 | /* Spec says: Client must [...] stop ring upon receiving VHOST_USER_GET_VRING_BASE. */ |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1030 | vhost_user_vring_close (vui, msg.state.index); |
Yoann Desmouceaux | 4667c22 | 2016-02-24 22:51:00 +0100 | [diff] [blame] | 1031 | |
| 1032 | msg.state.num = vui->vrings[msg.state.index].last_avail_idx; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1033 | msg.flags |= 4; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1034 | msg.size = sizeof (msg.state); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1035 | break; |
| 1036 | |
| 1037 | case VHOST_USER_NONE: |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1038 | DBG_SOCK ("if %d msg VHOST_USER_NONE", vui->hw_if_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1039 | |
| 1040 | break; |
| 1041 | |
| 1042 | case VHOST_USER_SET_LOG_BASE: |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1043 | { |
| 1044 | DBG_SOCK ("if %d msg VHOST_USER_SET_LOG_BASE", vui->hw_if_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1045 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1046 | if (msg.size != sizeof (msg.log)) |
| 1047 | { |
| 1048 | DBG_SOCK |
| 1049 | ("invalid msg size for VHOST_USER_SET_LOG_BASE: %d instead of %d", |
| 1050 | msg.size, sizeof (msg.log)); |
| 1051 | goto close_socket; |
| 1052 | } |
| 1053 | |
| 1054 | if (! |
| 1055 | (vui->protocol_features & (1 << VHOST_USER_PROTOCOL_F_LOG_SHMFD))) |
| 1056 | { |
| 1057 | DBG_SOCK |
| 1058 | ("VHOST_USER_PROTOCOL_F_LOG_SHMFD not set but VHOST_USER_SET_LOG_BASE received"); |
| 1059 | goto close_socket; |
| 1060 | } |
| 1061 | |
| 1062 | fd = fds[0]; |
| 1063 | /* align size to 2M page */ |
| 1064 | long page_sz = get_huge_page_size (fd); |
| 1065 | ssize_t map_sz = |
| 1066 | (msg.log.size + msg.log.offset + page_sz) & ~(page_sz - 1); |
| 1067 | |
| 1068 | vui->log_base_addr = mmap (0, map_sz, PROT_READ | PROT_WRITE, |
| 1069 | MAP_SHARED, fd, 0); |
| 1070 | |
| 1071 | DBG_SOCK |
| 1072 | ("map log region addr 0 len 0x%lx off 0x%lx fd %d mapped 0x%lx", |
| 1073 | map_sz, msg.log.offset, fd, vui->log_base_addr); |
| 1074 | |
| 1075 | if (vui->log_base_addr == MAP_FAILED) |
| 1076 | { |
| 1077 | clib_warning ("failed to map memory. errno is %d", errno); |
| 1078 | goto close_socket; |
| 1079 | } |
| 1080 | |
| 1081 | vui->log_base_addr += msg.log.offset; |
| 1082 | vui->log_size = msg.log.size; |
| 1083 | |
| 1084 | msg.flags |= 4; |
| 1085 | msg.size = sizeof (msg.u64); |
| 1086 | |
| 1087 | break; |
Yoann Desmouceaux | 4667c22 | 2016-02-24 22:51:00 +0100 | [diff] [blame] | 1088 | } |
| 1089 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1090 | case VHOST_USER_SET_LOG_FD: |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1091 | DBG_SOCK ("if %d msg VHOST_USER_SET_LOG_FD", vui->hw_if_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1092 | |
| 1093 | break; |
| 1094 | |
Yoann Desmouceaux | 4667c22 | 2016-02-24 22:51:00 +0100 | [diff] [blame] | 1095 | case VHOST_USER_GET_PROTOCOL_FEATURES: |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1096 | DBG_SOCK ("if %d msg VHOST_USER_GET_PROTOCOL_FEATURES", |
| 1097 | vui->hw_if_index); |
Yoann Desmouceaux | 4667c22 | 2016-02-24 22:51:00 +0100 | [diff] [blame] | 1098 | |
| 1099 | msg.flags |= 4; |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1100 | msg.u64 = (1 << VHOST_USER_PROTOCOL_F_LOG_SHMFD) | |
| 1101 | (1 << VHOST_USER_PROTOCOL_F_MQ); |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1102 | msg.size = sizeof (msg.u64); |
Yoann Desmouceaux | 4667c22 | 2016-02-24 22:51:00 +0100 | [diff] [blame] | 1103 | break; |
| 1104 | |
| 1105 | case VHOST_USER_SET_PROTOCOL_FEATURES: |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1106 | DBG_SOCK ("if %d msg VHOST_USER_SET_PROTOCOL_FEATURES features 0x%lx", |
| 1107 | vui->hw_if_index, msg.u64); |
Yoann Desmouceaux | 4667c22 | 2016-02-24 22:51:00 +0100 | [diff] [blame] | 1108 | |
| 1109 | vui->protocol_features = msg.u64; |
| 1110 | |
| 1111 | break; |
| 1112 | |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1113 | case VHOST_USER_GET_QUEUE_NUM: |
| 1114 | DBG_SOCK ("if %d msg VHOST_USER_GET_QUEUE_NUM", vui->hw_if_index); |
| 1115 | msg.flags |= 4; |
| 1116 | msg.u64 = VHOST_VRING_MAX_N; |
| 1117 | msg.size = sizeof (msg.u64); |
| 1118 | break; |
| 1119 | |
Yoann Desmouceaux | 4667c22 | 2016-02-24 22:51:00 +0100 | [diff] [blame] | 1120 | case VHOST_USER_SET_VRING_ENABLE: |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1121 | DBG_SOCK ("if %d VHOST_USER_SET_VRING_ENABLE: %s queue %d", |
| 1122 | vui->hw_if_index, msg.state.num ? "enable" : "disable", |
| 1123 | msg.state.index); |
| 1124 | if (msg.state.index >= VHOST_VRING_MAX_N) |
| 1125 | { |
| 1126 | DBG_SOCK ("invalid vring index VHOST_USER_SET_VRING_ENABLE:" |
| 1127 | " %d >= %d", msg.state.index, VHOST_VRING_MAX_N); |
| 1128 | goto close_socket; |
| 1129 | } |
| 1130 | |
Yoann Desmouceaux | 4667c22 | 2016-02-24 22:51:00 +0100 | [diff] [blame] | 1131 | vui->vrings[msg.state.index].enabled = msg.state.num; |
| 1132 | break; |
| 1133 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1134 | default: |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1135 | DBG_SOCK ("unknown vhost-user message %d received. closing socket", |
| 1136 | msg.request); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1137 | goto close_socket; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1138 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1139 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1140 | /* if we need to reply */ |
| 1141 | if (msg.flags & 4) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1142 | { |
| 1143 | n = |
| 1144 | send (uf->file_descriptor, &msg, VHOST_USER_MSG_HDR_SZ + msg.size, 0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1145 | if (n != (msg.size + VHOST_USER_MSG_HDR_SZ)) |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1146 | { |
| 1147 | DBG_SOCK ("could not send message response"); |
| 1148 | goto close_socket; |
| 1149 | } |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1150 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1151 | |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1152 | vhost_user_update_iface_state (vui); |
| 1153 | vlib_worker_thread_barrier_release (vlib_get_main ()); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1154 | return 0; |
| 1155 | |
| 1156 | close_socket: |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1157 | vhost_user_if_disconnect (vui); |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1158 | vhost_user_update_iface_state (vui); |
| 1159 | vlib_worker_thread_barrier_release (vlib_get_main ()); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1160 | return 0; |
| 1161 | } |
| 1162 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1163 | static clib_error_t * |
| 1164 | vhost_user_socket_error (unix_file_t * uf) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1165 | { |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1166 | vlib_main_t *vm = vlib_get_main (); |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1167 | vhost_user_main_t *vum = &vhost_user_main; |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 1168 | vhost_user_intf_t *vui = |
| 1169 | pool_elt_at_index (vum->vhost_user_interfaces, uf->private_data); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1170 | |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1171 | DBG_SOCK ("socket error on if %d", vui->sw_if_index); |
| 1172 | vlib_worker_thread_barrier_sync (vm); |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1173 | vhost_user_if_disconnect (vui); |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1174 | vhost_user_rx_thread_placement (); |
| 1175 | vlib_worker_thread_barrier_release (vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1176 | return 0; |
| 1177 | } |
| 1178 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1179 | static clib_error_t * |
| 1180 | vhost_user_socksvr_accept_ready (unix_file_t * uf) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1181 | { |
| 1182 | int client_fd, client_len; |
| 1183 | struct sockaddr_un client; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1184 | unix_file_t template = { 0 }; |
| 1185 | vhost_user_main_t *vum = &vhost_user_main; |
| 1186 | vhost_user_intf_t *vui; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1187 | |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 1188 | vui = pool_elt_at_index (vum->vhost_user_interfaces, uf->private_data); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1189 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1190 | client_len = sizeof (client); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1191 | client_fd = accept (uf->file_descriptor, |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1192 | (struct sockaddr *) &client, |
| 1193 | (socklen_t *) & client_len); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1194 | |
| 1195 | if (client_fd < 0) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1196 | return clib_error_return_unix (0, "accept"); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1197 | |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1198 | DBG_SOCK ("New client socket for vhost interface %d", vui->sw_if_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1199 | template.read_function = vhost_user_socket_read; |
| 1200 | template.error_function = vhost_user_socket_error; |
| 1201 | template.file_descriptor = client_fd; |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 1202 | template.private_data = vui - vhost_user_main.vhost_user_interfaces; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1203 | vui->unix_file_index = unix_file_add (&unix_main, &template); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1204 | return 0; |
| 1205 | } |
| 1206 | |
| 1207 | static clib_error_t * |
| 1208 | vhost_user_init (vlib_main_t * vm) |
| 1209 | { |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1210 | clib_error_t *error; |
| 1211 | vhost_user_main_t *vum = &vhost_user_main; |
| 1212 | vlib_thread_main_t *tm = vlib_get_thread_main (); |
Damjan Marion | 0dafaa7 | 2016-09-20 23:21:02 +0200 | [diff] [blame] | 1213 | vlib_thread_registration_t *tr; |
| 1214 | uword *p; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1215 | |
| 1216 | error = vlib_call_init_function (vm, ip4_init); |
| 1217 | if (error) |
| 1218 | return error; |
| 1219 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1220 | vum->coalesce_frames = 32; |
| 1221 | vum->coalesce_time = 1e-3; |
| 1222 | |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1223 | vec_validate (vum->cpus, tm->n_vlib_mains - 1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1224 | |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1225 | vhost_cpu_t *cpu; |
| 1226 | vec_foreach (cpu, vum->cpus) |
| 1227 | { |
| 1228 | /* This is actually not necessary as validate already zeroes it |
| 1229 | * Just keeping the loop here for later because I am lazy. */ |
| 1230 | cpu->rx_buffers_len = 0; |
| 1231 | } |
| 1232 | |
Damjan Marion | 0dafaa7 | 2016-09-20 23:21:02 +0200 | [diff] [blame] | 1233 | /* find out which cpus will be used for input */ |
| 1234 | vum->input_cpu_first_index = 0; |
| 1235 | vum->input_cpu_count = 1; |
| 1236 | p = hash_get_mem (tm->thread_registrations_by_name, "workers"); |
| 1237 | tr = p ? (vlib_thread_registration_t *) p[0] : 0; |
| 1238 | |
| 1239 | if (tr && tr->count > 0) |
| 1240 | { |
| 1241 | vum->input_cpu_first_index = tr->first_index; |
| 1242 | vum->input_cpu_count = tr->count; |
| 1243 | } |
| 1244 | |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 1245 | vum->random = random_default_seed (); |
| 1246 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1247 | return 0; |
| 1248 | } |
| 1249 | |
| 1250 | VLIB_INIT_FUNCTION (vhost_user_init); |
| 1251 | |
| 1252 | static clib_error_t * |
| 1253 | vhost_user_exit (vlib_main_t * vm) |
| 1254 | { |
| 1255 | /* TODO cleanup */ |
| 1256 | return 0; |
| 1257 | } |
| 1258 | |
| 1259 | VLIB_MAIN_LOOP_EXIT_FUNCTION (vhost_user_exit); |
| 1260 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1261 | static u8 * |
Pierre Pfister | 116ea4b | 2016-11-08 15:49:28 +0000 | [diff] [blame] | 1262 | format_vhost_trace (u8 * s, va_list * va) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1263 | { |
| 1264 | CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *); |
| 1265 | CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *); |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1266 | CLIB_UNUSED (vnet_main_t * vnm) = vnet_get_main (); |
| 1267 | vhost_user_main_t *vum = &vhost_user_main; |
Pierre Pfister | 116ea4b | 2016-11-08 15:49:28 +0000 | [diff] [blame] | 1268 | vhost_trace_t *t = va_arg (*va, vhost_trace_t *); |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 1269 | vhost_user_intf_t *vui = pool_elt_at_index (vum->vhost_user_interfaces, |
| 1270 | t->device_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1271 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1272 | vnet_sw_interface_t *sw = vnet_get_sw_interface (vnm, vui->sw_if_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1273 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1274 | uword indent = format_get_indent (s); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1275 | |
Pierre Pfister | 116ea4b | 2016-11-08 15:49:28 +0000 | [diff] [blame] | 1276 | s = format (s, "%U %U queue %d\n", format_white_space, indent, |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1277 | format_vnet_sw_interface_name, vnm, sw, t->qid); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1278 | |
Pierre Pfister | 116ea4b | 2016-11-08 15:49:28 +0000 | [diff] [blame] | 1279 | s = format (s, "%U virtio flags:\n", format_white_space, indent); |
| 1280 | #define _(n,i,st) \ |
| 1281 | if (t->virtio_ring_flags & (1 << VIRTIO_TRACE_F_##n)) \ |
| 1282 | s = format (s, "%U %s %s\n", format_white_space, indent, #n, st); |
| 1283 | foreach_virtio_trace_flags |
| 1284 | #undef _ |
| 1285 | s = format (s, "%U virtio_net_hdr first_desc_len %u\n", |
| 1286 | format_white_space, indent, t->first_desc_len); |
| 1287 | |
| 1288 | s = format (s, "%U flags 0x%02x gso_type %u\n", |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1289 | format_white_space, indent, |
Pierre Pfister | 116ea4b | 2016-11-08 15:49:28 +0000 | [diff] [blame] | 1290 | t->hdr.hdr.flags, t->hdr.hdr.gso_type); |
| 1291 | |
| 1292 | if (vui->virtio_net_hdr_sz == 12) |
| 1293 | s = format (s, "%U num_buff %u", |
| 1294 | format_white_space, indent, t->hdr.num_buffers); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1295 | |
| 1296 | return s; |
| 1297 | } |
| 1298 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1299 | void |
Pierre Pfister | 116ea4b | 2016-11-08 15:49:28 +0000 | [diff] [blame] | 1300 | vhost_user_rx_trace (vhost_trace_t * t, |
| 1301 | vhost_user_intf_t * vui, u16 qid, |
| 1302 | vlib_buffer_t * b, vhost_user_vring_t * txvq) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1303 | { |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1304 | vhost_user_main_t *vum = &vhost_user_main; |
Pierre Pfister | 116ea4b | 2016-11-08 15:49:28 +0000 | [diff] [blame] | 1305 | u32 qsz_mask = txvq->qsz - 1; |
| 1306 | u32 last_avail_idx = txvq->last_avail_idx; |
| 1307 | u32 desc_current = txvq->avail->ring[last_avail_idx & qsz_mask]; |
| 1308 | vring_desc_t *hdr_desc = 0; |
| 1309 | virtio_net_hdr_mrg_rxbuf_t *hdr; |
| 1310 | u32 hint = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1311 | |
Pierre Pfister | 116ea4b | 2016-11-08 15:49:28 +0000 | [diff] [blame] | 1312 | memset (t, 0, sizeof (*t)); |
| 1313 | t->device_index = vui - vum->vhost_user_interfaces; |
| 1314 | t->qid = qid; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1315 | |
Pierre Pfister | 116ea4b | 2016-11-08 15:49:28 +0000 | [diff] [blame] | 1316 | hdr_desc = &txvq->desc[desc_current]; |
| 1317 | if (txvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1318 | { |
Pierre Pfister | 116ea4b | 2016-11-08 15:49:28 +0000 | [diff] [blame] | 1319 | t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_INDIRECT; |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1320 | /* Header is the first here */ |
Pierre Pfister | 116ea4b | 2016-11-08 15:49:28 +0000 | [diff] [blame] | 1321 | hdr_desc = map_guest_mem (vui, txvq->desc[desc_current].addr, &hint); |
| 1322 | } |
| 1323 | if (txvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT) |
| 1324 | { |
| 1325 | t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SIMPLE_CHAINED; |
| 1326 | } |
| 1327 | if (!(txvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT) && |
| 1328 | !(txvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT)) |
| 1329 | { |
| 1330 | t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SINGLE_DESC; |
| 1331 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1332 | |
Pierre Pfister | 116ea4b | 2016-11-08 15:49:28 +0000 | [diff] [blame] | 1333 | t->first_desc_len = hdr_desc ? hdr_desc->len : 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1334 | |
Pierre Pfister | 116ea4b | 2016-11-08 15:49:28 +0000 | [diff] [blame] | 1335 | if (!hdr_desc || !(hdr = map_guest_mem (vui, hdr_desc->addr, &hint))) |
| 1336 | { |
| 1337 | t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_MAP_ERROR; |
| 1338 | } |
| 1339 | else |
| 1340 | { |
| 1341 | u32 len = vui->virtio_net_hdr_sz; |
| 1342 | memcpy (&t->hdr, hdr, len > hdr_desc->len ? hdr_desc->len : len); |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1343 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1344 | } |
| 1345 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1346 | static inline void |
| 1347 | vhost_user_send_call (vlib_main_t * vm, vhost_user_vring_t * vq) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1348 | { |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1349 | vhost_user_main_t *vum = &vhost_user_main; |
| 1350 | u64 x = 1; |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 1351 | int fd = UNIX_GET_FD (vq->callfd_idx); |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1352 | int rv __attribute__ ((unused)); |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 1353 | /* TODO: pay attention to rv */ |
| 1354 | rv = write (fd, &x, sizeof (x)); |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1355 | vq->n_since_last_int = 0; |
| 1356 | vq->int_deadline = vlib_time_now (vm) + vum->coalesce_time; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1357 | } |
| 1358 | |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1359 | static_always_inline u32 |
| 1360 | vhost_user_input_copy (vhost_user_intf_t * vui, vhost_copy_t * cpy, |
| 1361 | u16 copy_len, u32 * map_hint) |
| 1362 | { |
| 1363 | void *src0, *src1, *src2, *src3; |
| 1364 | if (PREDICT_TRUE (copy_len >= 4)) |
| 1365 | { |
| 1366 | if (PREDICT_FALSE (!(src2 = map_guest_mem (vui, cpy[0].src, map_hint)))) |
| 1367 | return 1; |
| 1368 | if (PREDICT_FALSE (!(src3 = map_guest_mem (vui, cpy[1].src, map_hint)))) |
| 1369 | return 1; |
| 1370 | |
| 1371 | while (PREDICT_TRUE (copy_len >= 4)) |
| 1372 | { |
| 1373 | src0 = src2; |
| 1374 | src1 = src3; |
| 1375 | |
| 1376 | if (PREDICT_FALSE |
| 1377 | (!(src2 = map_guest_mem (vui, cpy[2].src, map_hint)))) |
| 1378 | return 1; |
| 1379 | if (PREDICT_FALSE |
| 1380 | (!(src3 = map_guest_mem (vui, cpy[3].src, map_hint)))) |
| 1381 | return 1; |
| 1382 | |
| 1383 | CLIB_PREFETCH (src2, 64, LOAD); |
| 1384 | CLIB_PREFETCH (src3, 64, LOAD); |
| 1385 | |
| 1386 | clib_memcpy ((void *) cpy[0].dst, src0, cpy[0].len); |
| 1387 | clib_memcpy ((void *) cpy[1].dst, src1, cpy[1].len); |
| 1388 | copy_len -= 2; |
| 1389 | cpy += 2; |
| 1390 | } |
| 1391 | } |
| 1392 | while (copy_len) |
| 1393 | { |
| 1394 | if (PREDICT_FALSE (!(src0 = map_guest_mem (vui, cpy->src, map_hint)))) |
| 1395 | return 1; |
| 1396 | clib_memcpy ((void *) cpy->dst, src0, cpy->len); |
| 1397 | copy_len -= 1; |
| 1398 | cpy += 1; |
| 1399 | } |
| 1400 | return 0; |
| 1401 | } |
| 1402 | |
| 1403 | /** |
| 1404 | * Try to discard packets from the tx ring (VPP RX path). |
| 1405 | * Returns the number of discarded packets. |
| 1406 | */ |
| 1407 | u32 |
| 1408 | vhost_user_rx_discard_packet (vlib_main_t * vm, |
| 1409 | vhost_user_intf_t * vui, |
| 1410 | vhost_user_vring_t * txvq, u32 discard_max) |
| 1411 | { |
| 1412 | /* |
| 1413 | * On the RX side, each packet corresponds to one descriptor |
| 1414 | * (it is the same whether it is a shallow descriptor, chained, or indirect). |
| 1415 | * Therefore, discarding a packet is like discarding a descriptor. |
| 1416 | */ |
| 1417 | u32 discarded_packets = 0; |
| 1418 | u32 avail_idx = txvq->avail->idx; |
| 1419 | u16 qsz_mask = txvq->qsz - 1; |
| 1420 | while (discarded_packets != discard_max) |
| 1421 | { |
| 1422 | if (avail_idx == txvq->last_avail_idx) |
| 1423 | goto out; |
| 1424 | |
| 1425 | u16 desc_chain_head = |
| 1426 | txvq->avail->ring[txvq->last_avail_idx & qsz_mask]; |
| 1427 | txvq->last_avail_idx++; |
| 1428 | txvq->used->ring[txvq->last_used_idx & qsz_mask].id = desc_chain_head; |
| 1429 | txvq->used->ring[txvq->last_used_idx & qsz_mask].len = 0; |
| 1430 | vhost_user_log_dirty_ring (vui, txvq, |
| 1431 | ring[txvq->last_used_idx & qsz_mask]); |
| 1432 | txvq->last_used_idx++; |
| 1433 | discarded_packets++; |
| 1434 | } |
| 1435 | |
| 1436 | out: |
| 1437 | CLIB_MEMORY_BARRIER (); |
| 1438 | txvq->used->idx = txvq->last_used_idx; |
| 1439 | vhost_user_log_dirty_ring (vui, txvq, idx); |
| 1440 | return discarded_packets; |
| 1441 | } |
| 1442 | |
| 1443 | /* |
| 1444 | * In case of overflow, we need to rewind the array of allocated buffers. |
| 1445 | */ |
| 1446 | static void |
| 1447 | vhost_user_input_rewind_buffers (vlib_main_t * vm, |
| 1448 | vhost_cpu_t * cpu, vlib_buffer_t * b_head) |
| 1449 | { |
| 1450 | u32 bi_current = cpu->rx_buffers[cpu->rx_buffers_len]; |
| 1451 | vlib_buffer_t *b_current = vlib_get_buffer (vm, bi_current); |
| 1452 | b_current->current_length = 0; |
| 1453 | b_current->flags = 0; |
| 1454 | while (b_current != b_head) |
| 1455 | { |
| 1456 | cpu->rx_buffers_len++; |
| 1457 | bi_current = cpu->rx_buffers[cpu->rx_buffers_len]; |
| 1458 | b_current = vlib_get_buffer (vm, bi_current); |
| 1459 | b_current->current_length = 0; |
| 1460 | b_current->flags = 0; |
| 1461 | } |
| 1462 | } |
Yoann Desmouceaux | 4667c22 | 2016-02-24 22:51:00 +0100 | [diff] [blame] | 1463 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1464 | static u32 |
| 1465 | vhost_user_if_input (vlib_main_t * vm, |
| 1466 | vhost_user_main_t * vum, |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1467 | vhost_user_intf_t * vui, |
| 1468 | u16 qid, vlib_node_runtime_t * node) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1469 | { |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1470 | vhost_user_vring_t *txvq = &vui->vrings[VHOST_VRING_IDX_TX (qid)]; |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1471 | u16 n_rx_packets = 0; |
| 1472 | u32 n_rx_bytes = 0; |
| 1473 | u16 n_left; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1474 | u32 n_left_to_next, *to_next; |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1475 | u32 next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT; |
| 1476 | u32 n_trace = vlib_get_trace_count (vm, node); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1477 | u16 qsz_mask; |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1478 | u32 map_hint = 0; |
| 1479 | u16 cpu_index = os_get_cpu_number (); |
| 1480 | u16 copy_len = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1481 | |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1482 | { |
| 1483 | /* do we have pending interrupts ? */ |
| 1484 | vhost_user_vring_t *rxvq = &vui->vrings[VHOST_VRING_IDX_RX (qid)]; |
| 1485 | f64 now = vlib_time_now (vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1486 | |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1487 | if ((txvq->n_since_last_int) && (txvq->int_deadline < now)) |
| 1488 | vhost_user_send_call (vm, txvq); |
| 1489 | |
| 1490 | if ((rxvq->n_since_last_int) && (rxvq->int_deadline < now)) |
| 1491 | vhost_user_send_call (vm, rxvq); |
| 1492 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1493 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1494 | if (PREDICT_FALSE (txvq->avail->flags & 0xFFFE)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1495 | return 0; |
| 1496 | |
Pierre Pfister | ba1d046 | 2016-07-27 16:38:20 +0100 | [diff] [blame] | 1497 | n_left = (u16) (txvq->avail->idx - txvq->last_avail_idx); |
| 1498 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1499 | /* nothing to do */ |
Pierre Pfister | ba1d046 | 2016-07-27 16:38:20 +0100 | [diff] [blame] | 1500 | if (PREDICT_FALSE (n_left == 0)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1501 | return 0; |
| 1502 | |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1503 | if (PREDICT_FALSE (!vui->admin_up || !(txvq->enabled))) |
Pierre Pfister | ba1d046 | 2016-07-27 16:38:20 +0100 | [diff] [blame] | 1504 | { |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1505 | /* |
| 1506 | * Discard input packet if interface is admin down or vring is not |
| 1507 | * enabled. |
| 1508 | * "For example, for a networking device, in the disabled state |
| 1509 | * client must not supply any new RX packets, but must process |
| 1510 | * and discard any TX packets." |
| 1511 | */ |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1512 | vhost_user_rx_discard_packet (vm, vui, txvq, |
| 1513 | VHOST_USER_DOWN_DISCARD_COUNT); |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1514 | return 0; |
| 1515 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1516 | |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1517 | if (PREDICT_FALSE (n_left == txvq->qsz)) |
| 1518 | { |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1519 | /* |
| 1520 | * Informational error logging when VPP is not |
| 1521 | * receiving packets fast enough. |
| 1522 | */ |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1523 | vlib_error_count (vm, node->node_index, |
| 1524 | VHOST_USER_INPUT_FUNC_ERROR_FULL_RX_QUEUE, 1); |
| 1525 | } |
| 1526 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1527 | qsz_mask = txvq->qsz - 1; |
| 1528 | |
Pierre Pfister | 328e99b | 2016-02-12 13:18:42 +0000 | [diff] [blame] | 1529 | if (n_left > VLIB_FRAME_SIZE) |
| 1530 | n_left = VLIB_FRAME_SIZE; |
| 1531 | |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1532 | /* |
| 1533 | * For small packets (<2kB), we will not need more than one vlib buffer |
| 1534 | * per packet. In case packets are bigger, we will just yeld at some point |
| 1535 | * in the loop and come back later. This is not an issue as for big packet, |
| 1536 | * processing cost really comes from the memory copy. |
Pierre Pfister | 328e99b | 2016-02-12 13:18:42 +0000 | [diff] [blame] | 1537 | */ |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1538 | if (PREDICT_FALSE (vum->cpus[cpu_index].rx_buffers_len < n_left + 1)) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1539 | { |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1540 | u32 curr_len = vum->cpus[cpu_index].rx_buffers_len; |
| 1541 | vum->cpus[cpu_index].rx_buffers_len += |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1542 | vlib_buffer_alloc_from_free_list (vm, |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1543 | vum->cpus[cpu_index].rx_buffers + |
Pierre Pfister | ba1d046 | 2016-07-27 16:38:20 +0100 | [diff] [blame] | 1544 | curr_len, |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1545 | VHOST_USER_RX_BUFFERS_N - curr_len, |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1546 | VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1547 | |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1548 | if (PREDICT_FALSE |
| 1549 | (vum->cpus[cpu_index].rx_buffers_len < |
| 1550 | VHOST_USER_RX_BUFFER_STARVATION)) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1551 | { |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1552 | /* In case of buffer starvation, discard some packets from the queue |
| 1553 | * and log the event. |
| 1554 | * We keep doing best effort for the remaining packets. */ |
| 1555 | u32 flush = (n_left + 1 > vum->cpus[cpu_index].rx_buffers_len) ? |
| 1556 | n_left + 1 - vum->cpus[cpu_index].rx_buffers_len : 1; |
| 1557 | flush = vhost_user_rx_discard_packet (vm, vui, txvq, flush); |
| 1558 | |
| 1559 | n_left -= flush; |
| 1560 | vlib_increment_simple_counter (vnet_main. |
| 1561 | interface_main.sw_if_counters + |
| 1562 | VNET_INTERFACE_COUNTER_DROP, |
| 1563 | os_get_cpu_number (), |
| 1564 | vui->sw_if_index, flush); |
| 1565 | |
| 1566 | vlib_error_count (vm, vhost_user_input_node.index, |
| 1567 | VHOST_USER_INPUT_FUNC_ERROR_NO_BUFFER, flush); |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1568 | } |
| 1569 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1570 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1571 | while (n_left > 0) |
| 1572 | { |
| 1573 | vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1574 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1575 | while (n_left > 0 && n_left_to_next > 0) |
| 1576 | { |
| 1577 | vlib_buffer_t *b_head, *b_current; |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1578 | u32 bi_current; |
| 1579 | u16 desc_current; |
| 1580 | u32 desc_data_offset; |
| 1581 | vring_desc_t *desc_table = txvq->desc; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1582 | |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1583 | if (PREDICT_FALSE (vum->cpus[cpu_index].rx_buffers_len <= 1)) |
Pierre Pfister | ba1d046 | 2016-07-27 16:38:20 +0100 | [diff] [blame] | 1584 | { |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1585 | /* Not enough rx_buffers |
| 1586 | * Note: We yeld on 1 so we don't need to do an additional |
| 1587 | * check for the next buffer prefetch. |
| 1588 | */ |
| 1589 | n_left = 0; |
| 1590 | break; |
Pierre Pfister | ba1d046 | 2016-07-27 16:38:20 +0100 | [diff] [blame] | 1591 | } |
| 1592 | |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1593 | desc_current = txvq->avail->ring[txvq->last_avail_idx & qsz_mask]; |
| 1594 | vum->cpus[cpu_index].rx_buffers_len--; |
| 1595 | bi_current = (vum->cpus[cpu_index].rx_buffers) |
| 1596 | [vum->cpus[cpu_index].rx_buffers_len]; |
| 1597 | b_head = b_current = vlib_get_buffer (vm, bi_current); |
| 1598 | to_next[0] = bi_current; //We do that now so we can forget about bi_current |
| 1599 | to_next++; |
| 1600 | n_left_to_next--; |
| 1601 | |
| 1602 | vlib_prefetch_buffer_with_index (vm, |
| 1603 | (vum->cpus[cpu_index].rx_buffers) |
| 1604 | [vum->cpus[cpu_index]. |
| 1605 | rx_buffers_len - 1], LOAD); |
| 1606 | |
| 1607 | /* Just preset the used descriptor id and length for later */ |
| 1608 | txvq->used->ring[txvq->last_used_idx & qsz_mask].id = desc_current; |
| 1609 | txvq->used->ring[txvq->last_used_idx & qsz_mask].len = 0; |
| 1610 | vhost_user_log_dirty_ring (vui, txvq, |
| 1611 | ring[txvq->last_used_idx & qsz_mask]); |
| 1612 | |
| 1613 | /* The buffer should already be initialized */ |
| 1614 | b_head->total_length_not_including_first_buffer = 0; |
| 1615 | b_head->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; |
| 1616 | |
Pierre Pfister | 116ea4b | 2016-11-08 15:49:28 +0000 | [diff] [blame] | 1617 | if (PREDICT_FALSE (n_trace)) |
| 1618 | { |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1619 | //TODO: next_index is not exactly known at that point |
Pierre Pfister | 116ea4b | 2016-11-08 15:49:28 +0000 | [diff] [blame] | 1620 | vlib_trace_buffer (vm, node, next_index, b_head, |
| 1621 | /* follow_chain */ 0); |
| 1622 | vhost_trace_t *t0 = |
| 1623 | vlib_add_trace (vm, node, b_head, sizeof (t0[0])); |
| 1624 | vhost_user_rx_trace (t0, vui, qid, b_head, txvq); |
| 1625 | n_trace--; |
| 1626 | vlib_set_trace_count (vm, node, n_trace); |
| 1627 | } |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1628 | |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1629 | /* This depends on the setup but is very consistent |
| 1630 | * So I think the CPU branch predictor will make a pretty good job |
| 1631 | * at optimizing the decision. */ |
| 1632 | if (txvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT) |
| 1633 | { |
| 1634 | desc_table = map_guest_mem (vui, txvq->desc[desc_current].addr, |
| 1635 | &map_hint); |
| 1636 | desc_current = 0; |
| 1637 | if (PREDICT_FALSE (desc_table == 0)) |
| 1638 | { |
| 1639 | //FIXME: Handle error by shutdown the queue |
| 1640 | goto out; |
| 1641 | } |
| 1642 | } |
| 1643 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1644 | if (PREDICT_TRUE (vui->is_any_layout) || |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1645 | (!(desc_table[desc_current].flags & VIRTQ_DESC_F_NEXT))) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1646 | { |
| 1647 | /* ANYLAYOUT or single buffer */ |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1648 | desc_data_offset = vui->virtio_net_hdr_sz; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1649 | } |
| 1650 | else |
| 1651 | { |
| 1652 | /* CSR case without ANYLAYOUT, skip 1st buffer */ |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1653 | desc_data_offset = desc_table[desc_current].len; |
Pierre Pfister | ba1d046 | 2016-07-27 16:38:20 +0100 | [diff] [blame] | 1654 | } |
| 1655 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1656 | while (1) |
| 1657 | { |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1658 | /* Get more input if necessary. Or end of packet. */ |
| 1659 | if (desc_data_offset == desc_table[desc_current].len) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1660 | { |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1661 | if (PREDICT_FALSE (desc_table[desc_current].flags & |
| 1662 | VIRTQ_DESC_F_NEXT)) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1663 | { |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1664 | desc_current = desc_table[desc_current].next; |
| 1665 | desc_data_offset = 0; |
| 1666 | } |
| 1667 | else |
| 1668 | { |
| 1669 | goto out; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1670 | } |
| 1671 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1672 | |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1673 | /* Get more output if necessary. Or end of packet. */ |
| 1674 | if (PREDICT_FALSE |
| 1675 | (b_current->current_length == VLIB_BUFFER_DATA_SIZE)) |
| 1676 | { |
| 1677 | if (PREDICT_FALSE |
| 1678 | (vum->cpus[cpu_index].rx_buffers_len == 0)) |
| 1679 | { |
| 1680 | /* |
| 1681 | * Checking if there are some left buffers. |
| 1682 | * If not, just rewind the used buffers and stop. |
| 1683 | * Note: Scheduled copies are not cancelled. This is |
| 1684 | * not an issue as they would still be valid. Useless, |
| 1685 | * but valid. |
| 1686 | */ |
| 1687 | vhost_user_input_rewind_buffers (vm, |
| 1688 | &vum->cpus[cpu_index], |
| 1689 | b_head); |
| 1690 | n_left = 0; |
| 1691 | goto stop; |
| 1692 | } |
| 1693 | |
| 1694 | /* Get next output */ |
| 1695 | vum->cpus[cpu_index].rx_buffers_len--; |
| 1696 | u32 bi_next = |
| 1697 | (vum->cpus[cpu_index].rx_buffers)[vum->cpus |
| 1698 | [cpu_index].rx_buffers_len]; |
| 1699 | b_current->next_buffer = bi_next; |
| 1700 | b_current->flags |= VLIB_BUFFER_NEXT_PRESENT; |
| 1701 | bi_current = bi_next; |
| 1702 | b_current = vlib_get_buffer (vm, bi_current); |
| 1703 | } |
| 1704 | |
| 1705 | /* Prepare a copy order executed later for the data */ |
| 1706 | vhost_copy_t *cpy = &vum->cpus[cpu_index].copy[copy_len]; |
| 1707 | copy_len++; |
| 1708 | u32 desc_data_l = |
| 1709 | desc_table[desc_current].len - desc_data_offset; |
| 1710 | cpy->len = VLIB_BUFFER_DATA_SIZE - b_current->current_length; |
| 1711 | cpy->len = (cpy->len > desc_data_l) ? desc_data_l : cpy->len; |
| 1712 | cpy->dst = (uword) vlib_buffer_get_current (b_current); |
| 1713 | cpy->src = desc_table[desc_current].addr + desc_data_offset; |
| 1714 | |
| 1715 | desc_data_offset += cpy->len; |
| 1716 | |
| 1717 | b_current->current_length += cpy->len; |
| 1718 | b_head->total_length_not_including_first_buffer += cpy->len; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1719 | } |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1720 | |
Pierre Pfister | ba1d046 | 2016-07-27 16:38:20 +0100 | [diff] [blame] | 1721 | out: |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1722 | CLIB_PREFETCH (&n_left, sizeof (n_left), LOAD); |
| 1723 | |
| 1724 | n_rx_bytes += b_head->total_length_not_including_first_buffer; |
| 1725 | n_rx_packets++; |
| 1726 | |
| 1727 | b_head->total_length_not_including_first_buffer -= |
| 1728 | b_head->current_length; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1729 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1730 | /* consume the descriptor and return it as used */ |
| 1731 | txvq->last_avail_idx++; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1732 | txvq->last_used_idx++; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1733 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1734 | VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b_head); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1735 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1736 | vnet_buffer (b_head)->sw_if_index[VLIB_RX] = vui->sw_if_index; |
| 1737 | vnet_buffer (b_head)->sw_if_index[VLIB_TX] = (u32) ~ 0; |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1738 | b_head->error = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1739 | |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1740 | { |
| 1741 | u32 next0 = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT; |
Pierre Pfister | 328e99b | 2016-02-12 13:18:42 +0000 | [diff] [blame] | 1742 | |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1743 | /* redirect if feature path enabled */ |
| 1744 | vnet_feature_start_device_input_x1 (vui->sw_if_index, &next0, |
| 1745 | b_head, 0); |
Damjan Marion | 2231150 | 2016-10-28 20:30:15 +0200 | [diff] [blame] | 1746 | |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1747 | u32 bi = to_next[-1]; //Cannot use to_next[-1] in the macro |
| 1748 | vlib_validate_buffer_enqueue_x1 (vm, node, next_index, |
| 1749 | to_next, n_left_to_next, |
| 1750 | bi, next0); |
| 1751 | } |
Damjan Marion | 2231150 | 2016-10-28 20:30:15 +0200 | [diff] [blame] | 1752 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1753 | n_left--; |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1754 | |
| 1755 | /* |
| 1756 | * Although separating memory copies from virtio ring parsing |
| 1757 | * is beneficial, we can offer to perform the copies from time |
| 1758 | * to time in order to free some space in the ring. |
| 1759 | */ |
| 1760 | if (PREDICT_FALSE (copy_len >= VHOST_USER_RX_COPY_THRESHOLD)) |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1761 | { |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1762 | if (PREDICT_FALSE |
| 1763 | (vhost_user_input_copy (vui, vum->cpus[cpu_index].copy, |
| 1764 | copy_len, &map_hint))) |
| 1765 | { |
| 1766 | clib_warning |
| 1767 | ("Memory mapping error on interface hw_if_index=%d " |
| 1768 | "(Shutting down - Switch interface down and up to restart)", |
| 1769 | vui->hw_if_index); |
| 1770 | vui->admin_up = 0; |
| 1771 | copy_len = 0; |
| 1772 | break; |
| 1773 | } |
| 1774 | copy_len = 0; |
| 1775 | |
| 1776 | /* give buffers back to driver */ |
| 1777 | CLIB_MEMORY_BARRIER (); |
| 1778 | txvq->used->idx = txvq->last_used_idx; |
| 1779 | vhost_user_log_dirty_ring (vui, txvq, idx); |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1780 | } |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1781 | } |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1782 | stop: |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1783 | vlib_put_next_frame (vm, node, next_index, n_left_to_next); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1784 | } |
| 1785 | |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1786 | /* Do the memory copies */ |
| 1787 | if (PREDICT_FALSE |
| 1788 | (vhost_user_input_copy (vui, vum->cpus[cpu_index].copy, |
| 1789 | copy_len, &map_hint))) |
| 1790 | { |
| 1791 | clib_warning ("Memory mapping error on interface hw_if_index=%d " |
| 1792 | "(Shutting down - Switch interface down and up to restart)", |
| 1793 | vui->hw_if_index); |
| 1794 | vui->admin_up = 0; |
| 1795 | } |
Pierre Pfister | 328e99b | 2016-02-12 13:18:42 +0000 | [diff] [blame] | 1796 | |
| 1797 | /* give buffers back to driver */ |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1798 | CLIB_MEMORY_BARRIER (); |
Pierre Pfister | 328e99b | 2016-02-12 13:18:42 +0000 | [diff] [blame] | 1799 | txvq->used->idx = txvq->last_used_idx; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1800 | vhost_user_log_dirty_ring (vui, txvq, idx); |
Pierre Pfister | 328e99b | 2016-02-12 13:18:42 +0000 | [diff] [blame] | 1801 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1802 | /* interrupt (call) handling */ |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 1803 | if ((txvq->callfd_idx != ~0) && !(txvq->avail->flags & 1)) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1804 | { |
| 1805 | txvq->n_since_last_int += n_rx_packets; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1806 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1807 | if (txvq->n_since_last_int > vum->coalesce_frames) |
| 1808 | vhost_user_send_call (vm, txvq); |
| 1809 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1810 | |
| 1811 | /* increase rx counters */ |
| 1812 | vlib_increment_combined_counter |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1813 | (vnet_main.interface_main.combined_sw_if_counters |
| 1814 | + VNET_INTERFACE_COUNTER_RX, |
| 1815 | os_get_cpu_number (), vui->sw_if_index, n_rx_packets, n_rx_bytes); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1816 | |
| 1817 | return n_rx_packets; |
| 1818 | } |
| 1819 | |
| 1820 | static uword |
| 1821 | vhost_user_input (vlib_main_t * vm, |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1822 | vlib_node_runtime_t * node, vlib_frame_t * f) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1823 | { |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1824 | vhost_user_main_t *vum = &vhost_user_main; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1825 | uword n_rx_packets = 0; |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1826 | u32 cpu_index = os_get_cpu_number (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1827 | |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1828 | |
| 1829 | vhost_iface_and_queue_t *vhiq; |
| 1830 | vec_foreach (vhiq, vum->cpus[cpu_index].rx_queues) |
| 1831 | { |
| 1832 | vhost_user_intf_t *vui = |
| 1833 | &vum->vhost_user_interfaces[vhiq->vhost_iface_index]; |
| 1834 | n_rx_packets += vhost_user_if_input (vm, vum, vui, vhiq->qid, node); |
| 1835 | } |
| 1836 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1837 | return n_rx_packets; |
| 1838 | } |
| 1839 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1840 | /* *INDENT-OFF* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1841 | VLIB_REGISTER_NODE (vhost_user_input_node) = { |
| 1842 | .function = vhost_user_input, |
| 1843 | .type = VLIB_NODE_TYPE_INPUT, |
| 1844 | .name = "vhost-user-input", |
Damjan Marion | 51327ac | 2016-11-09 11:59:42 +0100 | [diff] [blame] | 1845 | .sibling_of = "device-input", |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1846 | |
| 1847 | /* Will be enabled if/when hardware is detected. */ |
| 1848 | .state = VLIB_NODE_STATE_DISABLED, |
| 1849 | |
| 1850 | .format_buffer = format_ethernet_header_with_length, |
Pierre Pfister | 116ea4b | 2016-11-08 15:49:28 +0000 | [diff] [blame] | 1851 | .format_trace = format_vhost_trace, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1852 | |
| 1853 | .n_errors = VHOST_USER_INPUT_FUNC_N_ERROR, |
| 1854 | .error_strings = vhost_user_input_func_error_strings, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1855 | }; |
| 1856 | |
Damjan Marion | 1c80e83 | 2016-05-11 23:07:18 +0200 | [diff] [blame] | 1857 | VLIB_NODE_FUNCTION_MULTIARCH (vhost_user_input_node, vhost_user_input) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1858 | /* *INDENT-ON* */ |
Damjan Marion | 1c80e83 | 2016-05-11 23:07:18 +0200 | [diff] [blame] | 1859 | |
Pierre Pfister | 116ea4b | 2016-11-08 15:49:28 +0000 | [diff] [blame] | 1860 | |
| 1861 | void |
| 1862 | vhost_user_tx_trace (vhost_trace_t * t, |
| 1863 | vhost_user_intf_t * vui, u16 qid, |
| 1864 | vlib_buffer_t * b, vhost_user_vring_t * rxvq) |
| 1865 | { |
| 1866 | vhost_user_main_t *vum = &vhost_user_main; |
| 1867 | u32 qsz_mask = rxvq->qsz - 1; |
| 1868 | u32 last_avail_idx = rxvq->last_avail_idx; |
| 1869 | u32 desc_current = rxvq->avail->ring[last_avail_idx & qsz_mask]; |
| 1870 | vring_desc_t *hdr_desc = 0; |
| 1871 | u32 hint = 0; |
| 1872 | |
| 1873 | memset (t, 0, sizeof (*t)); |
| 1874 | t->device_index = vui - vum->vhost_user_interfaces; |
| 1875 | t->qid = qid; |
| 1876 | |
| 1877 | hdr_desc = &rxvq->desc[desc_current]; |
| 1878 | if (rxvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT) |
| 1879 | { |
| 1880 | t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_INDIRECT; |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1881 | /* Header is the first here */ |
Pierre Pfister | 116ea4b | 2016-11-08 15:49:28 +0000 | [diff] [blame] | 1882 | hdr_desc = map_guest_mem (vui, rxvq->desc[desc_current].addr, &hint); |
| 1883 | } |
| 1884 | if (rxvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT) |
| 1885 | { |
| 1886 | t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SIMPLE_CHAINED; |
| 1887 | } |
| 1888 | if (!(rxvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT) && |
| 1889 | !(rxvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT)) |
| 1890 | { |
| 1891 | t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SINGLE_DESC; |
| 1892 | } |
| 1893 | |
| 1894 | t->first_desc_len = hdr_desc ? hdr_desc->len : 0; |
| 1895 | } |
| 1896 | |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1897 | static_always_inline u32 |
| 1898 | vhost_user_tx_copy (vhost_user_intf_t * vui, vhost_copy_t * cpy, |
| 1899 | u16 copy_len, u32 * map_hint) |
| 1900 | { |
| 1901 | void *dst0, *dst1, *dst2, *dst3; |
| 1902 | if (PREDICT_TRUE (copy_len >= 4)) |
| 1903 | { |
| 1904 | if (PREDICT_FALSE (!(dst2 = map_guest_mem (vui, cpy[0].dst, map_hint)))) |
| 1905 | return 1; |
| 1906 | if (PREDICT_FALSE (!(dst3 = map_guest_mem (vui, cpy[1].dst, map_hint)))) |
| 1907 | return 1; |
| 1908 | while (PREDICT_TRUE (copy_len >= 4)) |
| 1909 | { |
| 1910 | dst0 = dst2; |
| 1911 | dst1 = dst3; |
| 1912 | |
| 1913 | if (PREDICT_FALSE |
| 1914 | (!(dst2 = map_guest_mem (vui, cpy[2].dst, map_hint)))) |
| 1915 | return 1; |
| 1916 | if (PREDICT_FALSE |
| 1917 | (!(dst3 = map_guest_mem (vui, cpy[3].dst, map_hint)))) |
| 1918 | return 1; |
| 1919 | |
| 1920 | CLIB_PREFETCH ((void *) cpy[2].src, 64, LOAD); |
| 1921 | CLIB_PREFETCH ((void *) cpy[3].src, 64, LOAD); |
| 1922 | |
| 1923 | clib_memcpy (dst0, (void *) cpy[0].src, cpy[0].len); |
| 1924 | clib_memcpy (dst1, (void *) cpy[1].src, cpy[1].len); |
| 1925 | |
| 1926 | vhost_user_log_dirty_pages_2 (vui, cpy[0].dst, cpy[0].len, 1); |
| 1927 | vhost_user_log_dirty_pages_2 (vui, cpy[1].dst, cpy[1].len, 1); |
| 1928 | copy_len -= 2; |
| 1929 | cpy += 2; |
| 1930 | } |
| 1931 | } |
| 1932 | while (copy_len) |
| 1933 | { |
| 1934 | if (PREDICT_FALSE (!(dst0 = map_guest_mem (vui, cpy->dst, map_hint)))) |
| 1935 | return 1; |
| 1936 | clib_memcpy (dst0, (void *) cpy->src, cpy->len); |
| 1937 | vhost_user_log_dirty_pages_2 (vui, cpy->dst, cpy->len, 1); |
| 1938 | copy_len -= 1; |
| 1939 | cpy += 1; |
| 1940 | } |
| 1941 | return 0; |
| 1942 | } |
| 1943 | |
| 1944 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1945 | static uword |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 1946 | vhost_user_tx (vlib_main_t * vm, |
| 1947 | vlib_node_runtime_t * node, vlib_frame_t * frame) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1948 | { |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1949 | u32 *buffers = vlib_frame_args (frame); |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1950 | u32 n_left = frame->n_vectors; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1951 | vhost_user_main_t *vum = &vhost_user_main; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1952 | vnet_interface_output_runtime_t *rd = (void *) node->runtime_data; |
| 1953 | vhost_user_intf_t *vui = |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 1954 | pool_elt_at_index (vum->vhost_user_interfaces, rd->dev_instance); |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1955 | u32 qid = ~0; |
| 1956 | vhost_user_vring_t *rxvq; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1957 | u16 qsz_mask; |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1958 | u8 error; |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1959 | u32 cpu_index = os_get_cpu_number (); |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1960 | u32 map_hint = 0; |
| 1961 | u8 retry = 8; |
| 1962 | u16 copy_len; |
| 1963 | u16 tx_headers_len; |
Pierre Pfister | ba1d046 | 2016-07-27 16:38:20 +0100 | [diff] [blame] | 1964 | |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1965 | if (PREDICT_FALSE (!vui->admin_up)) |
| 1966 | { |
| 1967 | error = VHOST_USER_TX_FUNC_ERROR_DOWN; |
| 1968 | goto done3; |
| 1969 | } |
| 1970 | |
| 1971 | if (PREDICT_FALSE (!vui->is_up)) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1972 | { |
| 1973 | error = VHOST_USER_TX_FUNC_ERROR_NOT_READY; |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1974 | goto done3; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1975 | } |
Damjan Marion | 920ecc2 | 2016-01-12 18:34:24 +0100 | [diff] [blame] | 1976 | |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1977 | qid = |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1978 | VHOST_VRING_IDX_RX (*vec_elt_at_index |
| 1979 | (vui->per_cpu_tx_qid, os_get_cpu_number ())); |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 1980 | rxvq = &vui->vrings[qid]; |
| 1981 | if (PREDICT_FALSE (vui->use_tx_spinlock)) |
| 1982 | vhost_user_vring_lock (vui, qid); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1983 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1984 | qsz_mask = rxvq->qsz - 1; /* qsz is always power of 2 */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1985 | |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1986 | retry: |
| 1987 | error = VHOST_USER_TX_FUNC_ERROR_NONE; |
| 1988 | tx_headers_len = 0; |
| 1989 | copy_len = 0; |
| 1990 | while (n_left > 0) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 1991 | { |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1992 | vlib_buffer_t *b0, *current_b0; |
| 1993 | u16 desc_head, desc_index, desc_len; |
| 1994 | vring_desc_t *desc_table; |
| 1995 | uword buffer_map_addr; |
| 1996 | u32 buffer_len; |
| 1997 | u16 bytes_left; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1998 | |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 1999 | if (PREDICT_TRUE (n_left > 1)) |
| 2000 | vlib_prefetch_buffer_with_index (vm, buffers[1], LOAD); |
| 2001 | |
| 2002 | b0 = vlib_get_buffer (vm, buffers[0]); |
| 2003 | |
| 2004 | if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) |
Pierre Pfister | 116ea4b | 2016-11-08 15:49:28 +0000 | [diff] [blame] | 2005 | { |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 2006 | vum->cpus[cpu_index].current_trace = |
| 2007 | vlib_add_trace (vm, node, b0, |
| 2008 | sizeof (*vum->cpus[cpu_index].current_trace)); |
| 2009 | vhost_user_tx_trace (vum->cpus[cpu_index].current_trace, |
| 2010 | vui, qid / 2, b0, rxvq); |
| 2011 | } |
Pierre Pfister | 116ea4b | 2016-11-08 15:49:28 +0000 | [diff] [blame] | 2012 | |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 2013 | if (PREDICT_FALSE (rxvq->last_avail_idx == rxvq->avail->idx)) |
| 2014 | { |
| 2015 | error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF; |
| 2016 | goto done; |
| 2017 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2018 | |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 2019 | desc_table = rxvq->desc; |
| 2020 | desc_head = desc_index = |
| 2021 | rxvq->avail->ring[rxvq->last_avail_idx & qsz_mask]; |
| 2022 | |
| 2023 | /* Go deeper in case of indirect descriptor |
| 2024 | * I don't know of any driver providing indirect for RX. */ |
| 2025 | if (PREDICT_FALSE (rxvq->desc[desc_head].flags & VIRTQ_DESC_F_INDIRECT)) |
| 2026 | { |
| 2027 | if (PREDICT_FALSE |
| 2028 | (rxvq->desc[desc_head].len < sizeof (vring_desc_t))) |
Pierre Pfister | ba1d046 | 2016-07-27 16:38:20 +0100 | [diff] [blame] | 2029 | { |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 2030 | error = VHOST_USER_TX_FUNC_ERROR_INDIRECT_OVERFLOW; |
Pierre Pfister | ba1d046 | 2016-07-27 16:38:20 +0100 | [diff] [blame] | 2031 | goto done; |
| 2032 | } |
| 2033 | if (PREDICT_FALSE |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 2034 | (!(desc_table = |
| 2035 | map_guest_mem (vui, rxvq->desc[desc_index].addr, |
| 2036 | &map_hint)))) |
Pierre Pfister | ba1d046 | 2016-07-27 16:38:20 +0100 | [diff] [blame] | 2037 | { |
| 2038 | error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL; |
| 2039 | goto done; |
| 2040 | } |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 2041 | desc_index = 0; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2042 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2043 | |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 2044 | desc_len = vui->virtio_net_hdr_sz; |
| 2045 | buffer_map_addr = desc_table[desc_index].addr; |
| 2046 | buffer_len = desc_table[desc_index].len; |
| 2047 | |
| 2048 | { |
| 2049 | // Get a header from the header array |
| 2050 | virtio_net_hdr_mrg_rxbuf_t *hdr = |
| 2051 | &vum->cpus[cpu_index].tx_headers[tx_headers_len]; |
| 2052 | tx_headers_len++; |
| 2053 | hdr->hdr.flags = 0; |
| 2054 | hdr->hdr.gso_type = 0; |
| 2055 | hdr->num_buffers = 1; //This is local, no need to check |
| 2056 | |
| 2057 | // Prepare a copy order executed later for the header |
| 2058 | vhost_copy_t *cpy = &vum->cpus[cpu_index].copy[copy_len]; |
| 2059 | copy_len++; |
| 2060 | cpy->len = vui->virtio_net_hdr_sz; |
| 2061 | cpy->dst = buffer_map_addr; |
| 2062 | cpy->src = (uword) hdr; |
| 2063 | } |
| 2064 | |
| 2065 | buffer_map_addr += vui->virtio_net_hdr_sz; |
| 2066 | buffer_len -= vui->virtio_net_hdr_sz; |
| 2067 | bytes_left = b0->current_length; |
| 2068 | current_b0 = b0; |
| 2069 | while (1) |
| 2070 | { |
| 2071 | if (buffer_len == 0) |
| 2072 | { //Get new output |
| 2073 | if (desc_table[desc_index].flags & VIRTQ_DESC_F_NEXT) |
| 2074 | { |
| 2075 | //Next one is chained |
| 2076 | desc_index = desc_table[desc_index].next; |
| 2077 | buffer_map_addr = desc_table[desc_index].addr; |
| 2078 | buffer_len = desc_table[desc_index].len; |
| 2079 | } |
| 2080 | else if (vui->virtio_net_hdr_sz == 12) //MRG is available |
| 2081 | { |
| 2082 | virtio_net_hdr_mrg_rxbuf_t *hdr = |
| 2083 | &vum->cpus[cpu_index].tx_headers[tx_headers_len - 1]; |
| 2084 | |
| 2085 | //Move from available to used buffer |
| 2086 | rxvq->used->ring[rxvq->last_used_idx & qsz_mask].id = |
| 2087 | desc_head; |
| 2088 | rxvq->used->ring[rxvq->last_used_idx & qsz_mask].len = |
| 2089 | desc_len; |
| 2090 | vhost_user_log_dirty_ring (vui, rxvq, |
| 2091 | ring[rxvq->last_used_idx & |
| 2092 | qsz_mask]); |
| 2093 | |
| 2094 | rxvq->last_avail_idx++; |
| 2095 | rxvq->last_used_idx++; |
| 2096 | hdr->num_buffers++; |
| 2097 | desc_len = 0; |
| 2098 | |
| 2099 | if (PREDICT_FALSE |
| 2100 | (rxvq->last_avail_idx == rxvq->avail->idx)) |
| 2101 | { |
| 2102 | //Dequeue queued descriptors for this packet |
| 2103 | rxvq->last_used_idx -= hdr->num_buffers - 1; |
| 2104 | rxvq->last_avail_idx -= hdr->num_buffers - 1; |
| 2105 | error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF; |
| 2106 | goto done; |
| 2107 | } |
| 2108 | |
| 2109 | desc_table = rxvq->desc; |
| 2110 | desc_head = desc_index = |
| 2111 | rxvq->avail->ring[rxvq->last_avail_idx & qsz_mask]; |
| 2112 | if (PREDICT_FALSE |
| 2113 | (rxvq->desc[desc_head].flags & VIRTQ_DESC_F_INDIRECT)) |
| 2114 | { |
| 2115 | //It is seriously unlikely that a driver will put indirect descriptor |
| 2116 | //after non-indirect descriptor. |
| 2117 | if (PREDICT_FALSE |
| 2118 | (rxvq->desc[desc_head].len < sizeof (vring_desc_t))) |
| 2119 | { |
| 2120 | error = VHOST_USER_TX_FUNC_ERROR_INDIRECT_OVERFLOW; |
| 2121 | goto done; |
| 2122 | } |
| 2123 | if (PREDICT_FALSE |
| 2124 | (!(desc_table = |
| 2125 | map_guest_mem (vui, |
| 2126 | rxvq->desc[desc_index].addr, |
| 2127 | &map_hint)))) |
| 2128 | { |
| 2129 | error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL; |
| 2130 | goto done; |
| 2131 | } |
| 2132 | desc_index = 0; |
| 2133 | } |
| 2134 | buffer_map_addr = desc_table[desc_index].addr; |
| 2135 | buffer_len = desc_table[desc_index].len; |
| 2136 | } |
| 2137 | else |
| 2138 | { |
| 2139 | error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOMRG; |
| 2140 | goto done; |
| 2141 | } |
| 2142 | } |
| 2143 | |
| 2144 | { |
| 2145 | vhost_copy_t *cpy = &vum->cpus[cpu_index].copy[copy_len]; |
| 2146 | copy_len++; |
| 2147 | cpy->len = bytes_left; |
| 2148 | cpy->len = (cpy->len > buffer_len) ? buffer_len : cpy->len; |
| 2149 | cpy->dst = buffer_map_addr; |
| 2150 | cpy->src = (uword) vlib_buffer_get_current (current_b0) + |
| 2151 | current_b0->current_length - bytes_left; |
| 2152 | |
| 2153 | bytes_left -= cpy->len; |
| 2154 | buffer_len -= cpy->len; |
| 2155 | buffer_map_addr += cpy->len; |
| 2156 | desc_len += cpy->len; |
| 2157 | |
Pierre Pfister | 14ac801 | 2016-12-08 07:58:47 +0000 | [diff] [blame] | 2158 | CLIB_PREFETCH (&rxvq->desc, CLIB_CACHE_LINE_BYTES, LOAD); |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 2159 | } |
| 2160 | |
| 2161 | // Check if vlib buffer has more data. If not, get more or break. |
| 2162 | if (PREDICT_TRUE (!bytes_left)) |
| 2163 | { |
| 2164 | if (PREDICT_FALSE |
| 2165 | (current_b0->flags & VLIB_BUFFER_NEXT_PRESENT)) |
| 2166 | { |
| 2167 | current_b0 = vlib_get_buffer (vm, current_b0->next_buffer); |
| 2168 | bytes_left = current_b0->current_length; |
| 2169 | } |
| 2170 | else |
| 2171 | { |
| 2172 | //End of packet |
| 2173 | break; |
| 2174 | } |
| 2175 | } |
| 2176 | } |
| 2177 | |
| 2178 | //Move from available to used ring |
| 2179 | rxvq->used->ring[rxvq->last_used_idx & qsz_mask].id = desc_head; |
| 2180 | rxvq->used->ring[rxvq->last_used_idx & qsz_mask].len = desc_len; |
| 2181 | vhost_user_log_dirty_ring (vui, rxvq, |
| 2182 | ring[rxvq->last_used_idx & qsz_mask]); |
| 2183 | rxvq->last_avail_idx++; |
| 2184 | rxvq->last_used_idx++; |
| 2185 | |
| 2186 | if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) |
| 2187 | { |
| 2188 | vum->cpus[cpu_index].current_trace->hdr = |
| 2189 | vum->cpus[cpu_index].tx_headers[tx_headers_len - 1]; |
| 2190 | } |
| 2191 | |
| 2192 | n_left--; //At the end for error counting when 'goto done' is invoked |
| 2193 | buffers++; |
| 2194 | } |
| 2195 | |
| 2196 | done: |
| 2197 | //Do the memory copies |
| 2198 | if (PREDICT_FALSE |
| 2199 | (vhost_user_tx_copy (vui, vum->cpus[cpu_index].copy, |
| 2200 | copy_len, &map_hint))) |
| 2201 | { |
| 2202 | clib_warning ("Memory mapping error on interface hw_if_index=%d " |
| 2203 | "(Shutting down - Switch interface down and up to restart)", |
| 2204 | vui->hw_if_index); |
| 2205 | vui->admin_up = 0; |
| 2206 | } |
| 2207 | |
| 2208 | CLIB_MEMORY_BARRIER (); |
| 2209 | rxvq->used->idx = rxvq->last_used_idx; |
| 2210 | vhost_user_log_dirty_ring (vui, rxvq, idx); |
| 2211 | |
| 2212 | /* |
| 2213 | * When n_left is set, error is always set to something too. |
| 2214 | * In case error is due to lack of remaining buffers, we go back up and |
| 2215 | * retry. |
| 2216 | * The idea is that it is better to waste some time on packets |
| 2217 | * that have been processed already than dropping them and get |
| 2218 | * more fresh packets with a good likelyhood that they will be dropped too. |
| 2219 | * This technique also gives more time to VM driver to pick-up packets. |
| 2220 | * In case the traffic flows from physical to virtual interfaces, this |
| 2221 | * technique will end-up leveraging the physical NIC buffer in order to |
| 2222 | * absorb the VM's CPU jitter. |
| 2223 | */ |
| 2224 | if (n_left && (error == VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF) && retry) |
| 2225 | { |
| 2226 | retry--; |
| 2227 | goto retry; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2228 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2229 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2230 | /* interrupt (call) handling */ |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2231 | if ((rxvq->callfd_idx != ~0) && !(rxvq->avail->flags & 1)) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2232 | { |
Pierre Pfister | d3eb90e | 2016-11-29 15:36:14 +0000 | [diff] [blame] | 2233 | rxvq->n_since_last_int += frame->n_vectors - n_left; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2234 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2235 | if (rxvq->n_since_last_int > vum->coalesce_frames) |
| 2236 | vhost_user_send_call (vm, rxvq); |
| 2237 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2238 | |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 2239 | vhost_user_vring_unlock (vui, qid); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2240 | |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 2241 | done3: |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2242 | if (PREDICT_FALSE (n_left && error != VHOST_USER_TX_FUNC_ERROR_NONE)) |
| 2243 | { |
| 2244 | vlib_error_count (vm, node->node_index, error, n_left); |
| 2245 | vlib_increment_simple_counter |
| 2246 | (vnet_main.interface_main.sw_if_counters |
| 2247 | + VNET_INTERFACE_COUNTER_DROP, |
| 2248 | os_get_cpu_number (), vui->sw_if_index, n_left); |
| 2249 | } |
Pierre Pfister | 328e99b | 2016-02-12 13:18:42 +0000 | [diff] [blame] | 2250 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2251 | vlib_buffer_free (vm, vlib_frame_args (frame), frame->n_vectors); |
| 2252 | return frame->n_vectors; |
| 2253 | } |
| 2254 | |
| 2255 | static clib_error_t * |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2256 | vhost_user_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, |
| 2257 | u32 flags) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2258 | { |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2259 | vnet_hw_interface_t *hif = vnet_get_hw_interface (vnm, hw_if_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2260 | uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2261 | vhost_user_main_t *vum = &vhost_user_main; |
| 2262 | vhost_user_intf_t *vui = |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2263 | pool_elt_at_index (vum->vhost_user_interfaces, hif->dev_instance); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2264 | |
| 2265 | vui->admin_up = is_up; |
| 2266 | |
| 2267 | if (is_up) |
| 2268 | vnet_hw_interface_set_flags (vnm, vui->hw_if_index, |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2269 | VNET_HW_INTERFACE_FLAG_LINK_UP); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2270 | |
| 2271 | return /* no error */ 0; |
| 2272 | } |
| 2273 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2274 | /* *INDENT-OFF* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2275 | VNET_DEVICE_CLASS (vhost_user_dev_class,static) = { |
| 2276 | .name = "vhost-user", |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2277 | .tx_function = vhost_user_tx, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2278 | .tx_function_n_errors = VHOST_USER_TX_FUNC_N_ERROR, |
| 2279 | .tx_function_error_strings = vhost_user_tx_func_error_strings, |
| 2280 | .format_device_name = format_vhost_user_interface_name, |
| 2281 | .name_renumber = vhost_user_name_renumber, |
| 2282 | .admin_up_down_function = vhost_user_interface_admin_up_down, |
Pierre Pfister | 116ea4b | 2016-11-08 15:49:28 +0000 | [diff] [blame] | 2283 | .format_tx_trace = format_vhost_trace, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2284 | }; |
| 2285 | |
Damjan Marion | 1c80e83 | 2016-05-11 23:07:18 +0200 | [diff] [blame] | 2286 | VLIB_DEVICE_TX_FUNCTION_MULTIARCH (vhost_user_dev_class, |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2287 | vhost_user_tx) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2288 | /* *INDENT-ON* */ |
Damjan Marion | 1c80e83 | 2016-05-11 23:07:18 +0200 | [diff] [blame] | 2289 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2290 | static uword |
| 2291 | vhost_user_process (vlib_main_t * vm, |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2292 | vlib_node_runtime_t * rt, vlib_frame_t * f) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2293 | { |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2294 | vhost_user_main_t *vum = &vhost_user_main; |
| 2295 | vhost_user_intf_t *vui; |
| 2296 | struct sockaddr_un sun; |
| 2297 | int sockfd; |
| 2298 | unix_file_t template = { 0 }; |
| 2299 | f64 timeout = 3153600000.0 /* 100 years */ ; |
| 2300 | uword *event_data = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2301 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2302 | sockfd = socket (AF_UNIX, SOCK_STREAM, 0); |
| 2303 | sun.sun_family = AF_UNIX; |
| 2304 | template.read_function = vhost_user_socket_read; |
| 2305 | template.error_function = vhost_user_socket_error; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2306 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2307 | if (sockfd < 0) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2308 | return 0; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2309 | |
| 2310 | while (1) |
| 2311 | { |
| 2312 | vlib_process_wait_for_event_or_clock (vm, timeout); |
| 2313 | vlib_process_get_events (vm, &event_data); |
| 2314 | vec_reset_length (event_data); |
| 2315 | |
| 2316 | timeout = 3.0; |
| 2317 | |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2318 | /* *INDENT-OFF* */ |
| 2319 | pool_foreach (vui, vum->vhost_user_interfaces, { |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2320 | |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2321 | if (vui->unix_server_index == ~0) { //Nothing to do for server sockets |
| 2322 | if (vui->unix_file_index == ~0) |
| 2323 | { |
| 2324 | /* try to connect */ |
| 2325 | strncpy (sun.sun_path, (char *) vui->sock_filename, |
| 2326 | sizeof (sun.sun_path) - 1); |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2327 | |
Andrew Yourtchenko | 0c3d467 | 2017-01-03 16:52:22 +0000 | [diff] [blame] | 2328 | /* Avoid hanging VPP if the other end does not accept */ |
| 2329 | fcntl(sockfd, F_SETFL, O_NONBLOCK); |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2330 | if (connect (sockfd, (struct sockaddr *) &sun, |
| 2331 | sizeof (struct sockaddr_un)) == 0) |
| 2332 | { |
Andrew Yourtchenko | 0c3d467 | 2017-01-03 16:52:22 +0000 | [diff] [blame] | 2333 | /* Set the socket to blocking as it was before */ |
| 2334 | fcntl(sockfd, F_SETFL, 0); |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2335 | vui->sock_errno = 0; |
| 2336 | template.file_descriptor = sockfd; |
| 2337 | template.private_data = |
| 2338 | vui - vhost_user_main.vhost_user_interfaces; |
| 2339 | vui->unix_file_index = unix_file_add (&unix_main, &template); |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2340 | |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2341 | //Re-open for next connect |
| 2342 | if ((sockfd = socket (AF_UNIX, SOCK_STREAM, 0)) < 0) { |
| 2343 | clib_warning("Critical: Could not open unix socket"); |
| 2344 | return 0; |
| 2345 | } |
| 2346 | } |
| 2347 | else |
| 2348 | { |
| 2349 | vui->sock_errno = errno; |
| 2350 | } |
| 2351 | } |
| 2352 | else |
| 2353 | { |
| 2354 | /* check if socket is alive */ |
| 2355 | int error = 0; |
| 2356 | socklen_t len = sizeof (error); |
| 2357 | int fd = UNIX_GET_FD(vui->unix_file_index); |
| 2358 | int retval = |
| 2359 | getsockopt (fd, SOL_SOCKET, SO_ERROR, &error, &len); |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2360 | |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2361 | if (retval) |
| 2362 | { |
| 2363 | DBG_SOCK ("getsockopt returned %d", retval); |
| 2364 | vhost_user_if_disconnect (vui); |
| 2365 | } |
| 2366 | } |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2367 | } |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2368 | }); |
| 2369 | /* *INDENT-ON* */ |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2370 | } |
| 2371 | return 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2372 | } |
| 2373 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2374 | /* *INDENT-OFF* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2375 | VLIB_REGISTER_NODE (vhost_user_process_node,static) = { |
| 2376 | .function = vhost_user_process, |
| 2377 | .type = VLIB_NODE_TYPE_PROCESS, |
| 2378 | .name = "vhost-user-process", |
| 2379 | }; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2380 | /* *INDENT-ON* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2381 | |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2382 | /** |
| 2383 | * Disables and reset interface structure. |
| 2384 | * It can then be either init again, or removed from used interfaces. |
| 2385 | */ |
| 2386 | static void |
| 2387 | vhost_user_term_if (vhost_user_intf_t * vui) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2388 | { |
Ole Troan | 553a411 | 2017-01-10 10:07:04 +0100 | [diff] [blame] | 2389 | int q; |
| 2390 | |
Pierre Pfister | 7a91b46 | 2016-11-21 12:50:38 +0000 | [diff] [blame] | 2391 | // Delete configured thread pinning |
| 2392 | vec_reset_length (vui->workers); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2393 | // disconnect interface sockets |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2394 | vhost_user_if_disconnect (vui); |
Pierre Pfister | fbb2ef6 | 2016-11-16 02:43:29 +0000 | [diff] [blame] | 2395 | vhost_user_update_iface_state (vui); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2396 | |
Ole Troan | 553a411 | 2017-01-10 10:07:04 +0100 | [diff] [blame] | 2397 | for (q = 0; q < VHOST_VRING_MAX_N; q++) |
| 2398 | { |
| 2399 | clib_mem_free ((void *) vui->vring_locks[q]); |
| 2400 | } |
| 2401 | |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2402 | if (vui->unix_server_index != ~0) |
| 2403 | { |
| 2404 | //Close server socket |
| 2405 | unix_file_t *uf = pool_elt_at_index (unix_main.file_pool, |
| 2406 | vui->unix_server_index); |
| 2407 | unix_file_del (&unix_main, uf); |
| 2408 | vui->unix_server_index = ~0; |
| 2409 | } |
| 2410 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2411 | |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2412 | int |
| 2413 | vhost_user_delete_if (vnet_main_t * vnm, vlib_main_t * vm, u32 sw_if_index) |
| 2414 | { |
| 2415 | vhost_user_main_t *vum = &vhost_user_main; |
| 2416 | vhost_user_intf_t *vui; |
| 2417 | int rv = 0; |
| 2418 | vnet_hw_interface_t *hwif; |
| 2419 | |
| 2420 | if (!(hwif = vnet_get_sup_hw_interface (vnm, sw_if_index)) || |
| 2421 | hwif->dev_class_index != vhost_user_dev_class.index) |
| 2422 | return VNET_API_ERROR_INVALID_SW_IF_INDEX; |
| 2423 | |
| 2424 | DBG_SOCK ("Deleting vhost-user interface %s (instance %d)", |
| 2425 | hwif->name, hwif->dev_instance); |
| 2426 | |
| 2427 | vui = pool_elt_at_index (vum->vhost_user_interfaces, hwif->dev_instance); |
| 2428 | |
| 2429 | // Disable and reset interface |
| 2430 | vhost_user_term_if (vui); |
| 2431 | |
| 2432 | // Back to pool |
| 2433 | pool_put (vum->vhost_user_interfaces, vui); |
| 2434 | |
| 2435 | // Reset renumbered iface |
| 2436 | if (hwif->dev_instance < |
| 2437 | vec_len (vum->show_dev_instance_by_real_dev_instance)) |
| 2438 | vum->show_dev_instance_by_real_dev_instance[hwif->dev_instance] = ~0; |
| 2439 | |
| 2440 | // Delete ethernet interface |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2441 | ethernet_delete_interface (vnm, vui->hw_if_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2442 | return rv; |
| 2443 | } |
| 2444 | |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2445 | /** |
| 2446 | * Open server unix socket on specified sock_filename. |
| 2447 | */ |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2448 | static int |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2449 | vhost_user_init_server_sock (const char *sock_filename, int *sock_fd) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2450 | { |
Pierre Pfister | 5afccb2 | 2016-07-25 14:32:02 +0100 | [diff] [blame] | 2451 | int rv = 0; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2452 | struct sockaddr_un un = { }; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2453 | int fd; |
| 2454 | /* create listening socket */ |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2455 | if ((fd = socket (AF_UNIX, SOCK_STREAM, 0)) < 0) |
| 2456 | return VNET_API_ERROR_SYSCALL_ERROR_1; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2457 | |
| 2458 | un.sun_family = AF_UNIX; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2459 | strncpy ((char *) un.sun_path, (char *) sock_filename, |
| 2460 | sizeof (un.sun_path) - 1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2461 | |
| 2462 | /* remove if exists */ |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2463 | unlink ((char *) sock_filename); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2464 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2465 | if (bind (fd, (struct sockaddr *) &un, sizeof (un)) == -1) |
| 2466 | { |
| 2467 | rv = VNET_API_ERROR_SYSCALL_ERROR_2; |
| 2468 | goto error; |
| 2469 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2470 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2471 | if (listen (fd, 1) == -1) |
| 2472 | { |
| 2473 | rv = VNET_API_ERROR_SYSCALL_ERROR_3; |
| 2474 | goto error; |
| 2475 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2476 | |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2477 | *sock_fd = fd; |
| 2478 | return 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2479 | |
| 2480 | error: |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2481 | close (fd); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2482 | return rv; |
| 2483 | } |
| 2484 | |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2485 | /** |
| 2486 | * Create ethernet interface for vhost user interface. |
| 2487 | */ |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2488 | static void |
| 2489 | vhost_user_create_ethernet (vnet_main_t * vnm, vlib_main_t * vm, |
| 2490 | vhost_user_intf_t * vui, u8 * hwaddress) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2491 | { |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2492 | vhost_user_main_t *vum = &vhost_user_main; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2493 | u8 hwaddr[6]; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2494 | clib_error_t *error; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2495 | |
| 2496 | /* create hw and sw interface */ |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2497 | if (hwaddress) |
| 2498 | { |
| 2499 | clib_memcpy (hwaddr, hwaddress, 6); |
| 2500 | } |
| 2501 | else |
| 2502 | { |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2503 | random_u32 (&vum->random); |
| 2504 | clib_memcpy (hwaddr + 2, &vum->random, sizeof (vum->random)); |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2505 | hwaddr[0] = 2; |
| 2506 | hwaddr[1] = 0xfe; |
| 2507 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2508 | |
| 2509 | error = ethernet_register_interface |
| 2510 | (vnm, |
| 2511 | vhost_user_dev_class.index, |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2512 | vui - vum->vhost_user_interfaces /* device instance */ , |
| 2513 | hwaddr /* ethernet address */ , |
| 2514 | &vui->hw_if_index, 0 /* flag change */ ); |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2515 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2516 | if (error) |
| 2517 | clib_error_report (error); |
Pierre Pfister | 328e99b | 2016-02-12 13:18:42 +0000 | [diff] [blame] | 2518 | |
| 2519 | vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, vui->hw_if_index); |
| 2520 | hi->max_l3_packet_bytes[VLIB_RX] = hi->max_l3_packet_bytes[VLIB_TX] = 9000; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2521 | } |
| 2522 | |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2523 | /* |
| 2524 | * Initialize vui with specified attributes |
| 2525 | */ |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2526 | static void |
| 2527 | vhost_user_vui_init (vnet_main_t * vnm, |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2528 | vhost_user_intf_t * vui, |
| 2529 | int server_sock_fd, |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2530 | const char *sock_filename, |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2531 | u64 feature_mask, u32 * sw_if_index) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2532 | { |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2533 | vnet_sw_interface_t *sw; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2534 | sw = vnet_get_hw_sw_interface (vnm, vui->hw_if_index); |
Yoann Desmouceaux | 4667c22 | 2016-02-24 22:51:00 +0100 | [diff] [blame] | 2535 | int q; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2536 | |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2537 | if (server_sock_fd != -1) |
| 2538 | { |
| 2539 | unix_file_t template = { 0 }; |
| 2540 | template.read_function = vhost_user_socksvr_accept_ready; |
| 2541 | template.file_descriptor = server_sock_fd; |
| 2542 | template.private_data = vui - vhost_user_main.vhost_user_interfaces; //hw index |
| 2543 | vui->unix_server_index = unix_file_add (&unix_main, &template); |
| 2544 | } |
| 2545 | else |
| 2546 | { |
| 2547 | vui->unix_server_index = ~0; |
| 2548 | } |
| 2549 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2550 | vui->sw_if_index = sw->sw_if_index; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2551 | strncpy (vui->sock_filename, sock_filename, |
| 2552 | ARRAY_LEN (vui->sock_filename) - 1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2553 | vui->sock_errno = 0; |
| 2554 | vui->is_up = 0; |
| 2555 | vui->feature_mask = feature_mask; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2556 | vui->unix_file_index = ~0; |
Yoann Desmouceaux | 4667c22 | 2016-02-24 22:51:00 +0100 | [diff] [blame] | 2557 | vui->log_base_addr = 0; |
| 2558 | |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 2559 | for (q = 0; q < VHOST_VRING_MAX_N; q++) |
| 2560 | vhost_user_vring_init (vui, q); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2561 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2562 | vnet_hw_interface_set_flags (vnm, vui->hw_if_index, 0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2563 | |
| 2564 | if (sw_if_index) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2565 | *sw_if_index = vui->sw_if_index; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2566 | |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 2567 | for (q = 0; q < VHOST_VRING_MAX_N; q++) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2568 | { |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 2569 | vui->vring_locks[q] = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES, |
| 2570 | CLIB_CACHE_LINE_BYTES); |
| 2571 | memset ((void *) vui->vring_locks[q], 0, CLIB_CACHE_LINE_BYTES); |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2572 | } |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 2573 | |
| 2574 | vec_validate (vui->per_cpu_tx_qid, |
| 2575 | vlib_get_thread_main ()->n_vlib_mains - 1); |
| 2576 | vhost_user_tx_thread_placement (vui); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2577 | } |
| 2578 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2579 | int |
| 2580 | vhost_user_create_if (vnet_main_t * vnm, vlib_main_t * vm, |
| 2581 | const char *sock_filename, |
| 2582 | u8 is_server, |
| 2583 | u32 * sw_if_index, |
| 2584 | u64 feature_mask, |
| 2585 | u8 renumber, u32 custom_dev_instance, u8 * hwaddr) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2586 | { |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2587 | vhost_user_intf_t *vui = NULL; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2588 | u32 sw_if_idx = ~0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2589 | int rv = 0; |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2590 | int server_sock_fd = -1; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2591 | |
Wojciech Dec | 3cd9eed | 2017-01-03 10:38:37 +0100 | [diff] [blame] | 2592 | if (sock_filename == NULL || !(strlen (sock_filename) > 0)) |
| 2593 | { |
| 2594 | return VNET_API_ERROR_INVALID_ARGUMENT; |
| 2595 | } |
| 2596 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2597 | if (is_server) |
| 2598 | { |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2599 | if ((rv = |
| 2600 | vhost_user_init_server_sock (sock_filename, &server_sock_fd)) != 0) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2601 | { |
| 2602 | return rv; |
| 2603 | } |
| 2604 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2605 | |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2606 | pool_get (vhost_user_main.vhost_user_interfaces, vui); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2607 | |
Pierre Pfister | ef65cb0 | 2016-02-19 13:52:44 +0000 | [diff] [blame] | 2608 | vhost_user_create_ethernet (vnm, vm, vui, hwaddr); |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2609 | vhost_user_vui_init (vnm, vui, server_sock_fd, sock_filename, |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2610 | feature_mask, &sw_if_idx); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2611 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2612 | if (renumber) |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2613 | vnet_interface_name_renumber (sw_if_idx, custom_dev_instance); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2614 | |
| 2615 | if (sw_if_index) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2616 | *sw_if_index = sw_if_idx; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2617 | |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2618 | // Process node must connect |
| 2619 | vlib_process_signal_event (vm, vhost_user_process_node.index, 0, 0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2620 | return rv; |
| 2621 | } |
| 2622 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2623 | int |
| 2624 | vhost_user_modify_if (vnet_main_t * vnm, vlib_main_t * vm, |
| 2625 | const char *sock_filename, |
| 2626 | u8 is_server, |
| 2627 | u32 sw_if_index, |
| 2628 | u64 feature_mask, u8 renumber, u32 custom_dev_instance) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2629 | { |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2630 | vhost_user_main_t *vum = &vhost_user_main; |
| 2631 | vhost_user_intf_t *vui = NULL; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2632 | u32 sw_if_idx = ~0; |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2633 | int server_sock_fd = -1; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2634 | int rv = 0; |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2635 | vnet_hw_interface_t *hwif; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2636 | |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2637 | if (!(hwif = vnet_get_sup_hw_interface (vnm, sw_if_index)) || |
| 2638 | hwif->dev_class_index != vhost_user_dev_class.index) |
| 2639 | return VNET_API_ERROR_INVALID_SW_IF_INDEX; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2640 | |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2641 | vui = vec_elt_at_index (vum->vhost_user_interfaces, hwif->dev_instance); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2642 | |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2643 | // First try to open server socket |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2644 | if (is_server) |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2645 | if ((rv = vhost_user_init_server_sock (sock_filename, |
| 2646 | &server_sock_fd)) != 0) |
| 2647 | return rv; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2648 | |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2649 | vhost_user_term_if (vui); |
| 2650 | vhost_user_vui_init (vnm, vui, server_sock_fd, |
| 2651 | sock_filename, feature_mask, &sw_if_idx); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2652 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2653 | if (renumber) |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2654 | vnet_interface_name_renumber (sw_if_idx, custom_dev_instance); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2655 | |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2656 | // Process node must connect |
| 2657 | vlib_process_signal_event (vm, vhost_user_process_node.index, 0, 0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2658 | return rv; |
| 2659 | } |
| 2660 | |
| 2661 | clib_error_t * |
| 2662 | vhost_user_connect_command_fn (vlib_main_t * vm, |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2663 | unformat_input_t * input, |
| 2664 | vlib_cli_command_t * cmd) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2665 | { |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2666 | unformat_input_t _line_input, *line_input = &_line_input; |
| 2667 | u8 *sock_filename = NULL; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2668 | u32 sw_if_index; |
| 2669 | u8 is_server = 0; |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 2670 | u64 feature_mask = (u64) ~ (0ULL); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2671 | u8 renumber = 0; |
| 2672 | u32 custom_dev_instance = ~0; |
Pierre Pfister | ef65cb0 | 2016-02-19 13:52:44 +0000 | [diff] [blame] | 2673 | u8 hwaddr[6]; |
| 2674 | u8 *hw = NULL; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2675 | |
| 2676 | /* Get a line of input. */ |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2677 | if (!unformat_user (input, unformat_line_input, line_input)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2678 | return 0; |
| 2679 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2680 | while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) |
| 2681 | { |
| 2682 | if (unformat (line_input, "socket %s", &sock_filename)) |
| 2683 | ; |
| 2684 | else if (unformat (line_input, "server")) |
| 2685 | is_server = 1; |
| 2686 | else if (unformat (line_input, "feature-mask 0x%llx", &feature_mask)) |
| 2687 | ; |
| 2688 | else |
| 2689 | if (unformat |
| 2690 | (line_input, "hwaddr %U", unformat_ethernet_address, hwaddr)) |
| 2691 | hw = hwaddr; |
| 2692 | else if (unformat (line_input, "renumber %d", &custom_dev_instance)) |
| 2693 | { |
| 2694 | renumber = 1; |
| 2695 | } |
| 2696 | else |
| 2697 | return clib_error_return (0, "unknown input `%U'", |
| 2698 | format_unformat_error, input); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2699 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2700 | unformat_free (line_input); |
| 2701 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2702 | vnet_main_t *vnm = vnet_get_main (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2703 | |
Pierre Pfister | 5afccb2 | 2016-07-25 14:32:02 +0100 | [diff] [blame] | 2704 | int rv; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2705 | if ((rv = vhost_user_create_if (vnm, vm, (char *) sock_filename, |
| 2706 | is_server, &sw_if_index, feature_mask, |
| 2707 | renumber, custom_dev_instance, hw))) |
| 2708 | { |
| 2709 | vec_free (sock_filename); |
Pierre Pfister | 5afccb2 | 2016-07-25 14:32:02 +0100 | [diff] [blame] | 2710 | return clib_error_return (0, "vhost_user_create_if returned %d", rv); |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2711 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2712 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2713 | vec_free (sock_filename); |
| 2714 | vlib_cli_output (vm, "%U\n", format_vnet_sw_if_index_name, vnet_get_main (), |
| 2715 | sw_if_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2716 | return 0; |
| 2717 | } |
| 2718 | |
| 2719 | clib_error_t * |
| 2720 | vhost_user_delete_command_fn (vlib_main_t * vm, |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2721 | unformat_input_t * input, |
| 2722 | vlib_cli_command_t * cmd) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2723 | { |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2724 | unformat_input_t _line_input, *line_input = &_line_input; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2725 | u32 sw_if_index = ~0; |
Pierre Pfister | ece983d | 2016-11-21 12:52:22 +0000 | [diff] [blame] | 2726 | vnet_main_t *vnm = vnet_get_main (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2727 | |
| 2728 | /* Get a line of input. */ |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2729 | if (!unformat_user (input, unformat_line_input, line_input)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2730 | return 0; |
| 2731 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2732 | while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) |
| 2733 | { |
| 2734 | if (unformat (line_input, "sw_if_index %d", &sw_if_index)) |
| 2735 | ; |
Pierre Pfister | ece983d | 2016-11-21 12:52:22 +0000 | [diff] [blame] | 2736 | else if (unformat |
| 2737 | (line_input, "%U", unformat_vnet_sw_interface, vnm, |
| 2738 | &sw_if_index)) |
| 2739 | { |
| 2740 | vnet_hw_interface_t *hwif = |
| 2741 | vnet_get_sup_hw_interface (vnm, sw_if_index); |
| 2742 | if (hwif == NULL || |
| 2743 | vhost_user_dev_class.index != hwif->dev_class_index) |
| 2744 | return clib_error_return (0, "Not a vhost interface"); |
| 2745 | } |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2746 | else |
| 2747 | return clib_error_return (0, "unknown input `%U'", |
| 2748 | format_unformat_error, input); |
| 2749 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2750 | unformat_free (line_input); |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2751 | vhost_user_delete_if (vnm, vm, sw_if_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2752 | return 0; |
| 2753 | } |
| 2754 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2755 | int |
| 2756 | vhost_user_dump_ifs (vnet_main_t * vnm, vlib_main_t * vm, |
| 2757 | vhost_user_intf_details_t ** out_vuids) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2758 | { |
| 2759 | int rv = 0; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2760 | vhost_user_main_t *vum = &vhost_user_main; |
| 2761 | vhost_user_intf_t *vui; |
| 2762 | vhost_user_intf_details_t *r_vuids = NULL; |
| 2763 | vhost_user_intf_details_t *vuid = NULL; |
| 2764 | u32 *hw_if_indices = 0; |
| 2765 | vnet_hw_interface_t *hi; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2766 | u8 *s = NULL; |
| 2767 | int i; |
| 2768 | |
| 2769 | if (!out_vuids) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2770 | return -1; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2771 | |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2772 | pool_foreach (vui, vum->vhost_user_interfaces, |
| 2773 | vec_add1 (hw_if_indices, vui->hw_if_index); |
| 2774 | ); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2775 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2776 | for (i = 0; i < vec_len (hw_if_indices); i++) |
| 2777 | { |
| 2778 | hi = vnet_get_hw_interface (vnm, hw_if_indices[i]); |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2779 | vui = pool_elt_at_index (vum->vhost_user_interfaces, hi->dev_instance); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2780 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2781 | vec_add2 (r_vuids, vuid, 1); |
| 2782 | vuid->sw_if_index = vui->sw_if_index; |
| 2783 | vuid->virtio_net_hdr_sz = vui->virtio_net_hdr_sz; |
| 2784 | vuid->features = vui->features; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2785 | vuid->num_regions = vui->nregions; |
| 2786 | vuid->sock_errno = vui->sock_errno; |
| 2787 | strncpy ((char *) vuid->sock_filename, (char *) vui->sock_filename, |
| 2788 | ARRAY_LEN (vuid->sock_filename) - 1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2789 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2790 | s = format (s, "%v%c", hi->name, 0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2791 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2792 | strncpy ((char *) vuid->if_name, (char *) s, |
| 2793 | ARRAY_LEN (vuid->if_name) - 1); |
| 2794 | _vec_len (s) = 0; |
| 2795 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2796 | |
| 2797 | vec_free (s); |
| 2798 | vec_free (hw_if_indices); |
| 2799 | |
| 2800 | *out_vuids = r_vuids; |
| 2801 | |
| 2802 | return rv; |
| 2803 | } |
| 2804 | |
| 2805 | clib_error_t * |
| 2806 | show_vhost_user_command_fn (vlib_main_t * vm, |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2807 | unformat_input_t * input, |
| 2808 | vlib_cli_command_t * cmd) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2809 | { |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2810 | clib_error_t *error = 0; |
| 2811 | vnet_main_t *vnm = vnet_get_main (); |
| 2812 | vhost_user_main_t *vum = &vhost_user_main; |
| 2813 | vhost_user_intf_t *vui; |
| 2814 | u32 hw_if_index, *hw_if_indices = 0; |
| 2815 | vnet_hw_interface_t *hi; |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 2816 | vhost_cpu_t *vhc; |
| 2817 | vhost_iface_and_queue_t *vhiq; |
| 2818 | u32 ci; |
| 2819 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2820 | int i, j, q; |
| 2821 | int show_descr = 0; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2822 | struct feat_struct |
| 2823 | { |
| 2824 | u8 bit; |
| 2825 | char *str; |
| 2826 | }; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2827 | struct feat_struct *feat_entry; |
| 2828 | |
| 2829 | static struct feat_struct feat_array[] = { |
| 2830 | #define _(s,b) { .str = #s, .bit = b, }, |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2831 | foreach_virtio_net_feature |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2832 | #undef _ |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2833 | {.str = NULL} |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2834 | }; |
| 2835 | |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 2836 | #define foreach_protocol_feature \ |
| 2837 | _(VHOST_USER_PROTOCOL_F_MQ) \ |
| 2838 | _(VHOST_USER_PROTOCOL_F_LOG_SHMFD) |
| 2839 | |
| 2840 | static struct feat_struct proto_feat_array[] = { |
| 2841 | #define _(s) { .str = #s, .bit = s}, |
| 2842 | foreach_protocol_feature |
| 2843 | #undef _ |
| 2844 | {.str = NULL} |
| 2845 | }; |
| 2846 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2847 | while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) |
| 2848 | { |
| 2849 | if (unformat |
| 2850 | (input, "%U", unformat_vnet_hw_interface, vnm, &hw_if_index)) |
| 2851 | { |
| 2852 | vec_add1 (hw_if_indices, hw_if_index); |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2853 | } |
| 2854 | else if (unformat (input, "descriptors") || unformat (input, "desc")) |
| 2855 | show_descr = 1; |
| 2856 | else |
| 2857 | { |
| 2858 | error = clib_error_return (0, "unknown input `%U'", |
| 2859 | format_unformat_error, input); |
| 2860 | goto done; |
| 2861 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2862 | } |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2863 | if (vec_len (hw_if_indices) == 0) |
| 2864 | { |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2865 | pool_foreach (vui, vum->vhost_user_interfaces, |
| 2866 | vec_add1 (hw_if_indices, vui->hw_if_index); |
| 2867 | ); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2868 | } |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2869 | vlib_cli_output (vm, "Virtio vhost-user interfaces"); |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 2870 | vlib_cli_output (vm, "Global:\n coalesce frames %d time %e", |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2871 | vum->coalesce_frames, vum->coalesce_time); |
| 2872 | |
| 2873 | for (i = 0; i < vec_len (hw_if_indices); i++) |
| 2874 | { |
| 2875 | hi = vnet_get_hw_interface (vnm, hw_if_indices[i]); |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2876 | vui = pool_elt_at_index (vum->vhost_user_interfaces, hi->dev_instance); |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2877 | vlib_cli_output (vm, "Interface: %s (ifindex %d)", |
| 2878 | hi->name, hw_if_indices[i]); |
| 2879 | |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 2880 | vlib_cli_output (vm, "virtio_net_hdr_sz %d\n" |
| 2881 | " features mask (0x%llx): \n" |
| 2882 | " features (0x%llx): \n", |
| 2883 | vui->virtio_net_hdr_sz, vui->feature_mask, |
| 2884 | vui->features); |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2885 | |
| 2886 | feat_entry = (struct feat_struct *) &feat_array; |
| 2887 | while (feat_entry->str) |
| 2888 | { |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 2889 | if (vui->features & (1ULL << feat_entry->bit)) |
| 2890 | vlib_cli_output (vm, " %s (%d)", feat_entry->str, |
| 2891 | feat_entry->bit); |
| 2892 | feat_entry++; |
| 2893 | } |
| 2894 | |
| 2895 | vlib_cli_output (vm, " protocol features (0x%llx)", |
| 2896 | vui->protocol_features); |
| 2897 | feat_entry = (struct feat_struct *) &proto_feat_array; |
| 2898 | while (feat_entry->str) |
| 2899 | { |
| 2900 | if (vui->protocol_features & (1ULL << feat_entry->bit)) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2901 | vlib_cli_output (vm, " %s (%d)", feat_entry->str, |
| 2902 | feat_entry->bit); |
| 2903 | feat_entry++; |
| 2904 | } |
| 2905 | |
| 2906 | vlib_cli_output (vm, "\n"); |
| 2907 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2908 | vlib_cli_output (vm, " socket filename %s type %s errno \"%s\"\n\n", |
| 2909 | vui->sock_filename, |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2910 | (vui->unix_server_index != ~0) ? "server" : "client", |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2911 | strerror (vui->sock_errno)); |
| 2912 | |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 2913 | vlib_cli_output (vm, " rx placement: "); |
| 2914 | vec_foreach (vhc, vum->cpus) |
| 2915 | { |
| 2916 | vec_foreach (vhiq, vhc->rx_queues) |
| 2917 | { |
| 2918 | if (vhiq->vhost_iface_index == vui - vum->vhost_user_interfaces) |
| 2919 | vlib_cli_output (vm, " thread %d on vring %d\n", |
| 2920 | vhc - vum->cpus, VHOST_VRING_IDX_TX (vhiq->qid)); |
| 2921 | } |
| 2922 | } |
| 2923 | |
| 2924 | vlib_cli_output (vm, " tx placement: %s\n", |
| 2925 | vui->use_tx_spinlock ? "spin-lock" : "lock-free"); |
| 2926 | |
| 2927 | vec_foreach_index (ci, vui->per_cpu_tx_qid) |
| 2928 | { |
| 2929 | vlib_cli_output (vm, " thread %d on vring %d\n", ci, |
| 2930 | VHOST_VRING_IDX_RX (vui->per_cpu_tx_qid[ci])); |
| 2931 | } |
| 2932 | |
| 2933 | vlib_cli_output (vm, "\n"); |
| 2934 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2935 | vlib_cli_output (vm, " Memory regions (total %d)\n", vui->nregions); |
| 2936 | |
| 2937 | if (vui->nregions) |
| 2938 | { |
| 2939 | vlib_cli_output (vm, |
| 2940 | " region fd guest_phys_addr memory_size userspace_addr mmap_offset mmap_addr\n"); |
| 2941 | vlib_cli_output (vm, |
| 2942 | " ====== ===== ================== ================== ================== ================== ==================\n"); |
| 2943 | } |
| 2944 | for (j = 0; j < vui->nregions; j++) |
| 2945 | { |
| 2946 | vlib_cli_output (vm, |
| 2947 | " %d %-5d 0x%016lx 0x%016lx 0x%016lx 0x%016lx 0x%016lx\n", |
| 2948 | j, vui->region_mmap_fd[j], |
| 2949 | vui->regions[j].guest_phys_addr, |
| 2950 | vui->regions[j].memory_size, |
| 2951 | vui->regions[j].userspace_addr, |
| 2952 | vui->regions[j].mmap_offset, |
| 2953 | pointer_to_uword (vui->region_mmap_addr[j])); |
| 2954 | } |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 2955 | for (q = 0; q < VHOST_VRING_MAX_N; q++) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2956 | { |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 2957 | if (!vui->vrings[q].started) |
| 2958 | continue; |
| 2959 | |
| 2960 | vlib_cli_output (vm, "\n Virtqueue %d (%s%s)\n", q, |
| 2961 | (q & 1) ? "RX" : "TX", |
| 2962 | vui->vrings[q].enabled ? "" : " disabled"); |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2963 | |
| 2964 | vlib_cli_output (vm, |
| 2965 | " qsz %d last_avail_idx %d last_used_idx %d\n", |
| 2966 | vui->vrings[q].qsz, vui->vrings[q].last_avail_idx, |
| 2967 | vui->vrings[q].last_used_idx); |
| 2968 | |
| 2969 | if (vui->vrings[q].avail && vui->vrings[q].used) |
| 2970 | vlib_cli_output (vm, |
| 2971 | " avail.flags %x avail.idx %d used.flags %x used.idx %d\n", |
| 2972 | vui->vrings[q].avail->flags, |
| 2973 | vui->vrings[q].avail->idx, |
| 2974 | vui->vrings[q].used->flags, |
| 2975 | vui->vrings[q].used->idx); |
| 2976 | |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2977 | int kickfd = UNIX_GET_FD (vui->vrings[q].kickfd_idx); |
| 2978 | int callfd = UNIX_GET_FD (vui->vrings[q].callfd_idx); |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2979 | vlib_cli_output (vm, " kickfd %d callfd %d errfd %d\n", |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 2980 | kickfd, callfd, vui->vrings[q].errfd); |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2981 | |
| 2982 | if (show_descr) |
| 2983 | { |
| 2984 | vlib_cli_output (vm, "\n descriptor table:\n"); |
| 2985 | vlib_cli_output (vm, |
| 2986 | " id addr len flags next user_addr\n"); |
| 2987 | vlib_cli_output (vm, |
| 2988 | " ===== ================== ===== ====== ===== ==================\n"); |
| 2989 | for (j = 0; j < vui->vrings[q].qsz; j++) |
| 2990 | { |
Pierre Pfister | 11f9205 | 2016-09-21 08:08:55 +0100 | [diff] [blame] | 2991 | u32 mem_hint = 0; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 2992 | vlib_cli_output (vm, |
| 2993 | " %-5d 0x%016lx %-5d 0x%04x %-5d 0x%016lx\n", |
| 2994 | j, vui->vrings[q].desc[j].addr, |
| 2995 | vui->vrings[q].desc[j].len, |
| 2996 | vui->vrings[q].desc[j].flags, |
| 2997 | vui->vrings[q].desc[j].next, |
| 2998 | pointer_to_uword (map_guest_mem |
| 2999 | (vui, |
Pierre Pfister | ba1d046 | 2016-07-27 16:38:20 +0100 | [diff] [blame] | 3000 | vui->vrings[q].desc[j]. |
Pierre Pfister | 11f9205 | 2016-09-21 08:08:55 +0100 | [diff] [blame] | 3001 | addr, &mem_hint))); |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 3002 | } |
| 3003 | } |
| 3004 | } |
| 3005 | vlib_cli_output (vm, "\n"); |
| 3006 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 3007 | done: |
| 3008 | vec_free (hw_if_indices); |
| 3009 | return error; |
| 3010 | } |
| 3011 | |
Damjan Marion | 8d281b3 | 2016-08-24 14:32:39 +0200 | [diff] [blame] | 3012 | /* |
| 3013 | * CLI functions |
| 3014 | */ |
| 3015 | |
Billy McFall | a92501a | 2016-11-23 12:45:29 -0500 | [diff] [blame] | 3016 | /*? |
| 3017 | * Create a vHost User interface. Once created, a new virtual interface |
| 3018 | * will exist with the name '<em>VirtualEthernet0/0/x</em>', where '<em>x</em>' |
| 3019 | * is the next free index. |
| 3020 | * |
| 3021 | * There are several parameters associated with a vHost interface: |
| 3022 | * |
| 3023 | * - <b>socket <socket-filename></b> - Name of the linux socket used by QEMU/VM and |
| 3024 | * VPP to manage the vHost interface. If socket does not already exist, VPP will |
| 3025 | * create the socket. |
| 3026 | * |
| 3027 | * - <b>server</b> - Optional flag to indicate that VPP should be the server for the |
| 3028 | * linux socket. If not provided, VPP will be the client. |
| 3029 | * |
| 3030 | * - <b>feature-mask <hex></b> - Optional virtio/vhost feature set negotiated at |
| 3031 | * startup. By default, all supported features will be advertised. Otherwise, |
| 3032 | * provide the set of features desired. |
| 3033 | * - 0x000008000 (15) - VIRTIO_NET_F_MRG_RXBUF |
| 3034 | * - 0x000020000 (17) - VIRTIO_NET_F_CTRL_VQ |
| 3035 | * - 0x000200000 (21) - VIRTIO_NET_F_GUEST_ANNOUNCE |
| 3036 | * - 0x000400000 (22) - VIRTIO_NET_F_MQ |
| 3037 | * - 0x004000000 (26) - VHOST_F_LOG_ALL |
| 3038 | * - 0x008000000 (27) - VIRTIO_F_ANY_LAYOUT |
| 3039 | * - 0x010000000 (28) - VIRTIO_F_INDIRECT_DESC |
| 3040 | * - 0x040000000 (30) - VHOST_USER_F_PROTOCOL_FEATURES |
| 3041 | * - 0x100000000 (32) - VIRTIO_F_VERSION_1 |
| 3042 | * |
| 3043 | * - <b>hwaddr <mac-addr></b> - Optional ethernet address, can be in either |
| 3044 | * X:X:X:X:X:X unix or X.X.X cisco format. |
| 3045 | * |
| 3046 | * - <b>renumber <dev_instance></b> - Optional parameter which allows the instance |
| 3047 | * in the name to be specified. If instance already exists, name will be used |
| 3048 | * anyway and multiple instances will have the same name. Use with caution. |
| 3049 | * |
| 3050 | * @cliexpar |
| 3051 | * Example of how to create a vhost interface with VPP as the client and all features enabled: |
| 3052 | * @cliexstart{create vhost-user socket /tmp/vhost1.sock} |
| 3053 | * VirtualEthernet0/0/0 |
| 3054 | * @cliexend |
| 3055 | * Example of how to create a vhost interface with VPP as the server and with just |
| 3056 | * multiple queues enabled: |
| 3057 | * @cliexstart{create vhost-user socket /tmp/vhost2.sock server feature-mask 0x40400000} |
| 3058 | * VirtualEthernet0/0/1 |
| 3059 | * @cliexend |
| 3060 | * Once the vHost interface is created, enable the interface using: |
| 3061 | * @cliexcmd{set interface state VirtualEthernet0/0/0 up} |
| 3062 | ?*/ |
Damjan Marion | 8d281b3 | 2016-08-24 14:32:39 +0200 | [diff] [blame] | 3063 | /* *INDENT-OFF* */ |
| 3064 | VLIB_CLI_COMMAND (vhost_user_connect_command, static) = { |
| 3065 | .path = "create vhost-user", |
Billy McFall | a92501a | 2016-11-23 12:45:29 -0500 | [diff] [blame] | 3066 | .short_help = "create vhost-user socket <socket-filename> [server] [feature-mask <hex>] [hwaddr <mac-addr>] [renumber <dev_instance>]", |
Damjan Marion | 8d281b3 | 2016-08-24 14:32:39 +0200 | [diff] [blame] | 3067 | .function = vhost_user_connect_command_fn, |
| 3068 | }; |
Billy McFall | a92501a | 2016-11-23 12:45:29 -0500 | [diff] [blame] | 3069 | /* *INDENT-ON* */ |
Damjan Marion | 8d281b3 | 2016-08-24 14:32:39 +0200 | [diff] [blame] | 3070 | |
Billy McFall | a92501a | 2016-11-23 12:45:29 -0500 | [diff] [blame] | 3071 | /*? |
| 3072 | * Delete a vHost User interface using the interface name or the |
| 3073 | * software interface index. Use the '<em>show interfaces</em>' |
| 3074 | * command to determine the software interface index. On deletion, |
| 3075 | * the linux socket will not be deleted. |
| 3076 | * |
| 3077 | * @cliexpar |
| 3078 | * Example of how to delete a vhost interface by name: |
| 3079 | * @cliexcmd{delete vhost-user VirtualEthernet0/0/1} |
| 3080 | * Example of how to delete a vhost interface by software interface index: |
| 3081 | * @cliexcmd{delete vhost-user sw_if_index 1} |
| 3082 | ?*/ |
| 3083 | /* *INDENT-OFF* */ |
Damjan Marion | 8d281b3 | 2016-08-24 14:32:39 +0200 | [diff] [blame] | 3084 | VLIB_CLI_COMMAND (vhost_user_delete_command, static) = { |
| 3085 | .path = "delete vhost-user", |
Billy McFall | a92501a | 2016-11-23 12:45:29 -0500 | [diff] [blame] | 3086 | .short_help = "delete vhost-user {<interface> | sw_if_index <sw_idx>}", |
Damjan Marion | 8d281b3 | 2016-08-24 14:32:39 +0200 | [diff] [blame] | 3087 | .function = vhost_user_delete_command_fn, |
| 3088 | }; |
| 3089 | |
Billy McFall | a92501a | 2016-11-23 12:45:29 -0500 | [diff] [blame] | 3090 | /*? |
| 3091 | * Display the attributes of a single vHost User interface (provide interface |
| 3092 | * name), multiple vHost User interfaces (provide a list of interface names seperated |
| 3093 | * by spaces) or all Vhost User interfaces (omit an interface name to display all |
| 3094 | * vHost interfaces). |
| 3095 | * |
| 3096 | * @cliexpar |
| 3097 | * @parblock |
| 3098 | * Example of how to display a vhost interface: |
| 3099 | * @cliexstart{show vhost-user VirtualEthernet0/0/0} |
| 3100 | * Virtio vhost-user interfaces |
| 3101 | * Global: |
| 3102 | * coalesce frames 32 time 1e-3 |
| 3103 | * Interface: VirtualEthernet0/0/0 (ifindex 1) |
| 3104 | * virtio_net_hdr_sz 12 |
| 3105 | * features mask (0xffffffffffffffff): |
| 3106 | * features (0x50408000): |
| 3107 | * VIRTIO_NET_F_MRG_RXBUF (15) |
| 3108 | * VIRTIO_NET_F_MQ (22) |
| 3109 | * VIRTIO_F_INDIRECT_DESC (28) |
| 3110 | * VHOST_USER_F_PROTOCOL_FEATURES (30) |
| 3111 | * protocol features (0x3) |
| 3112 | * VHOST_USER_PROTOCOL_F_MQ (0) |
| 3113 | * VHOST_USER_PROTOCOL_F_LOG_SHMFD (1) |
| 3114 | * |
| 3115 | * socket filename /tmp/vhost1.sock type client errno "Success" |
| 3116 | * |
| 3117 | * rx placement: |
| 3118 | * thread 1 on vring 1 |
| 3119 | * thread 1 on vring 5 |
| 3120 | * thread 2 on vring 3 |
| 3121 | * thread 2 on vring 7 |
| 3122 | * tx placement: spin-lock |
| 3123 | * thread 0 on vring 0 |
| 3124 | * thread 1 on vring 2 |
| 3125 | * thread 2 on vring 0 |
| 3126 | * |
| 3127 | * Memory regions (total 2) |
| 3128 | * region fd guest_phys_addr memory_size userspace_addr mmap_offset mmap_addr |
| 3129 | * ====== ===== ================== ================== ================== ================== ================== |
| 3130 | * 0 60 0x0000000000000000 0x00000000000a0000 0x00002aaaaac00000 0x0000000000000000 0x00002aab2b400000 |
| 3131 | * 1 61 0x00000000000c0000 0x000000003ff40000 0x00002aaaaacc0000 0x00000000000c0000 0x00002aababcc0000 |
| 3132 | * |
| 3133 | * Virtqueue 0 (TX) |
| 3134 | * qsz 256 last_avail_idx 0 last_used_idx 0 |
| 3135 | * avail.flags 1 avail.idx 128 used.flags 1 used.idx 0 |
| 3136 | * kickfd 62 callfd 64 errfd -1 |
| 3137 | * |
| 3138 | * Virtqueue 1 (RX) |
| 3139 | * qsz 256 last_avail_idx 0 last_used_idx 0 |
| 3140 | * avail.flags 1 avail.idx 0 used.flags 1 used.idx 0 |
| 3141 | * kickfd 65 callfd 66 errfd -1 |
| 3142 | * |
| 3143 | * Virtqueue 2 (TX) |
| 3144 | * qsz 256 last_avail_idx 0 last_used_idx 0 |
| 3145 | * avail.flags 1 avail.idx 128 used.flags 1 used.idx 0 |
| 3146 | * kickfd 63 callfd 70 errfd -1 |
| 3147 | * |
| 3148 | * Virtqueue 3 (RX) |
| 3149 | * qsz 256 last_avail_idx 0 last_used_idx 0 |
| 3150 | * avail.flags 1 avail.idx 0 used.flags 1 used.idx 0 |
| 3151 | * kickfd 72 callfd 74 errfd -1 |
| 3152 | * |
| 3153 | * Virtqueue 4 (TX disabled) |
| 3154 | * qsz 256 last_avail_idx 0 last_used_idx 0 |
| 3155 | * avail.flags 1 avail.idx 0 used.flags 1 used.idx 0 |
| 3156 | * kickfd 76 callfd 78 errfd -1 |
| 3157 | * |
| 3158 | * Virtqueue 5 (RX disabled) |
| 3159 | * qsz 256 last_avail_idx 0 last_used_idx 0 |
| 3160 | * avail.flags 1 avail.idx 0 used.flags 1 used.idx 0 |
| 3161 | * kickfd 80 callfd 82 errfd -1 |
| 3162 | * |
| 3163 | * Virtqueue 6 (TX disabled) |
| 3164 | * qsz 256 last_avail_idx 0 last_used_idx 0 |
| 3165 | * avail.flags 1 avail.idx 0 used.flags 1 used.idx 0 |
| 3166 | * kickfd 84 callfd 86 errfd -1 |
| 3167 | * |
| 3168 | * Virtqueue 7 (RX disabled) |
| 3169 | * qsz 256 last_avail_idx 0 last_used_idx 0 |
| 3170 | * avail.flags 1 avail.idx 0 used.flags 1 used.idx 0 |
| 3171 | * kickfd 88 callfd 90 errfd -1 |
| 3172 | * |
| 3173 | * @cliexend |
| 3174 | * |
| 3175 | * The optional '<em>descriptors</em>' parameter will display the same output as |
| 3176 | * the previous example but will include the descriptor table for each queue. |
| 3177 | * The output is truncated below: |
| 3178 | * @cliexstart{show vhost-user VirtualEthernet0/0/0 descriptors} |
| 3179 | * Virtio vhost-user interfaces |
| 3180 | * Global: |
| 3181 | * coalesce frames 32 time 1e-3 |
| 3182 | * Interface: VirtualEthernet0/0/0 (ifindex 1) |
| 3183 | * virtio_net_hdr_sz 12 |
| 3184 | * features mask (0xffffffffffffffff): |
| 3185 | * features (0x50408000): |
| 3186 | * VIRTIO_NET_F_MRG_RXBUF (15) |
| 3187 | * VIRTIO_NET_F_MQ (22) |
| 3188 | * : |
| 3189 | * Virtqueue 0 (TX) |
| 3190 | * qsz 256 last_avail_idx 0 last_used_idx 0 |
| 3191 | * avail.flags 1 avail.idx 128 used.flags 1 used.idx 0 |
| 3192 | * kickfd 62 callfd 64 errfd -1 |
| 3193 | * |
| 3194 | * descriptor table: |
| 3195 | * id addr len flags next user_addr |
| 3196 | * ===== ================== ===== ====== ===== ================== |
| 3197 | * 0 0x0000000010b6e974 2060 0x0002 1 0x00002aabbc76e974 |
| 3198 | * 1 0x0000000010b6e034 2060 0x0002 2 0x00002aabbc76e034 |
| 3199 | * 2 0x0000000010b6d6f4 2060 0x0002 3 0x00002aabbc76d6f4 |
| 3200 | * 3 0x0000000010b6cdb4 2060 0x0002 4 0x00002aabbc76cdb4 |
| 3201 | * 4 0x0000000010b6c474 2060 0x0002 5 0x00002aabbc76c474 |
| 3202 | * 5 0x0000000010b6bb34 2060 0x0002 6 0x00002aabbc76bb34 |
| 3203 | * 6 0x0000000010b6b1f4 2060 0x0002 7 0x00002aabbc76b1f4 |
| 3204 | * 7 0x0000000010b6a8b4 2060 0x0002 8 0x00002aabbc76a8b4 |
| 3205 | * 8 0x0000000010b69f74 2060 0x0002 9 0x00002aabbc769f74 |
| 3206 | * 9 0x0000000010b69634 2060 0x0002 10 0x00002aabbc769634 |
| 3207 | * 10 0x0000000010b68cf4 2060 0x0002 11 0x00002aabbc768cf4 |
| 3208 | * : |
| 3209 | * 249 0x0000000000000000 0 0x0000 250 0x00002aab2b400000 |
| 3210 | * 250 0x0000000000000000 0 0x0000 251 0x00002aab2b400000 |
| 3211 | * 251 0x0000000000000000 0 0x0000 252 0x00002aab2b400000 |
| 3212 | * 252 0x0000000000000000 0 0x0000 253 0x00002aab2b400000 |
| 3213 | * 253 0x0000000000000000 0 0x0000 254 0x00002aab2b400000 |
| 3214 | * 254 0x0000000000000000 0 0x0000 255 0x00002aab2b400000 |
| 3215 | * 255 0x0000000000000000 0 0x0000 32768 0x00002aab2b400000 |
| 3216 | * |
| 3217 | * Virtqueue 1 (RX) |
| 3218 | * qsz 256 last_avail_idx 0 last_used_idx 0 |
| 3219 | * : |
| 3220 | * @cliexend |
| 3221 | * @endparblock |
| 3222 | ?*/ |
| 3223 | /* *INDENT-OFF* */ |
Damjan Marion | 8d281b3 | 2016-08-24 14:32:39 +0200 | [diff] [blame] | 3224 | VLIB_CLI_COMMAND (show_vhost_user_command, static) = { |
| 3225 | .path = "show vhost-user", |
Billy McFall | a92501a | 2016-11-23 12:45:29 -0500 | [diff] [blame] | 3226 | .short_help = "show vhost-user [<interface> [<interface> [..]]] [descriptors]", |
Damjan Marion | 8d281b3 | 2016-08-24 14:32:39 +0200 | [diff] [blame] | 3227 | .function = show_vhost_user_command_fn, |
| 3228 | }; |
| 3229 | /* *INDENT-ON* */ |
Damjan Marion | 8d281b3 | 2016-08-24 14:32:39 +0200 | [diff] [blame] | 3230 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 3231 | static clib_error_t * |
| 3232 | vhost_user_config (vlib_main_t * vm, unformat_input_t * input) |
| 3233 | { |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 3234 | vhost_user_main_t *vum = &vhost_user_main; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 3235 | |
| 3236 | while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) |
| 3237 | { |
| 3238 | if (unformat (input, "coalesce-frames %d", &vum->coalesce_frames)) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 3239 | ; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 3240 | else if (unformat (input, "coalesce-time %f", &vum->coalesce_time)) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 3241 | ; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 3242 | else if (unformat (input, "dont-dump-memory")) |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 3243 | vum->dont_dump_vhost_user_memory = 1; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 3244 | else |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 3245 | return clib_error_return (0, "unknown input `%U'", |
| 3246 | format_unformat_error, input); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 3247 | } |
| 3248 | |
| 3249 | return 0; |
| 3250 | } |
| 3251 | |
| 3252 | /* vhost-user { ... } configuration. */ |
| 3253 | VLIB_CONFIG_FUNCTION (vhost_user_config, "vhost-user"); |
| 3254 | |
| 3255 | void |
| 3256 | vhost_user_unmap_all (void) |
| 3257 | { |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 3258 | vhost_user_main_t *vum = &vhost_user_main; |
| 3259 | vhost_user_intf_t *vui; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 3260 | |
| 3261 | if (vum->dont_dump_vhost_user_memory) |
| 3262 | { |
Pierre Pfister | dbb3c25 | 2016-11-22 10:33:34 +0000 | [diff] [blame] | 3263 | pool_foreach (vui, vum->vhost_user_interfaces, |
| 3264 | unmap_all_mem_regions (vui); |
| 3265 | ); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 3266 | } |
| 3267 | } |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 3268 | |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 3269 | static clib_error_t * |
| 3270 | vhost_thread_command_fn (vlib_main_t * vm, |
| 3271 | unformat_input_t * input, vlib_cli_command_t * cmd) |
| 3272 | { |
| 3273 | unformat_input_t _line_input, *line_input = &_line_input; |
| 3274 | u32 worker_thread_index; |
| 3275 | u32 sw_if_index; |
| 3276 | u8 del = 0; |
| 3277 | int rv; |
| 3278 | |
| 3279 | /* Get a line of input. */ |
| 3280 | if (!unformat_user (input, unformat_line_input, line_input)) |
| 3281 | return 0; |
| 3282 | |
| 3283 | if (!unformat |
| 3284 | (line_input, "%U %d", unformat_vnet_sw_interface, vnet_get_main (), |
| 3285 | &sw_if_index, &worker_thread_index)) |
| 3286 | { |
| 3287 | unformat_free (line_input); |
| 3288 | return clib_error_return (0, "unknown input `%U'", |
| 3289 | format_unformat_error, input); |
| 3290 | } |
| 3291 | |
| 3292 | if (unformat (line_input, "del")) |
| 3293 | del = 1; |
| 3294 | |
| 3295 | if ((rv = |
| 3296 | vhost_user_thread_placement (sw_if_index, worker_thread_index, del))) |
| 3297 | return clib_error_return (0, "vhost_user_thread_placement returned %d", |
| 3298 | rv); |
| 3299 | return 0; |
| 3300 | } |
| 3301 | |
| 3302 | |
Billy McFall | a92501a | 2016-11-23 12:45:29 -0500 | [diff] [blame] | 3303 | /*? |
| 3304 | * This command is used to move the RX processing for the given |
| 3305 | * interfaces to the provided thread. If the '<em>del</em>' option is used, |
| 3306 | * the forced thread assignment is removed and the thread assigment is |
| 3307 | * reassigned automatically. Use '<em>show vhost-user <interface></em>' |
| 3308 | * to see the thread assignment. |
| 3309 | * |
| 3310 | * @cliexpar |
| 3311 | * Example of how to move the RX processing for a given interface to a given thread: |
| 3312 | * @cliexcmd{vhost thread VirtualEthernet0/0/0 1} |
| 3313 | * Example of how to remove the forced thread assignment for a given interface: |
| 3314 | * @cliexcmd{vhost thread VirtualEthernet0/0/0 1 del} |
| 3315 | ?*/ |
Pierre Pfister | e21c528 | 2016-09-21 08:04:59 +0100 | [diff] [blame] | 3316 | /* *INDENT-OFF* */ |
| 3317 | VLIB_CLI_COMMAND (vhost_user_thread_command, static) = { |
| 3318 | .path = "vhost thread", |
| 3319 | .short_help = "vhost thread <iface> <worker-index> [del]", |
| 3320 | .function = vhost_thread_command_fn, |
| 3321 | }; |
| 3322 | /* *INDENT-ON* */ |
| 3323 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 3324 | /* |
| 3325 | * fd.io coding-style-patch-verification: ON |
| 3326 | * |
| 3327 | * Local Variables: |
| 3328 | * eval: (c-set-style "gnu") |
| 3329 | * End: |
| 3330 | */ |