Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 1 | /* |
| 2 | *------------------------------------------------------------------ |
| 3 | * vhost-user-input |
| 4 | * |
| 5 | * Copyright (c) 2014-2018 Cisco and/or its affiliates. |
| 6 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 7 | * you may not use this file except in compliance with the License. |
| 8 | * You may obtain a copy of the License at: |
| 9 | * |
| 10 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | * |
| 12 | * Unless required by applicable law or agreed to in writing, software |
| 13 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 15 | * See the License for the specific language governing permissions and |
| 16 | * limitations under the License. |
| 17 | *------------------------------------------------------------------ |
| 18 | */ |
| 19 | |
| 20 | #include <fcntl.h> /* for open */ |
| 21 | #include <sys/ioctl.h> |
| 22 | #include <sys/socket.h> |
| 23 | #include <sys/un.h> |
| 24 | #include <sys/stat.h> |
| 25 | #include <sys/types.h> |
| 26 | #include <sys/uio.h> /* for iovec */ |
| 27 | #include <netinet/in.h> |
| 28 | #include <sys/vfs.h> |
| 29 | |
| 30 | #include <linux/if_arp.h> |
| 31 | #include <linux/if_tun.h> |
| 32 | |
| 33 | #include <vlib/vlib.h> |
| 34 | #include <vlib/unix/unix.h> |
| 35 | |
| 36 | #include <vnet/ip/ip.h> |
| 37 | |
| 38 | #include <vnet/ethernet/ethernet.h> |
| 39 | #include <vnet/devices/devices.h> |
| 40 | #include <vnet/feature/feature.h> |
| 41 | |
| 42 | #include <vnet/devices/virtio/vhost_user.h> |
| 43 | #include <vnet/devices/virtio/vhost_user_inline.h> |
| 44 | |
| 45 | /* |
| 46 | * When an RX queue is down but active, received packets |
| 47 | * must be discarded. This value controls up to how many |
| 48 | * packets will be discarded during each round. |
| 49 | */ |
| 50 | #define VHOST_USER_DOWN_DISCARD_COUNT 256 |
| 51 | |
| 52 | /* |
| 53 | * When the number of available buffers gets under this threshold, |
| 54 | * RX node will start discarding packets. |
| 55 | */ |
| 56 | #define VHOST_USER_RX_BUFFER_STARVATION 32 |
| 57 | |
| 58 | /* |
| 59 | * On the receive side, the host should free descriptors as soon |
| 60 | * as possible in order to avoid TX drop in the VM. |
| 61 | * This value controls the number of copy operations that are stacked |
| 62 | * before copy is done for all and descriptors are given back to |
| 63 | * the guest. |
| 64 | * The value 64 was obtained by testing (48 and 128 were not as good). |
| 65 | */ |
| 66 | #define VHOST_USER_RX_COPY_THRESHOLD 64 |
| 67 | |
| 68 | vlib_node_registration_t vhost_user_input_node; |
| 69 | |
| 70 | #define foreach_vhost_user_input_func_error \ |
| 71 | _(NO_ERROR, "no error") \ |
| 72 | _(NO_BUFFER, "no available buffer") \ |
| 73 | _(MMAP_FAIL, "mmap failure") \ |
| 74 | _(INDIRECT_OVERFLOW, "indirect descriptor overflows table") \ |
| 75 | _(UNDERSIZED_FRAME, "undersized ethernet frame received (< 14 bytes)") \ |
| 76 | _(FULL_RX_QUEUE, "full rx queue (possible driver tx drop)") |
| 77 | |
| 78 | typedef enum |
| 79 | { |
| 80 | #define _(f,s) VHOST_USER_INPUT_FUNC_ERROR_##f, |
| 81 | foreach_vhost_user_input_func_error |
| 82 | #undef _ |
| 83 | VHOST_USER_INPUT_FUNC_N_ERROR, |
| 84 | } vhost_user_input_func_error_t; |
| 85 | |
| 86 | static __clib_unused char *vhost_user_input_func_error_strings[] = { |
| 87 | #define _(n,s) s, |
| 88 | foreach_vhost_user_input_func_error |
| 89 | #undef _ |
| 90 | }; |
| 91 | |
| 92 | static_always_inline void |
| 93 | vhost_user_rx_trace (vhost_trace_t * t, |
| 94 | vhost_user_intf_t * vui, u16 qid, |
| 95 | vlib_buffer_t * b, vhost_user_vring_t * txvq) |
| 96 | { |
| 97 | vhost_user_main_t *vum = &vhost_user_main; |
| 98 | u32 last_avail_idx = txvq->last_avail_idx; |
| 99 | u32 desc_current = txvq->avail->ring[last_avail_idx & txvq->qsz_mask]; |
| 100 | vring_desc_t *hdr_desc = 0; |
| 101 | virtio_net_hdr_mrg_rxbuf_t *hdr; |
| 102 | u32 hint = 0; |
| 103 | |
Dave Barach | b7b9299 | 2018-10-17 10:38:51 -0400 | [diff] [blame] | 104 | clib_memset (t, 0, sizeof (*t)); |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 105 | t->device_index = vui - vum->vhost_user_interfaces; |
| 106 | t->qid = qid; |
| 107 | |
| 108 | hdr_desc = &txvq->desc[desc_current]; |
| 109 | if (txvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT) |
| 110 | { |
| 111 | t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_INDIRECT; |
| 112 | /* Header is the first here */ |
| 113 | hdr_desc = map_guest_mem (vui, txvq->desc[desc_current].addr, &hint); |
| 114 | } |
| 115 | if (txvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT) |
| 116 | { |
| 117 | t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SIMPLE_CHAINED; |
| 118 | } |
| 119 | if (!(txvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT) && |
| 120 | !(txvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT)) |
| 121 | { |
| 122 | t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_SINGLE_DESC; |
| 123 | } |
| 124 | |
| 125 | t->first_desc_len = hdr_desc ? hdr_desc->len : 0; |
| 126 | |
| 127 | if (!hdr_desc || !(hdr = map_guest_mem (vui, hdr_desc->addr, &hint))) |
| 128 | { |
| 129 | t->virtio_ring_flags |= 1 << VIRTIO_TRACE_F_MAP_ERROR; |
| 130 | } |
| 131 | else |
| 132 | { |
| 133 | u32 len = vui->virtio_net_hdr_sz; |
| 134 | memcpy (&t->hdr, hdr, len > hdr_desc->len ? hdr_desc->len : len); |
| 135 | } |
| 136 | } |
| 137 | |
| 138 | static_always_inline u32 |
| 139 | vhost_user_input_copy (vhost_user_intf_t * vui, vhost_copy_t * cpy, |
| 140 | u16 copy_len, u32 * map_hint) |
| 141 | { |
| 142 | void *src0, *src1, *src2, *src3; |
| 143 | if (PREDICT_TRUE (copy_len >= 4)) |
| 144 | { |
| 145 | if (PREDICT_FALSE (!(src2 = map_guest_mem (vui, cpy[0].src, map_hint)))) |
| 146 | return 1; |
| 147 | if (PREDICT_FALSE (!(src3 = map_guest_mem (vui, cpy[1].src, map_hint)))) |
| 148 | return 1; |
| 149 | |
| 150 | while (PREDICT_TRUE (copy_len >= 4)) |
| 151 | { |
| 152 | src0 = src2; |
| 153 | src1 = src3; |
| 154 | |
| 155 | if (PREDICT_FALSE |
| 156 | (!(src2 = map_guest_mem (vui, cpy[2].src, map_hint)))) |
| 157 | return 1; |
| 158 | if (PREDICT_FALSE |
| 159 | (!(src3 = map_guest_mem (vui, cpy[3].src, map_hint)))) |
| 160 | return 1; |
| 161 | |
| 162 | CLIB_PREFETCH (src2, 64, LOAD); |
| 163 | CLIB_PREFETCH (src3, 64, LOAD); |
| 164 | |
Dave Barach | 178cf49 | 2018-11-13 16:34:13 -0500 | [diff] [blame] | 165 | clib_memcpy_fast ((void *) cpy[0].dst, src0, cpy[0].len); |
| 166 | clib_memcpy_fast ((void *) cpy[1].dst, src1, cpy[1].len); |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 167 | copy_len -= 2; |
| 168 | cpy += 2; |
| 169 | } |
| 170 | } |
| 171 | while (copy_len) |
| 172 | { |
| 173 | if (PREDICT_FALSE (!(src0 = map_guest_mem (vui, cpy->src, map_hint)))) |
| 174 | return 1; |
Dave Barach | 178cf49 | 2018-11-13 16:34:13 -0500 | [diff] [blame] | 175 | clib_memcpy_fast ((void *) cpy->dst, src0, cpy->len); |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 176 | copy_len -= 1; |
| 177 | cpy += 1; |
| 178 | } |
| 179 | return 0; |
| 180 | } |
| 181 | |
| 182 | /** |
| 183 | * Try to discard packets from the tx ring (VPP RX path). |
| 184 | * Returns the number of discarded packets. |
| 185 | */ |
| 186 | static_always_inline u32 |
| 187 | vhost_user_rx_discard_packet (vlib_main_t * vm, |
| 188 | vhost_user_intf_t * vui, |
| 189 | vhost_user_vring_t * txvq, u32 discard_max) |
| 190 | { |
| 191 | /* |
| 192 | * On the RX side, each packet corresponds to one descriptor |
| 193 | * (it is the same whether it is a shallow descriptor, chained, or indirect). |
| 194 | * Therefore, discarding a packet is like discarding a descriptor. |
| 195 | */ |
| 196 | u32 discarded_packets = 0; |
| 197 | u32 avail_idx = txvq->avail->idx; |
| 198 | while (discarded_packets != discard_max) |
| 199 | { |
| 200 | if (avail_idx == txvq->last_avail_idx) |
| 201 | goto out; |
| 202 | |
| 203 | u16 desc_chain_head = |
| 204 | txvq->avail->ring[txvq->last_avail_idx & txvq->qsz_mask]; |
| 205 | txvq->last_avail_idx++; |
| 206 | txvq->used->ring[txvq->last_used_idx & txvq->qsz_mask].id = |
| 207 | desc_chain_head; |
| 208 | txvq->used->ring[txvq->last_used_idx & txvq->qsz_mask].len = 0; |
| 209 | vhost_user_log_dirty_ring (vui, txvq, |
| 210 | ring[txvq->last_used_idx & txvq->qsz_mask]); |
| 211 | txvq->last_used_idx++; |
| 212 | discarded_packets++; |
| 213 | } |
| 214 | |
| 215 | out: |
| 216 | CLIB_MEMORY_BARRIER (); |
| 217 | txvq->used->idx = txvq->last_used_idx; |
| 218 | vhost_user_log_dirty_ring (vui, txvq, idx); |
| 219 | return discarded_packets; |
| 220 | } |
| 221 | |
| 222 | /* |
| 223 | * In case of overflow, we need to rewind the array of allocated buffers. |
| 224 | */ |
| 225 | static __clib_unused void |
| 226 | vhost_user_input_rewind_buffers (vlib_main_t * vm, |
| 227 | vhost_cpu_t * cpu, vlib_buffer_t * b_head) |
| 228 | { |
| 229 | u32 bi_current = cpu->rx_buffers[cpu->rx_buffers_len]; |
| 230 | vlib_buffer_t *b_current = vlib_get_buffer (vm, bi_current); |
| 231 | b_current->current_length = 0; |
| 232 | b_current->flags = 0; |
| 233 | while (b_current != b_head) |
| 234 | { |
| 235 | cpu->rx_buffers_len++; |
| 236 | bi_current = cpu->rx_buffers[cpu->rx_buffers_len]; |
| 237 | b_current = vlib_get_buffer (vm, bi_current); |
| 238 | b_current->current_length = 0; |
| 239 | b_current->flags = 0; |
| 240 | } |
| 241 | cpu->rx_buffers_len++; |
| 242 | } |
| 243 | |
| 244 | static __clib_unused u32 |
| 245 | vhost_user_if_input (vlib_main_t * vm, |
| 246 | vhost_user_main_t * vum, |
| 247 | vhost_user_intf_t * vui, |
| 248 | u16 qid, vlib_node_runtime_t * node, |
| 249 | vnet_hw_interface_rx_mode mode) |
| 250 | { |
| 251 | vhost_user_vring_t *txvq = &vui->vrings[VHOST_VRING_IDX_TX (qid)]; |
Damjan Marion | 9af4504 | 2018-11-21 09:51:42 +0100 | [diff] [blame] | 252 | vnet_feature_main_t *fm = &feature_main; |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 253 | u16 n_rx_packets = 0; |
| 254 | u32 n_rx_bytes = 0; |
| 255 | u16 n_left; |
| 256 | u32 n_left_to_next, *to_next; |
| 257 | u32 next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT; |
| 258 | u32 n_trace = vlib_get_trace_count (vm, node); |
| 259 | u32 map_hint = 0; |
Damjan Marion | 7e0b17d | 2018-11-20 21:07:03 +0100 | [diff] [blame] | 260 | vhost_cpu_t *cpu = &vum->cpus[vm->thread_index]; |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 261 | u16 copy_len = 0; |
Damjan Marion | 9af4504 | 2018-11-21 09:51:42 +0100 | [diff] [blame] | 262 | u8 feature_arc_idx = fm->device_input_feature_arc_index; |
| 263 | u32 current_config_index = ~(u32) 0; |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 264 | |
Yichen Wang | 28812a0 | 2018-08-28 23:05:27 -0700 | [diff] [blame] | 265 | /* The descriptor table is not ready yet */ |
| 266 | if (PREDICT_FALSE (txvq->avail == 0)) |
| 267 | return 0; |
| 268 | |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 269 | { |
| 270 | /* do we have pending interrupts ? */ |
| 271 | vhost_user_vring_t *rxvq = &vui->vrings[VHOST_VRING_IDX_RX (qid)]; |
| 272 | f64 now = vlib_time_now (vm); |
| 273 | |
| 274 | if ((txvq->n_since_last_int) && (txvq->int_deadline < now)) |
| 275 | vhost_user_send_call (vm, txvq); |
| 276 | |
| 277 | if ((rxvq->n_since_last_int) && (rxvq->int_deadline < now)) |
| 278 | vhost_user_send_call (vm, rxvq); |
| 279 | } |
| 280 | |
| 281 | /* |
| 282 | * For adaptive mode, it is optimized to reduce interrupts. |
| 283 | * If the scheduler switches the input node to polling due |
| 284 | * to burst of traffic, we tell the driver no interrupt. |
| 285 | * When the traffic subsides, the scheduler switches the node back to |
| 286 | * interrupt mode. We must tell the driver we want interrupt. |
| 287 | */ |
| 288 | if (PREDICT_FALSE (mode == VNET_HW_INTERFACE_RX_MODE_ADAPTIVE)) |
| 289 | { |
| 290 | if ((node->flags & |
| 291 | VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE) || |
| 292 | !(node->flags & |
| 293 | VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE)) |
| 294 | /* Tell driver we want notification */ |
| 295 | txvq->used->flags = 0; |
| 296 | else |
| 297 | /* Tell driver we don't want notification */ |
| 298 | txvq->used->flags = VRING_USED_F_NO_NOTIFY; |
| 299 | } |
| 300 | |
| 301 | if (PREDICT_FALSE (txvq->avail->flags & 0xFFFE)) |
| 302 | return 0; |
| 303 | |
| 304 | n_left = (u16) (txvq->avail->idx - txvq->last_avail_idx); |
| 305 | |
| 306 | /* nothing to do */ |
| 307 | if (PREDICT_FALSE (n_left == 0)) |
| 308 | return 0; |
| 309 | |
| 310 | if (PREDICT_FALSE (!vui->admin_up || !(txvq->enabled))) |
| 311 | { |
| 312 | /* |
| 313 | * Discard input packet if interface is admin down or vring is not |
| 314 | * enabled. |
| 315 | * "For example, for a networking device, in the disabled state |
| 316 | * client must not supply any new RX packets, but must process |
| 317 | * and discard any TX packets." |
| 318 | */ |
| 319 | vhost_user_rx_discard_packet (vm, vui, txvq, |
| 320 | VHOST_USER_DOWN_DISCARD_COUNT); |
| 321 | return 0; |
| 322 | } |
| 323 | |
| 324 | if (PREDICT_FALSE (n_left == (txvq->qsz_mask + 1))) |
| 325 | { |
| 326 | /* |
| 327 | * Informational error logging when VPP is not |
| 328 | * receiving packets fast enough. |
| 329 | */ |
| 330 | vlib_error_count (vm, node->node_index, |
| 331 | VHOST_USER_INPUT_FUNC_ERROR_FULL_RX_QUEUE, 1); |
| 332 | } |
| 333 | |
| 334 | if (n_left > VLIB_FRAME_SIZE) |
| 335 | n_left = VLIB_FRAME_SIZE; |
| 336 | |
| 337 | /* |
| 338 | * For small packets (<2kB), we will not need more than one vlib buffer |
| 339 | * per packet. In case packets are bigger, we will just yeld at some point |
| 340 | * in the loop and come back later. This is not an issue as for big packet, |
| 341 | * processing cost really comes from the memory copy. |
| 342 | * The assumption is that big packets will fit in 40 buffers. |
| 343 | */ |
Damjan Marion | 7e0b17d | 2018-11-20 21:07:03 +0100 | [diff] [blame] | 344 | if (PREDICT_FALSE (cpu->rx_buffers_len < n_left + 1 || |
| 345 | cpu->rx_buffers_len < 40)) |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 346 | { |
Damjan Marion | 7e0b17d | 2018-11-20 21:07:03 +0100 | [diff] [blame] | 347 | u32 curr_len = cpu->rx_buffers_len; |
| 348 | cpu->rx_buffers_len += |
| 349 | vlib_buffer_alloc_from_free_list (vm, cpu->rx_buffers + curr_len, |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 350 | VHOST_USER_RX_BUFFERS_N - curr_len, |
| 351 | VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); |
| 352 | |
| 353 | if (PREDICT_FALSE |
Damjan Marion | 7e0b17d | 2018-11-20 21:07:03 +0100 | [diff] [blame] | 354 | (cpu->rx_buffers_len < VHOST_USER_RX_BUFFER_STARVATION)) |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 355 | { |
| 356 | /* In case of buffer starvation, discard some packets from the queue |
| 357 | * and log the event. |
| 358 | * We keep doing best effort for the remaining packets. */ |
Damjan Marion | 7e0b17d | 2018-11-20 21:07:03 +0100 | [diff] [blame] | 359 | u32 flush = (n_left + 1 > cpu->rx_buffers_len) ? |
| 360 | n_left + 1 - cpu->rx_buffers_len : 1; |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 361 | flush = vhost_user_rx_discard_packet (vm, vui, txvq, flush); |
| 362 | |
| 363 | n_left -= flush; |
| 364 | vlib_increment_simple_counter (vnet_main. |
| 365 | interface_main.sw_if_counters + |
| 366 | VNET_INTERFACE_COUNTER_DROP, |
Damjan Marion | 7e0b17d | 2018-11-20 21:07:03 +0100 | [diff] [blame] | 367 | vm->thread_index, vui->sw_if_index, |
| 368 | flush); |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 369 | |
| 370 | vlib_error_count (vm, vhost_user_input_node.index, |
| 371 | VHOST_USER_INPUT_FUNC_ERROR_NO_BUFFER, flush); |
| 372 | } |
| 373 | } |
| 374 | |
Damjan Marion | 9af4504 | 2018-11-21 09:51:42 +0100 | [diff] [blame] | 375 | if (PREDICT_FALSE (vnet_have_features (feature_arc_idx, vui->sw_if_index))) |
| 376 | { |
| 377 | vnet_feature_config_main_t *cm; |
| 378 | cm = &fm->feature_config_mains[feature_arc_idx]; |
| 379 | current_config_index = vec_elt (cm->config_index_by_sw_if_index, |
| 380 | vui->sw_if_index); |
| 381 | vnet_get_config_data (&cm->config_main, ¤t_config_index, |
| 382 | &next_index, 0); |
| 383 | } |
| 384 | |
Damjan Marion | 9282538 | 2018-11-21 10:03:44 +0100 | [diff] [blame^] | 385 | vlib_get_new_next_frame (vm, node, next_index, to_next, n_left_to_next); |
| 386 | |
| 387 | if (next_index == VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT) |
| 388 | { |
| 389 | /* give some hints to ethernet-input */ |
| 390 | vlib_next_frame_t *nf; |
| 391 | vlib_frame_t *f; |
| 392 | ethernet_input_frame_t *ef; |
| 393 | nf = vlib_node_runtime_get_next_frame (vm, node, next_index); |
| 394 | f = vlib_get_frame (vm, nf->frame_index); |
| 395 | f->flags = ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX; |
| 396 | |
| 397 | ef = vlib_frame_scalar_args (f); |
| 398 | ef->sw_if_index = vui->sw_if_index; |
| 399 | ef->hw_if_index = vui->hw_if_index; |
| 400 | } |
| 401 | |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 402 | while (n_left > 0) |
| 403 | { |
Damjan Marion | 9282538 | 2018-11-21 10:03:44 +0100 | [diff] [blame^] | 404 | vlib_buffer_t *b_head, *b_current; |
| 405 | u32 bi_current; |
| 406 | u16 desc_current; |
| 407 | u32 desc_data_offset; |
| 408 | vring_desc_t *desc_table = txvq->desc; |
Damjan Marion | 6a8bfd4 | 2018-11-21 09:54:41 +0100 | [diff] [blame] | 409 | |
Damjan Marion | 9282538 | 2018-11-21 10:03:44 +0100 | [diff] [blame^] | 410 | if (PREDICT_FALSE (cpu->rx_buffers_len <= 1)) |
Damjan Marion | 6a8bfd4 | 2018-11-21 09:54:41 +0100 | [diff] [blame] | 411 | { |
Damjan Marion | 9282538 | 2018-11-21 10:03:44 +0100 | [diff] [blame^] | 412 | /* Not enough rx_buffers |
| 413 | * Note: We yeld on 1 so we don't need to do an additional |
| 414 | * check for the next buffer prefetch. |
| 415 | */ |
| 416 | n_left = 0; |
| 417 | break; |
Damjan Marion | 6a8bfd4 | 2018-11-21 09:54:41 +0100 | [diff] [blame] | 418 | } |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 419 | |
Damjan Marion | 9282538 | 2018-11-21 10:03:44 +0100 | [diff] [blame^] | 420 | desc_current = txvq->avail->ring[txvq->last_avail_idx & txvq->qsz_mask]; |
| 421 | cpu->rx_buffers_len--; |
| 422 | bi_current = cpu->rx_buffers[cpu->rx_buffers_len]; |
| 423 | b_head = b_current = vlib_get_buffer (vm, bi_current); |
| 424 | to_next[0] = bi_current; //We do that now so we can forget about bi_current |
| 425 | to_next++; |
| 426 | n_left_to_next--; |
| 427 | |
| 428 | vlib_prefetch_buffer_with_index |
| 429 | (vm, cpu->rx_buffers[cpu->rx_buffers_len - 1], LOAD); |
| 430 | |
| 431 | /* Just preset the used descriptor id and length for later */ |
| 432 | txvq->used->ring[txvq->last_used_idx & txvq->qsz_mask].id = |
| 433 | desc_current; |
| 434 | txvq->used->ring[txvq->last_used_idx & txvq->qsz_mask].len = 0; |
| 435 | vhost_user_log_dirty_ring (vui, txvq, |
| 436 | ring[txvq->last_used_idx & txvq->qsz_mask]); |
| 437 | |
| 438 | /* The buffer should already be initialized */ |
| 439 | b_head->total_length_not_including_first_buffer = 0; |
| 440 | b_head->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; |
| 441 | |
| 442 | if (PREDICT_FALSE (n_trace)) |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 443 | { |
Damjan Marion | 9282538 | 2018-11-21 10:03:44 +0100 | [diff] [blame^] | 444 | //TODO: next_index is not exactly known at that point |
| 445 | vlib_trace_buffer (vm, node, next_index, b_head, |
| 446 | /* follow_chain */ 0); |
| 447 | vhost_trace_t *t0 = |
| 448 | vlib_add_trace (vm, node, b_head, sizeof (t0[0])); |
| 449 | vhost_user_rx_trace (t0, vui, qid, b_head, txvq); |
| 450 | n_trace--; |
| 451 | vlib_set_trace_count (vm, node, n_trace); |
| 452 | } |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 453 | |
Damjan Marion | 9282538 | 2018-11-21 10:03:44 +0100 | [diff] [blame^] | 454 | /* This depends on the setup but is very consistent |
| 455 | * So I think the CPU branch predictor will make a pretty good job |
| 456 | * at optimizing the decision. */ |
| 457 | if (txvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT) |
| 458 | { |
| 459 | desc_table = map_guest_mem (vui, txvq->desc[desc_current].addr, |
| 460 | &map_hint); |
| 461 | desc_current = 0; |
| 462 | if (PREDICT_FALSE (desc_table == 0)) |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 463 | { |
Damjan Marion | 9282538 | 2018-11-21 10:03:44 +0100 | [diff] [blame^] | 464 | vlib_error_count (vm, node->node_index, |
| 465 | VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL, 1); |
| 466 | goto out; |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 467 | } |
Damjan Marion | 9282538 | 2018-11-21 10:03:44 +0100 | [diff] [blame^] | 468 | } |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 469 | |
Damjan Marion | 9282538 | 2018-11-21 10:03:44 +0100 | [diff] [blame^] | 470 | if (PREDICT_TRUE (vui->is_any_layout) || |
| 471 | (!(desc_table[desc_current].flags & VIRTQ_DESC_F_NEXT))) |
| 472 | { |
| 473 | /* ANYLAYOUT or single buffer */ |
| 474 | desc_data_offset = vui->virtio_net_hdr_sz; |
| 475 | } |
| 476 | else |
| 477 | { |
| 478 | /* CSR case without ANYLAYOUT, skip 1st buffer */ |
| 479 | desc_data_offset = desc_table[desc_current].len; |
| 480 | } |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 481 | |
Damjan Marion | 9282538 | 2018-11-21 10:03:44 +0100 | [diff] [blame^] | 482 | while (1) |
| 483 | { |
| 484 | /* Get more input if necessary. Or end of packet. */ |
| 485 | if (desc_data_offset == desc_table[desc_current].len) |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 486 | { |
Damjan Marion | 9282538 | 2018-11-21 10:03:44 +0100 | [diff] [blame^] | 487 | if (PREDICT_FALSE (desc_table[desc_current].flags & |
| 488 | VIRTQ_DESC_F_NEXT)) |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 489 | { |
Damjan Marion | 9282538 | 2018-11-21 10:03:44 +0100 | [diff] [blame^] | 490 | desc_current = desc_table[desc_current].next; |
| 491 | desc_data_offset = 0; |
| 492 | } |
| 493 | else |
| 494 | { |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 495 | goto out; |
| 496 | } |
| 497 | } |
| 498 | |
Damjan Marion | 9282538 | 2018-11-21 10:03:44 +0100 | [diff] [blame^] | 499 | /* Get more output if necessary. Or end of packet. */ |
| 500 | if (PREDICT_FALSE |
| 501 | (b_current->current_length == VLIB_BUFFER_DATA_SIZE)) |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 502 | { |
Damjan Marion | 9282538 | 2018-11-21 10:03:44 +0100 | [diff] [blame^] | 503 | if (PREDICT_FALSE (cpu->rx_buffers_len == 0)) |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 504 | { |
Damjan Marion | 9282538 | 2018-11-21 10:03:44 +0100 | [diff] [blame^] | 505 | /* Cancel speculation */ |
| 506 | to_next--; |
| 507 | n_left_to_next++; |
| 508 | |
| 509 | /* |
| 510 | * Checking if there are some left buffers. |
| 511 | * If not, just rewind the used buffers and stop. |
| 512 | * Note: Scheduled copies are not cancelled. This is |
| 513 | * not an issue as they would still be valid. Useless, |
| 514 | * but valid. |
| 515 | */ |
| 516 | vhost_user_input_rewind_buffers (vm, cpu, b_head); |
| 517 | n_left = 0; |
| 518 | goto stop; |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 519 | } |
| 520 | |
Damjan Marion | 9282538 | 2018-11-21 10:03:44 +0100 | [diff] [blame^] | 521 | /* Get next output */ |
| 522 | cpu->rx_buffers_len--; |
| 523 | u32 bi_next = cpu->rx_buffers[cpu->rx_buffers_len]; |
| 524 | b_current->next_buffer = bi_next; |
| 525 | b_current->flags |= VLIB_BUFFER_NEXT_PRESENT; |
| 526 | bi_current = bi_next; |
| 527 | b_current = vlib_get_buffer (vm, bi_current); |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 528 | } |
| 529 | |
Damjan Marion | 9282538 | 2018-11-21 10:03:44 +0100 | [diff] [blame^] | 530 | /* Prepare a copy order executed later for the data */ |
| 531 | vhost_copy_t *cpy = &cpu->copy[copy_len]; |
| 532 | copy_len++; |
| 533 | u32 desc_data_l = desc_table[desc_current].len - desc_data_offset; |
| 534 | cpy->len = VLIB_BUFFER_DATA_SIZE - b_current->current_length; |
| 535 | cpy->len = (cpy->len > desc_data_l) ? desc_data_l : cpy->len; |
| 536 | cpy->dst = (uword) (vlib_buffer_get_current (b_current) + |
| 537 | b_current->current_length); |
| 538 | cpy->src = desc_table[desc_current].addr + desc_data_offset; |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 539 | |
Damjan Marion | 9282538 | 2018-11-21 10:03:44 +0100 | [diff] [blame^] | 540 | desc_data_offset += cpy->len; |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 541 | |
Damjan Marion | 9282538 | 2018-11-21 10:03:44 +0100 | [diff] [blame^] | 542 | b_current->current_length += cpy->len; |
| 543 | b_head->total_length_not_including_first_buffer += cpy->len; |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 544 | } |
Damjan Marion | 9282538 | 2018-11-21 10:03:44 +0100 | [diff] [blame^] | 545 | |
| 546 | out: |
| 547 | |
| 548 | n_rx_bytes += b_head->total_length_not_including_first_buffer; |
| 549 | n_rx_packets++; |
| 550 | |
| 551 | b_head->total_length_not_including_first_buffer -= |
| 552 | b_head->current_length; |
| 553 | |
| 554 | /* consume the descriptor and return it as used */ |
| 555 | txvq->last_avail_idx++; |
| 556 | txvq->last_used_idx++; |
| 557 | |
| 558 | VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b_head); |
| 559 | |
| 560 | vnet_buffer (b_head)->sw_if_index[VLIB_RX] = vui->sw_if_index; |
| 561 | vnet_buffer (b_head)->sw_if_index[VLIB_TX] = (u32) ~ 0; |
| 562 | b_head->error = 0; |
| 563 | |
| 564 | if (current_config_index != ~(u32) 0) |
| 565 | { |
| 566 | b_head->current_config_index = current_config_index; |
| 567 | vnet_buffer (b_head)->feature_arc_index = feature_arc_idx; |
| 568 | } |
| 569 | |
| 570 | n_left--; |
| 571 | |
| 572 | /* |
| 573 | * Although separating memory copies from virtio ring parsing |
| 574 | * is beneficial, we can offer to perform the copies from time |
| 575 | * to time in order to free some space in the ring. |
| 576 | */ |
| 577 | if (PREDICT_FALSE (copy_len >= VHOST_USER_RX_COPY_THRESHOLD)) |
| 578 | { |
| 579 | if (PREDICT_FALSE (vhost_user_input_copy (vui, cpu->copy, |
| 580 | copy_len, &map_hint))) |
| 581 | { |
| 582 | vlib_error_count (vm, node->node_index, |
| 583 | VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL, 1); |
| 584 | } |
| 585 | copy_len = 0; |
| 586 | |
| 587 | /* give buffers back to driver */ |
| 588 | CLIB_MEMORY_BARRIER (); |
| 589 | txvq->used->idx = txvq->last_used_idx; |
| 590 | vhost_user_log_dirty_ring (vui, txvq, idx); |
| 591 | } |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 592 | } |
Damjan Marion | 9282538 | 2018-11-21 10:03:44 +0100 | [diff] [blame^] | 593 | stop: |
| 594 | vlib_put_next_frame (vm, node, next_index, n_left_to_next); |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 595 | |
| 596 | /* Do the memory copies */ |
Damjan Marion | 7e0b17d | 2018-11-20 21:07:03 +0100 | [diff] [blame] | 597 | if (PREDICT_FALSE (vhost_user_input_copy (vui, cpu->copy, copy_len, |
| 598 | &map_hint))) |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 599 | { |
| 600 | vlib_error_count (vm, node->node_index, |
| 601 | VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL, 1); |
| 602 | } |
| 603 | |
| 604 | /* give buffers back to driver */ |
| 605 | CLIB_MEMORY_BARRIER (); |
| 606 | txvq->used->idx = txvq->last_used_idx; |
| 607 | vhost_user_log_dirty_ring (vui, txvq, idx); |
| 608 | |
| 609 | /* interrupt (call) handling */ |
| 610 | if ((txvq->callfd_idx != ~0) && |
| 611 | !(txvq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) |
| 612 | { |
| 613 | txvq->n_since_last_int += n_rx_packets; |
| 614 | |
| 615 | if (txvq->n_since_last_int > vum->coalesce_frames) |
| 616 | vhost_user_send_call (vm, txvq); |
| 617 | } |
| 618 | |
| 619 | /* increase rx counters */ |
| 620 | vlib_increment_combined_counter |
| 621 | (vnet_main.interface_main.combined_sw_if_counters |
Damjan Marion | 7e0b17d | 2018-11-20 21:07:03 +0100 | [diff] [blame] | 622 | + VNET_INTERFACE_COUNTER_RX, vm->thread_index, vui->sw_if_index, |
| 623 | n_rx_packets, n_rx_bytes); |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 624 | |
Damjan Marion | 7e0b17d | 2018-11-20 21:07:03 +0100 | [diff] [blame] | 625 | vnet_device_increment_rx_packets (vm->thread_index, n_rx_packets); |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 626 | |
| 627 | return n_rx_packets; |
| 628 | } |
| 629 | |
| 630 | VLIB_NODE_FN (vhost_user_input_node) (vlib_main_t * vm, |
| 631 | vlib_node_runtime_t * node, |
| 632 | vlib_frame_t * frame) |
| 633 | { |
| 634 | vhost_user_main_t *vum = &vhost_user_main; |
| 635 | uword n_rx_packets = 0; |
| 636 | vhost_user_intf_t *vui; |
| 637 | vnet_device_input_runtime_t *rt = |
| 638 | (vnet_device_input_runtime_t *) node->runtime_data; |
| 639 | vnet_device_and_queue_t *dq; |
| 640 | |
| 641 | vec_foreach (dq, rt->devices_and_queues) |
| 642 | { |
Sirshak Das | 5b718d5 | 2018-10-12 09:38:27 -0500 | [diff] [blame] | 643 | if ((node->state == VLIB_NODE_STATE_POLLING) || |
| 644 | clib_atomic_swap_acq_n (&dq->interrupt_pending, 0)) |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 645 | { |
| 646 | vui = |
| 647 | pool_elt_at_index (vum->vhost_user_interfaces, dq->dev_instance); |
Damjan Marion | befe691 | 2018-11-22 15:45:54 +0100 | [diff] [blame] | 648 | n_rx_packets += vhost_user_if_input (vm, vum, vui, dq->queue_id, node, |
| 649 | dq->mode); |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 650 | } |
| 651 | } |
| 652 | |
| 653 | return n_rx_packets; |
| 654 | } |
| 655 | |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 656 | /* *INDENT-OFF* */ |
| 657 | VLIB_REGISTER_NODE (vhost_user_input_node) = { |
| 658 | .type = VLIB_NODE_TYPE_INPUT, |
| 659 | .name = "vhost-user-input", |
| 660 | .sibling_of = "device-input", |
| 661 | |
| 662 | /* Will be enabled if/when hardware is detected. */ |
| 663 | .state = VLIB_NODE_STATE_DISABLED, |
| 664 | |
| 665 | .format_buffer = format_ethernet_header_with_length, |
| 666 | .format_trace = format_vhost_trace, |
| 667 | |
| 668 | .n_errors = VHOST_USER_INPUT_FUNC_N_ERROR, |
| 669 | .error_strings = vhost_user_input_func_error_strings, |
| 670 | }; |
| 671 | /* *INDENT-ON* */ |
Mohsin Kazmi | e7cde31 | 2018-06-26 17:20:11 +0200 | [diff] [blame] | 672 | |
| 673 | /* |
| 674 | * fd.io coding-style-patch-verification: ON |
| 675 | * |
| 676 | * Local Variables: |
| 677 | * eval: (c-set-style "gnu") |
| 678 | * End: |
| 679 | */ |