blob: 56b65477b193ae2897376afe7693f8b364852022 [file] [log] [blame]
Ed Warnickecb9cada2015-12-08 15:45:58 -07001/*
2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15#ifndef __VIRTIO_VHOST_USER_H__
16#define __VIRTIO_VHOST_USER_H__
17/* vhost-user data structures */
18
19#define VHOST_MEMORY_MAX_NREGIONS 8
20#define VHOST_USER_MSG_HDR_SZ 12
21#define VHOST_VRING_MAX_SIZE 32768
Pierre Pfistere21c5282016-09-21 08:04:59 +010022#define VHOST_VRING_MAX_N 16 //8TX + 8RX
23#define VHOST_VRING_IDX_RX(qid) (2*qid)
24#define VHOST_VRING_IDX_TX(qid) (2*qid + 1)
Ed Warnickecb9cada2015-12-08 15:45:58 -070025
26#define VIRTQ_DESC_F_NEXT 1
Pierre Pfisterba1d0462016-07-27 16:38:20 +010027#define VIRTQ_DESC_F_INDIRECT 4
Shesha Sreenivasamurthyb8f45b32016-02-03 09:38:36 -080028#define VHOST_USER_REPLY_MASK (0x1 << 2)
29
Yoann Desmouceaux4667c222016-02-24 22:51:00 +010030#define VHOST_USER_PROTOCOL_F_MQ 0
31#define VHOST_USER_PROTOCOL_F_LOG_SHMFD 1
Yoann Desmouceauxfe2da0e2016-03-08 14:54:28 +010032#define VHOST_VRING_F_LOG 0
Yoann Desmouceaux4667c222016-02-24 22:51:00 +010033
Shesha Sreenivasamurthy9f50b0b2016-02-23 13:52:31 -080034#define VHOST_USER_F_PROTOCOL_FEATURES 30
Shesha Sreenivasamurthy0666dc42016-04-18 16:38:20 -040035#define VHOST_USER_PROTOCOL_FEATURES ((1ULL << VHOST_USER_PROTOCOL_F_MQ) | \
36 (1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD))
Shesha Sreenivasamurthyb8f45b32016-02-03 09:38:36 -080037
38/* If multiqueue is provided by host, then we suppport it. */
39#define VIRTIO_NET_CTRL_MQ 4
40#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0
41#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1
42#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000
Ed Warnickecb9cada2015-12-08 15:45:58 -070043
Pierre Pfistere21c5282016-09-21 08:04:59 +010044#define VRING_USED_F_NO_NOTIFY 1
Stevena1a09012017-03-08 00:23:13 -080045#define VRING_AVAIL_F_NO_INTERRUPT 1
Pierre Pfistere21c5282016-09-21 08:04:59 +010046
Ed Warnickecb9cada2015-12-08 15:45:58 -070047#define foreach_virtio_net_feature \
48 _ (VIRTIO_NET_F_MRG_RXBUF, 15) \
Pierre Pfistere21c5282016-09-21 08:04:59 +010049 _ (VIRTIO_NET_F_CTRL_VQ, 17) \
50 _ (VIRTIO_NET_F_GUEST_ANNOUNCE, 21) \
51 _ (VIRTIO_NET_F_MQ, 22) \
52 _ (VHOST_F_LOG_ALL, 26) \
Yoann Desmouceaux4667c222016-02-24 22:51:00 +010053 _ (VIRTIO_F_ANY_LAYOUT, 27) \
Pierre Pfisterba1d0462016-07-27 16:38:20 +010054 _ (VIRTIO_F_INDIRECT_DESC, 28) \
Damjan Marione83b7d52016-09-19 14:31:34 +020055 _ (VHOST_USER_F_PROTOCOL_FEATURES, 30) \
56 _ (VIRTIO_F_VERSION_1, 32)
Yoann Desmouceaux4667c222016-02-24 22:51:00 +010057
Ed Warnickecb9cada2015-12-08 15:45:58 -070058
Damjan Marion00a9dca2016-08-17 17:05:46 +020059typedef enum
60{
Ed Warnickecb9cada2015-12-08 15:45:58 -070061#define _(f,n) FEAT_##f = (n),
62 foreach_virtio_net_feature
63#undef _
64} virtio_net_feature_t;
65
Damjan Marion00a9dca2016-08-17 17:05:46 +020066int vhost_user_create_if (vnet_main_t * vnm, vlib_main_t * vm,
67 const char *sock_filename, u8 is_server,
68 u32 * sw_if_index, u64 feature_mask,
Stevena1a09012017-03-08 00:23:13 -080069 u8 renumber, u32 custom_dev_instance, u8 * hwaddr,
70 u8 operation_mode);
Damjan Marion00a9dca2016-08-17 17:05:46 +020071int vhost_user_modify_if (vnet_main_t * vnm, vlib_main_t * vm,
72 const char *sock_filename, u8 is_server,
73 u32 sw_if_index, u64 feature_mask,
Stevena1a09012017-03-08 00:23:13 -080074 u8 renumber, u32 custom_dev_instance,
75 u8 operation_mode);
Damjan Marion00a9dca2016-08-17 17:05:46 +020076int vhost_user_delete_if (vnet_main_t * vnm, vlib_main_t * vm,
77 u32 sw_if_index);
Ed Warnickecb9cada2015-12-08 15:45:58 -070078
Pierre Pfistere21c5282016-09-21 08:04:59 +010079/* *INDENT-OFF* */
Damjan Marion00a9dca2016-08-17 17:05:46 +020080typedef struct vhost_user_memory_region
81{
Ed Warnickecb9cada2015-12-08 15:45:58 -070082 u64 guest_phys_addr;
83 u64 memory_size;
84 u64 userspace_addr;
85 u64 mmap_offset;
Pierre Pfistere21c5282016-09-21 08:04:59 +010086} __attribute ((packed)) vhost_user_memory_region_t;
Ed Warnickecb9cada2015-12-08 15:45:58 -070087
Damjan Marion00a9dca2016-08-17 17:05:46 +020088typedef struct vhost_user_memory
89{
Ed Warnickecb9cada2015-12-08 15:45:58 -070090 u32 nregions;
91 u32 padding;
92 vhost_user_memory_region_t regions[VHOST_MEMORY_MAX_NREGIONS];
Pierre Pfistere21c5282016-09-21 08:04:59 +010093} __attribute ((packed)) vhost_user_memory_t;
Ed Warnickecb9cada2015-12-08 15:45:58 -070094
Damjan Marion00a9dca2016-08-17 17:05:46 +020095typedef struct
96{
Pierre Pfistere21c5282016-09-21 08:04:59 +010097 u32 index, num;
98} __attribute ((packed)) vhost_vring_state_t;
Ed Warnickecb9cada2015-12-08 15:45:58 -070099
Damjan Marion00a9dca2016-08-17 17:05:46 +0200100typedef struct
101{
Pierre Pfistere21c5282016-09-21 08:04:59 +0100102 u32 index, flags;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700103 u64 desc_user_addr, used_user_addr, avail_user_addr, log_guest_addr;
Pierre Pfistere21c5282016-09-21 08:04:59 +0100104} __attribute ((packed)) vhost_vring_addr_t;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700105
Damjan Marion00a9dca2016-08-17 17:05:46 +0200106typedef struct vhost_user_log
107{
Yoann Desmouceaux4667c222016-02-24 22:51:00 +0100108 u64 size;
109 u64 offset;
Pierre Pfistere21c5282016-09-21 08:04:59 +0100110} __attribute ((packed)) vhost_user_log_t;
Yoann Desmouceaux4667c222016-02-24 22:51:00 +0100111
Damjan Marion00a9dca2016-08-17 17:05:46 +0200112typedef enum vhost_user_req
113{
Ed Warnickecb9cada2015-12-08 15:45:58 -0700114 VHOST_USER_NONE = 0,
115 VHOST_USER_GET_FEATURES = 1,
116 VHOST_USER_SET_FEATURES = 2,
117 VHOST_USER_SET_OWNER = 3,
118 VHOST_USER_RESET_OWNER = 4,
119 VHOST_USER_SET_MEM_TABLE = 5,
120 VHOST_USER_SET_LOG_BASE = 6,
121 VHOST_USER_SET_LOG_FD = 7,
122 VHOST_USER_SET_VRING_NUM = 8,
123 VHOST_USER_SET_VRING_ADDR = 9,
124 VHOST_USER_SET_VRING_BASE = 10,
125 VHOST_USER_GET_VRING_BASE = 11,
126 VHOST_USER_SET_VRING_KICK = 12,
127 VHOST_USER_SET_VRING_CALL = 13,
128 VHOST_USER_SET_VRING_ERR = 14,
Shesha Sreenivasamurthyb8f45b32016-02-03 09:38:36 -0800129 VHOST_USER_GET_PROTOCOL_FEATURES = 15,
130 VHOST_USER_SET_PROTOCOL_FEATURES = 16,
131 VHOST_USER_GET_QUEUE_NUM = 17,
Yoann Desmouceaux4667c222016-02-24 22:51:00 +0100132 VHOST_USER_SET_VRING_ENABLE = 18,
Ed Warnickecb9cada2015-12-08 15:45:58 -0700133 VHOST_USER_MAX
134} vhost_user_req_t;
135
136// vring_desc I/O buffer descriptor
Damjan Marion00a9dca2016-08-17 17:05:46 +0200137typedef struct
138{
Ed Warnickecb9cada2015-12-08 15:45:58 -0700139 uint64_t addr; // packet data buffer address
140 uint32_t len; // packet data buffer size
141 uint16_t flags; // (see below)
142 uint16_t next; // optional index next descriptor in chain
143} __attribute ((packed)) vring_desc_t;
144
Damjan Marion00a9dca2016-08-17 17:05:46 +0200145typedef struct
146{
Ed Warnickecb9cada2015-12-08 15:45:58 -0700147 uint16_t flags;
Pierre Pfistere21c5282016-09-21 08:04:59 +0100148 volatile uint16_t idx;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700149 uint16_t ring[VHOST_VRING_MAX_SIZE];
150} __attribute ((packed)) vring_avail_t;
151
Damjan Marion00a9dca2016-08-17 17:05:46 +0200152typedef struct
153{
Ed Warnickecb9cada2015-12-08 15:45:58 -0700154 uint16_t flags;
155 uint16_t idx;
Damjan Marion00a9dca2016-08-17 17:05:46 +0200156 struct /* vring_used_elem */
157 {
158 uint32_t id;
159 uint32_t len;
160 } ring[VHOST_VRING_MAX_SIZE];
Ed Warnickecb9cada2015-12-08 15:45:58 -0700161} __attribute ((packed)) vring_used_t;
162
Damjan Marion00a9dca2016-08-17 17:05:46 +0200163typedef struct
164{
Ed Warnickecb9cada2015-12-08 15:45:58 -0700165 u8 flags;
166 u8 gso_type;
167 u16 hdr_len;
168 u16 gso_size;
169 u16 csum_start;
170 u16 csum_offset;
171} __attribute ((packed)) virtio_net_hdr_t;
172
173typedef struct {
174 virtio_net_hdr_t hdr;
175 u16 num_buffers;
176} __attribute ((packed)) virtio_net_hdr_mrg_rxbuf_t;
177
178typedef struct vhost_user_msg {
Damjan Marion00a9dca2016-08-17 17:05:46 +0200179 vhost_user_req_t request;
180 u32 flags;
181 u32 size;
182 union
183 {
184 u64 u64;
185 vhost_vring_state_t state;
186 vhost_vring_addr_t addr;
187 vhost_user_memory_t memory;
188 vhost_user_log_t log;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700189 };
190} __attribute ((packed)) vhost_user_msg_t;
Damjan Marion00a9dca2016-08-17 17:05:46 +0200191/* *INDENT-ON* */
Ed Warnickecb9cada2015-12-08 15:45:58 -0700192
Damjan Marion00a9dca2016-08-17 17:05:46 +0200193typedef struct
194{
Pierre Pfistere21c5282016-09-21 08:04:59 +0100195 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
196 u16 qsz;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700197 u16 last_avail_idx;
198 u16 last_used_idx;
Pierre Pfistere21c5282016-09-21 08:04:59 +0100199 u16 n_since_last_int;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700200 vring_desc_t *desc;
201 vring_avail_t *avail;
202 vring_used_t *used;
Pierre Pfistere21c5282016-09-21 08:04:59 +0100203 f64 int_deadline;
204 u8 started;
205 u8 enabled;
206 u8 log_used;
207 //Put non-runtime in a different cache line
208 CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700209 int errfd;
210 u32 callfd_idx;
Pierre Pfistere21c5282016-09-21 08:04:59 +0100211 u32 kickfd_idx;
212 u64 log_guest_addr;
Stevene4dcba82017-04-04 16:56:54 -0700213 u32 interrupt_thread_index;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700214} vhost_user_vring_t;
215
Stevena1a09012017-03-08 00:23:13 -0800216#define VHOST_USER_POLLING_MODE 0
217#define VHOST_USER_INTERRUPT_MODE 1
218#define VHOST_USER_ADAPTIVE_MODE 2
219
Steven7312cc72017-03-15 21:18:55 -0700220#define VHOST_USER_EVENT_START_TIMER 1
221
Damjan Marion00a9dca2016-08-17 17:05:46 +0200222typedef struct
223{
224 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700225 u32 is_up;
226 u32 admin_up;
Pierre Pfisterdbb3c252016-11-22 10:33:34 +0000227 u32 unix_server_index;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700228 u32 unix_file_index;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700229 char sock_filename[256];
230 int sock_errno;
Steven5445f5f2017-04-25 16:16:00 -0700231 uword if_index;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700232 u32 hw_if_index, sw_if_index;
Damjan Marion00a9dca2016-08-17 17:05:46 +0200233
Pierre Pfistere21c5282016-09-21 08:04:59 +0100234 //Feature negotiation
Ed Warnickecb9cada2015-12-08 15:45:58 -0700235 u64 features;
236 u64 feature_mask;
Yoann Desmouceaux4667c222016-02-24 22:51:00 +0100237 u64 protocol_features;
Pierre Pfistere21c5282016-09-21 08:04:59 +0100238
239 //Memory region information
240 u32 nregions;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700241 vhost_user_memory_region_t regions[VHOST_MEMORY_MAX_NREGIONS];
Damjan Marion00a9dca2016-08-17 17:05:46 +0200242 void *region_mmap_addr[VHOST_MEMORY_MAX_NREGIONS];
Damjan Marion37623702016-09-20 11:25:27 +0200243 u64 region_guest_addr_lo[VHOST_MEMORY_MAX_NREGIONS];
244 u64 region_guest_addr_hi[VHOST_MEMORY_MAX_NREGIONS];
Ed Warnickecb9cada2015-12-08 15:45:58 -0700245 u32 region_mmap_fd[VHOST_MEMORY_MAX_NREGIONS];
Pierre Pfistere21c5282016-09-21 08:04:59 +0100246
247 //Virtual rings
248 vhost_user_vring_t vrings[VHOST_VRING_MAX_N];
249 volatile u32 *vring_locks[VHOST_VRING_MAX_N];
250
Ed Warnickecb9cada2015-12-08 15:45:58 -0700251 int virtio_net_hdr_sz;
252 int is_any_layout;
Yoann Desmouceaux4667c222016-02-24 22:51:00 +0100253
Damjan Marion00a9dca2016-08-17 17:05:46 +0200254 void *log_base_addr;
Yoann Desmouceaux4667c222016-02-24 22:51:00 +0100255 u64 log_size;
Pierre Pfistere21c5282016-09-21 08:04:59 +0100256
257 /* Whether to use spinlock or per_cpu_tx_qid assignment */
258 u8 use_tx_spinlock;
259 u16 *per_cpu_tx_qid;
260
261 /* Vector of workers for this interface */
262 u32 *workers;
Stevena1a09012017-03-08 00:23:13 -0800263
264 u8 operation_mode;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700265} vhost_user_intf_t;
266
Damjan Marion00a9dca2016-08-17 17:05:46 +0200267typedef struct
268{
Pierre Pfistere21c5282016-09-21 08:04:59 +0100269 u16 vhost_iface_index;
270 u16 qid;
271} vhost_iface_and_queue_t;
272
273typedef struct
274{
Pierre Pfisterd3eb90e2016-11-29 15:36:14 +0000275 uword dst;
276 uword src;
277 u32 len;
278} vhost_copy_t;
279
280typedef struct
281{
282 u16 qid; /** The interface queue index (Not the virtio vring idx) */
283 u16 device_index; /** The device index */
284 u32 virtio_ring_flags; /** Runtime queue flags **/
285 u16 first_desc_len; /** Length of the first data descriptor **/
286 virtio_net_hdr_mrg_rxbuf_t hdr; /** Virtio header **/
287} vhost_trace_t;
288
289
290#define VHOST_USER_RX_BUFFERS_N (2 * VLIB_FRAME_SIZE + 2)
291#define VHOST_USER_COPY_ARRAY_N (4 * VLIB_FRAME_SIZE)
292
293typedef struct
294{
Pierre Pfistere21c5282016-09-21 08:04:59 +0100295 vhost_iface_and_queue_t *rx_queues;
Pierre Pfisterd3eb90e2016-11-29 15:36:14 +0000296 u32 rx_buffers_len;
297 u32 rx_buffers[VHOST_USER_RX_BUFFERS_N];
298
299 virtio_net_hdr_mrg_rxbuf_t tx_headers[VLIB_FRAME_SIZE];
300 vhost_copy_t copy[VHOST_USER_COPY_ARRAY_N];
301
302 /* This is here so it doesn't end-up
303 * using stack or registers. */
304 vhost_trace_t *current_trace;
Steven7312cc72017-03-15 21:18:55 -0700305
306 /* bitmap of pending rx interfaces */
307 uword *pending_input_bitmap;
308
309 /* The operation mode computed per cpu based on interface setting */
310 u8 operation_mode;
Pierre Pfistere21c5282016-09-21 08:04:59 +0100311} vhost_cpu_t;
312
313typedef struct
314{
Steven5445f5f2017-04-25 16:16:00 -0700315 mhash_t if_index_by_sock_name;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700316 u32 mtu_bytes;
Damjan Marion00a9dca2016-08-17 17:05:46 +0200317 vhost_user_intf_t *vhost_user_interfaces;
Damjan Marion00a9dca2016-08-17 17:05:46 +0200318 u32 *show_dev_instance_by_real_dev_instance;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700319 u32 coalesce_frames;
320 f64 coalesce_time;
321 int dont_dump_vhost_user_memory;
Damjan Marion0dafaa72016-09-20 23:21:02 +0200322
Pierre Pfisterdbb3c252016-11-22 10:33:34 +0000323 /** first cpu index */
Damjan Marion0dafaa72016-09-20 23:21:02 +0200324 u32 input_cpu_first_index;
325
Pierre Pfisterdbb3c252016-11-22 10:33:34 +0000326 /** total cpu count */
Damjan Marion0dafaa72016-09-20 23:21:02 +0200327 u32 input_cpu_count;
Pierre Pfistere21c5282016-09-21 08:04:59 +0100328
329 /** Per-CPU data for vhost-user */
330 vhost_cpu_t *cpus;
Pierre Pfisterdbb3c252016-11-22 10:33:34 +0000331
332 /** Pseudo random iterator */
333 u32 random;
Steven7312cc72017-03-15 21:18:55 -0700334
335 /* Node is in interrupt mode */
336 u8 interrupt_mode;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700337} vhost_user_main_t;
338
Damjan Marion00a9dca2016-08-17 17:05:46 +0200339typedef struct
340{
341 u8 if_name[64];
342 u32 sw_if_index;
343 u32 virtio_net_hdr_sz;
344 u64 features;
345 u8 is_server;
346 u8 sock_filename[256];
347 u32 num_regions;
348 int sock_errno;
Stevena1a09012017-03-08 00:23:13 -0800349 u8 operation_mode;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700350} vhost_user_intf_details_t;
351
Damjan Marion00a9dca2016-08-17 17:05:46 +0200352int vhost_user_dump_ifs (vnet_main_t * vnm, vlib_main_t * vm,
353 vhost_user_intf_details_t ** out_vuids);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700354
Ed Warnickecb9cada2015-12-08 15:45:58 -0700355#endif
Damjan Marion00a9dca2016-08-17 17:05:46 +0200356
357/*
358 * fd.io coding-style-patch-verification: ON
359 *
360 * Local Variables:
361 * eval: (c-set-style "gnu")
362 * End:
363 */