blob: a8ab944a5133364d43d3671d4811f389a40fa86c [file] [log] [blame]
Damjan Marion83243a02016-02-29 13:09:30 +01001/*
2 *------------------------------------------------------------------
3 * af_packet.c - linux kernel packet interface
4 *
5 * Copyright (c) 2016 Cisco and/or its affiliates.
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at:
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 *------------------------------------------------------------------
18 */
19
20#include <linux/if_ether.h>
21#include <linux/if_packet.h>
Mohsin Kazmia50a14c2018-04-25 15:58:05 +020022#include <sys/ioctl.h>
23#include <net/if.h>
Ray Kinsellac855b732017-04-21 12:24:43 +010024#include <dirent.h>
Ray Kinsella7bfa1192017-05-15 11:52:43 +010025#include <sys/stat.h>
26#include <sys/types.h>
27#include <fcntl.h>
Damjan Marion83243a02016-02-29 13:09:30 +010028
Damjan Marion01914ce2017-09-14 19:04:50 +020029#include <vppinfra/linux/sysfs.h>
Damjan Marion83243a02016-02-29 13:09:30 +010030#include <vlib/vlib.h>
31#include <vlib/unix/unix.h>
32#include <vnet/ip/ip.h>
Aloys Augustine39376e2021-03-29 22:08:09 +020033#include <vnet/devices/netlink.h>
Damjan Marion83243a02016-02-29 13:09:30 +010034#include <vnet/ethernet/ethernet.h>
Mohammed Hawari85c19432020-12-18 16:29:45 +010035#include <vnet/interface/rx_queue_funcs.h>
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +000036#include <vnet/interface/tx_queue_funcs.h>
Damjan Marion83243a02016-02-29 13:09:30 +010037
38#include <vnet/devices/af_packet/af_packet.h>
39
Dave Wallace71612d62017-10-24 01:32:41 -040040af_packet_main_t af_packet_main;
41
Mohsin Kazmicae84fa2021-10-08 15:10:49 +000042VNET_HW_INTERFACE_CLASS (af_packet_ip_device_hw_interface_class, static) = {
43 .name = "af-packet-ip-device",
44 .flags = VNET_HW_INTERFACE_CLASS_FLAG_P2P,
45};
46
Nathan Skrzypczak7d0e30b2021-06-23 11:28:39 +020047#define AF_PACKET_DEFAULT_TX_FRAMES_PER_BLOCK 1024
Mohsin Kazmic1fd17b2022-03-22 21:40:04 +000048#define AF_PACKET_DEFAULT_TX_FRAME_SIZE (2048 * 33) // GSO packet of 64KB
Damjan Marion83243a02016-02-29 13:09:30 +010049#define AF_PACKET_TX_BLOCK_NR 1
Damjan Marion83243a02016-02-29 13:09:30 +010050
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +000051#define AF_PACKET_DEFAULT_RX_FRAMES_PER_BLOCK 32
Mohsin Kazmi219cbcb2022-03-18 16:58:31 +000052#define AF_PACKET_DEFAULT_RX_FRAME_SIZE 2048
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +000053#define AF_PACKET_RX_BLOCK_NR 160
Damjan Marion83243a02016-02-29 13:09:30 +010054
Damjan Marion83243a02016-02-29 13:09:30 +010055/*defined in net/if.h but clashes with dpdk headers */
Damjan Marion00a9dca2016-08-17 17:05:46 +020056unsigned int if_nametoindex (const char *ifname);
Damjan Marion83243a02016-02-29 13:09:30 +010057
Damjan Marion88a9c0e2022-01-06 21:14:08 +010058static clib_error_t *
Damjan Marion1cd0e5d2022-01-17 14:49:17 +010059af_packet_eth_set_max_frame_size (vnet_main_t *vnm, vnet_hw_interface_t *hi,
60 u32 frame_size)
Damjan Marion83243a02016-02-29 13:09:30 +010061{
Damjan Marion88a9c0e2022-01-06 21:14:08 +010062 clib_error_t *error, *rv;
Ray Kinsella7bfa1192017-05-15 11:52:43 +010063 af_packet_main_t *apm = &af_packet_main;
Damjan Marion88a9c0e2022-01-06 21:14:08 +010064 af_packet_if_t *apif = pool_elt_at_index (apm->interfaces, hi->dev_instance);
Ray Kinsella7bfa1192017-05-15 11:52:43 +010065
Damjan Marion1cd0e5d2022-01-17 14:49:17 +010066 error = vnet_netlink_set_link_mtu (apif->host_if_index,
67 frame_size + hi->frame_overhead);
Damjan Marion88a9c0e2022-01-06 21:14:08 +010068
69 if (error)
Ray Kinsella7bfa1192017-05-15 11:52:43 +010070 {
Damjan Marion88a9c0e2022-01-06 21:14:08 +010071 vlib_log_err (apm->log_class, "netlink failed to change MTU: %U",
72 format_clib_error, error);
73 rv = vnet_error (VNET_ERR_SYSCALL_ERROR_1, "netlink error: %U",
74 format_clib_error, error);
75 clib_error_free (error);
76 return rv;
Ray Kinsella7bfa1192017-05-15 11:52:43 +010077 }
Damjan Marion88a9c0e2022-01-06 21:14:08 +010078 else
Damjan Marion1cd0e5d2022-01-17 14:49:17 +010079 apif->host_mtu = frame_size + hi->frame_overhead;
Damjan Marion83243a02016-02-29 13:09:30 +010080 return 0;
81}
82
Nathan Skrzypczakffc6bdc2021-02-01 17:13:59 +010083static int
84af_packet_read_mtu (af_packet_if_t *apif)
85{
86 af_packet_main_t *apm = &af_packet_main;
87 clib_error_t *error;
Aloys Augustine39376e2021-03-29 22:08:09 +020088 error = vnet_netlink_get_link_mtu (apif->host_if_index, &apif->host_mtu);
Nathan Skrzypczakffc6bdc2021-02-01 17:13:59 +010089 if (error)
90 {
Aloys Augustine39376e2021-03-29 22:08:09 +020091 vlib_log_err (apm->log_class, "netlink failed to get MTU: %U",
Nathan Skrzypczakffc6bdc2021-02-01 17:13:59 +010092 format_clib_error, error);
93 clib_error_free (error);
94 return VNET_API_ERROR_SYSCALL_ERROR_1;
95 }
96 return 0;
97}
98
Damjan Marion00a9dca2016-08-17 17:05:46 +020099static clib_error_t *
Damjan Marion56dd5432017-09-08 19:52:02 +0200100af_packet_fd_read_ready (clib_file_t * uf)
Damjan Marion83243a02016-02-29 13:09:30 +0100101{
Damjan Marioneb743fa2017-03-20 16:34:15 +0100102 vnet_main_t *vnm = vnet_get_main ();
Damjan Marion83243a02016-02-29 13:09:30 +0100103
Damjan Marion83243a02016-02-29 13:09:30 +0100104 /* Schedule the rx node */
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000105 vnet_hw_if_rx_queue_set_int_pending (vnm, uf->private_data);
Damjan Marion83243a02016-02-29 13:09:30 +0100106 return 0;
107}
108
109static int
Ray Kinsellac855b732017-04-21 12:24:43 +0100110is_bridge (const u8 * host_if_name)
111{
112 u8 *s;
113 DIR *dir = NULL;
114
115 s = format (0, "/sys/class/net/%s/bridge%c", host_if_name, 0);
116 dir = opendir ((char *) s);
117 vec_free (s);
118
119 if (dir)
120 {
121 closedir (dir);
122 return 0;
123 }
124
125 return -1;
126}
127
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000128static void
129af_packet_set_rx_queues (vlib_main_t *vm, af_packet_if_t *apif)
130{
131 vnet_main_t *vnm = vnet_get_main ();
132 af_packet_queue_t *rx_queue;
133
134 vnet_hw_if_set_input_node (vnm, apif->hw_if_index,
135 af_packet_input_node.index);
136
137 vec_foreach (rx_queue, apif->rx_queues)
138 {
139 rx_queue->queue_index = vnet_hw_if_register_rx_queue (
140 vnm, apif->hw_if_index, rx_queue->queue_id, VNET_HW_IF_RXQ_THREAD_ANY);
141
142 {
143 clib_file_t template = { 0 };
144 template.read_function = af_packet_fd_read_ready;
145 template.file_descriptor = rx_queue->fd;
146 template.private_data = rx_queue->queue_index;
147 template.flags = UNIX_FILE_EVENT_EDGE_TRIGGERED;
148 template.description =
149 format (0, "%U queue %u", format_af_packet_device_name,
150 apif->dev_instance, rx_queue->queue_id);
151 rx_queue->clib_file_index = clib_file_add (&file_main, &template);
152 }
153 vnet_hw_if_set_rx_queue_file_index (vnm, rx_queue->queue_index,
154 rx_queue->clib_file_index);
155 vnet_hw_if_set_rx_queue_mode (vnm, rx_queue->queue_index,
156 VNET_HW_IF_RX_MODE_INTERRUPT);
157 rx_queue->mode = VNET_HW_IF_RX_MODE_INTERRUPT;
158 }
159 vnet_hw_if_update_runtime_data (vnm, apif->hw_if_index);
160}
161
162static void
163af_packet_set_tx_queues (vlib_main_t *vm, af_packet_if_t *apif)
164{
165 vnet_main_t *vnm = vnet_get_main ();
166 af_packet_main_t *apm = &af_packet_main;
167 af_packet_queue_t *tx_queue;
168
169 vec_foreach (tx_queue, apif->tx_queues)
170 {
171 tx_queue->queue_index = vnet_hw_if_register_tx_queue (
172 vnm, apif->hw_if_index, tx_queue->queue_id);
173 }
174
175 if (apif->num_txqs == 0)
176 {
177 vlib_log_err (apm->log_class, "Interface %U has 0 txq",
178 format_vnet_hw_if_index_name, vnm, apif->hw_if_index);
179 return;
180 }
181
182 for (u32 j = 0; j < vlib_get_n_threads (); j++)
183 {
184 u32 qi = apif->tx_queues[j % apif->num_txqs].queue_index;
185 vnet_hw_if_tx_queue_assign_thread (vnm, qi, j);
186 }
187
188 vnet_hw_if_update_runtime_data (vnm, apif->hw_if_index);
189}
190
Ray Kinsellac855b732017-04-21 12:24:43 +0100191static int
Mohsin Kazmi219cbcb2022-03-18 16:58:31 +0000192create_packet_v3_sock (int host_if_index, tpacket_req3_t *rx_req,
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000193 tpacket_req3_t *tx_req, int *fd, af_packet_ring_t *ring,
Mohsin Kazmi2b6479c2022-04-05 12:03:47 +0000194 u8 *is_cksum_gso_enabled, u32 fanout_id, u8 is_fanout,
195 af_packet_if_flags_t *flags)
Damjan Marion83243a02016-02-29 13:09:30 +0100196{
Mohsin Kazmiacba9f72018-05-17 15:42:27 +0200197 af_packet_main_t *apm = &af_packet_main;
Damjan Marion83243a02016-02-29 13:09:30 +0100198 struct sockaddr_ll sll;
Mohsin Kazmi219cbcb2022-03-18 16:58:31 +0000199 socklen_t req_sz = sizeof (tpacket_req3_t);
200 int ret;
201 int ver = TPACKET_V3;
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000202 u32 ring_sz = 0;
203
204 if (rx_req)
205 ring_sz += rx_req->tp_block_size * rx_req->tp_block_nr;
206
207 if (tx_req)
208 ring_sz += tx_req->tp_block_size * tx_req->tp_block_nr;
Damjan Marion83243a02016-02-29 13:09:30 +0100209
Damjan Marion00a9dca2016-08-17 17:05:46 +0200210 if ((*fd = socket (AF_PACKET, SOCK_RAW, htons (ETH_P_ALL))) < 0)
Damjan Marion83243a02016-02-29 13:09:30 +0100211 {
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000212 vlib_log_err (apm->log_class,
213 "Failed to create AF_PACKET socket: %s (errno %d)",
214 strerror (errno), errno);
Damjan Marion83243a02016-02-29 13:09:30 +0100215 ret = VNET_API_ERROR_SYSCALL_ERROR_1;
216 goto error;
217 }
Damjan Marionc5170202017-10-11 14:15:47 +0200218
Chaoyu Jinafb19302018-03-13 07:37:41 -0700219 /* bind before rx ring is cfged so we don't receive packets from other interfaces */
Dave Barachb7b92992018-10-17 10:38:51 -0400220 clib_memset (&sll, 0, sizeof (sll));
Chaoyu Jinafb19302018-03-13 07:37:41 -0700221 sll.sll_family = PF_PACKET;
222 sll.sll_protocol = htons (ETH_P_ALL);
223 sll.sll_ifindex = host_if_index;
jackiechen19857fa41602019-05-07 18:59:13 +0800224 if (bind (*fd, (struct sockaddr *) &sll, sizeof (sll)) < 0)
Chaoyu Jinafb19302018-03-13 07:37:41 -0700225 {
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000226 vlib_log_err (apm->log_class,
227 "Failed to bind rx packet socket: %s (errno %d)",
228 strerror (errno), errno);
Chaoyu Jinafb19302018-03-13 07:37:41 -0700229 ret = VNET_API_ERROR_SYSCALL_ERROR_1;
230 goto error;
231 }
232
jackiechen19857fa41602019-05-07 18:59:13 +0800233 if (setsockopt (*fd, SOL_PACKET, PACKET_VERSION, &ver, sizeof (ver)) < 0)
Damjan Marion83243a02016-02-29 13:09:30 +0100234 {
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000235 vlib_log_err (apm->log_class,
236 "Failed to set rx packet interface version: %s (errno %d)",
237 strerror (errno), errno);
Damjan Marion83243a02016-02-29 13:09:30 +0100238 ret = VNET_API_ERROR_SYSCALL_ERROR_1;
239 goto error;
240 }
241
Damjan Marionc5170202017-10-11 14:15:47 +0200242 int opt = 1;
jackiechen19857fa41602019-05-07 18:59:13 +0800243 if (setsockopt (*fd, SOL_PACKET, PACKET_LOSS, &opt, sizeof (opt)) < 0)
Damjan Marion83243a02016-02-29 13:09:30 +0100244 {
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000245 vlib_log_err (
Mohsin Kazmi219cbcb2022-03-18 16:58:31 +0000246 apm->log_class,
247 "Failed to set packet tx ring error handling option: %s (errno %d)",
248 strerror (errno), errno);
Damjan Marion83243a02016-02-29 13:09:30 +0100249 ret = VNET_API_ERROR_SYSCALL_ERROR_1;
250 goto error;
251 }
252
Mohsin Kazmic1fd17b2022-03-22 21:40:04 +0000253 int opt2 = 1;
254 if (setsockopt (*fd, SOL_PACKET, PACKET_VNET_HDR, &opt2, sizeof (opt2)) < 0)
255 {
256 vlib_log_debug (
257 apm->log_class,
258 "Failed to set packet vnet hdr error handling option: %s (errno %d)",
259 strerror (errno), errno);
260 }
261 else
262 *is_cksum_gso_enabled = 1;
263
Florin Coras2dc942e2021-11-22 21:34:56 -0800264#if defined(PACKET_QDISC_BYPASS)
Mohsin Kazmi2b6479c2022-04-05 12:03:47 +0000265 if (*flags & AF_PACKET_IF_FLAGS_QDISC_BYPASS)
266 /* Introduced with Linux 3.14 so the ifdef should eventually be removed */
267 if (setsockopt (*fd, SOL_PACKET, PACKET_QDISC_BYPASS, &opt, sizeof (opt)) <
268 0)
269 {
270 // remove the flag
271 *flags &= ~AF_PACKET_IF_FLAGS_QDISC_BYPASS;
272 vlib_log_debug (apm->log_class,
273 "Failed to set qdisc bypass error "
274 "handling option: %s (errno %d)",
275 strerror (errno), errno);
276 }
Florin Coras2dc942e2021-11-22 21:34:56 -0800277#endif
Mohammed Hawarieed6fc92021-09-06 11:48:17 +0200278
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000279 if (is_fanout)
Damjan Marion83243a02016-02-29 13:09:30 +0100280 {
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000281 int fanout = ((fanout_id & 0xffff) | ((PACKET_FANOUT_HASH) << 16));
282 if (setsockopt (*fd, SOL_PACKET, PACKET_FANOUT, &fanout,
283 sizeof (fanout)) < 0)
284 {
285 vlib_log_err (apm->log_class,
286 "Failed to set fanout options: %s (errno %d)",
287 strerror (errno), errno);
288 ret = VNET_API_ERROR_SYSCALL_ERROR_1;
289 goto error;
290 }
291 }
292
293 if (rx_req)
294 if (setsockopt (*fd, SOL_PACKET, PACKET_RX_RING, rx_req, req_sz) < 0)
295 {
296 vlib_log_err (apm->log_class,
jackiechen19857fa41602019-05-07 18:59:13 +0800297 "Failed to set packet rx ring options: %s (errno %d)",
298 strerror (errno), errno);
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000299 ret = VNET_API_ERROR_SYSCALL_ERROR_1;
300 goto error;
301 }
Damjan Marion83243a02016-02-29 13:09:30 +0100302
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000303 if (tx_req)
304 if (setsockopt (*fd, SOL_PACKET, PACKET_TX_RING, tx_req, req_sz) < 0)
305 {
306 vlib_log_err (apm->log_class,
jackiechen19857fa41602019-05-07 18:59:13 +0800307 "Failed to set packet tx ring options: %s (errno %d)",
308 strerror (errno), errno);
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000309 ret = VNET_API_ERROR_SYSCALL_ERROR_1;
310 goto error;
311 }
312
313 ring->ring_start_addr = mmap (NULL, ring_sz, PROT_READ | PROT_WRITE,
314 MAP_SHARED | MAP_LOCKED, *fd, 0);
315 if (ring->ring_start_addr == MAP_FAILED)
316 {
317 vlib_log_err (apm->log_class, "mmap failure: %s (errno %d)",
318 strerror (errno), errno);
Damjan Marion83243a02016-02-29 13:09:30 +0100319 ret = VNET_API_ERROR_SYSCALL_ERROR_1;
320 goto error;
321 }
322
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000323 ring->ring_size = ring_sz;
Damjan Marion83243a02016-02-29 13:09:30 +0100324
Damjan Marion83243a02016-02-29 13:09:30 +0100325 return 0;
326error:
Dave Barach16ad6ae2016-07-28 17:55:30 -0400327 if (*fd >= 0)
jackiechen19857fa41602019-05-07 18:59:13 +0800328 {
329 close (*fd);
330 *fd = -1;
331 }
Damjan Marion83243a02016-02-29 13:09:30 +0100332 return ret;
333}
334
335int
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000336af_packet_queue_init (vlib_main_t *vm, af_packet_if_t *apif,
337 af_packet_create_if_arg_t *arg,
338 af_packet_queue_t *rx_queue, af_packet_queue_t *tx_queue,
339 u8 queue_id, u8 is_fanout)
340{
341 af_packet_main_t *apm = &af_packet_main;
342 tpacket_req3_t *rx_req = 0;
343 tpacket_req3_t *tx_req = 0;
344 int ret, fd = -1;
345 af_packet_ring_t ring = { 0 };
346 u8 *ring_addr = 0;
347 u32 rx_frames_per_block, tx_frames_per_block;
348 u32 rx_frame_size, tx_frame_size;
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000349 u32 i = 0;
350 u8 is_cksum_gso_enabled = 0;
351
352 if (rx_queue)
353 {
354 rx_frames_per_block = arg->rx_frames_per_block ?
355 arg->rx_frames_per_block :
356 AF_PACKET_DEFAULT_RX_FRAMES_PER_BLOCK;
357
358 rx_frame_size = arg->rx_frame_size ? arg->rx_frame_size :
359 AF_PACKET_DEFAULT_RX_FRAME_SIZE;
360 vec_validate (rx_queue->rx_req, 0);
361 rx_queue->rx_req->tp_block_size = rx_frame_size * rx_frames_per_block;
362 rx_queue->rx_req->tp_frame_size = rx_frame_size;
363 rx_queue->rx_req->tp_block_nr = AF_PACKET_RX_BLOCK_NR;
364 rx_queue->rx_req->tp_frame_nr =
365 AF_PACKET_RX_BLOCK_NR * rx_frames_per_block;
366 rx_queue->rx_req->tp_retire_blk_tov = 1; // 1 ms block timout
367 rx_queue->rx_req->tp_feature_req_word = 0;
368 rx_queue->rx_req->tp_sizeof_priv = 0;
369 rx_req = rx_queue->rx_req;
370 }
371
372 if (tx_queue)
373 {
374 tx_frames_per_block = arg->tx_frames_per_block ?
375 arg->tx_frames_per_block :
376 AF_PACKET_DEFAULT_TX_FRAMES_PER_BLOCK;
377 tx_frame_size = arg->tx_frame_size ? arg->tx_frame_size :
378 AF_PACKET_DEFAULT_TX_FRAME_SIZE;
379
380 vec_validate (tx_queue->tx_req, 0);
381 tx_queue->tx_req->tp_block_size = tx_frame_size * tx_frames_per_block;
382 tx_queue->tx_req->tp_frame_size = tx_frame_size;
383 tx_queue->tx_req->tp_block_nr = AF_PACKET_TX_BLOCK_NR;
384 tx_queue->tx_req->tp_frame_nr =
385 AF_PACKET_TX_BLOCK_NR * tx_frames_per_block;
386 tx_queue->tx_req->tp_retire_blk_tov = 0;
387 tx_queue->tx_req->tp_sizeof_priv = 0;
388 tx_queue->tx_req->tp_feature_req_word = 0;
389 tx_req = tx_queue->tx_req;
390 }
391
Mohsin Kazmi3ce1d142022-04-05 10:46:39 +0000392 if (rx_queue || tx_queue)
393 {
394 ret = create_packet_v3_sock (apif->host_if_index, rx_req, tx_req, &fd,
Mohsin Kazmi61a2c892022-04-05 11:28:29 +0000395 &ring, &is_cksum_gso_enabled,
Mohsin Kazmi2b6479c2022-04-05 12:03:47 +0000396 apif->dev_instance, is_fanout, &arg->flags);
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000397
Mohsin Kazmi3ce1d142022-04-05 10:46:39 +0000398 if (ret != 0)
399 goto error;
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000400
Mohsin Kazmi3ce1d142022-04-05 10:46:39 +0000401 vec_add1 (apif->rings, ring);
402 ring_addr = ring.ring_start_addr;
403 }
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000404
405 if (rx_queue)
406 {
407 rx_queue->fd = fd;
408 vec_validate (rx_queue->rx_ring, rx_queue->rx_req->tp_block_nr - 1);
409 vec_foreach_index (i, rx_queue->rx_ring)
410 {
411 rx_queue->rx_ring[i] =
412 ring_addr + i * rx_queue->rx_req->tp_block_size;
413 }
414
415 rx_queue->next_rx_block = 0;
416 rx_queue->queue_id = queue_id;
417 rx_queue->is_rx_pending = 0;
418 ring_addr = ring_addr + rx_queue->rx_req->tp_block_size *
419 rx_queue->rx_req->tp_block_nr;
420 }
421
422 if (tx_queue)
423 {
424 tx_queue->fd = fd;
425 vec_validate (tx_queue->tx_ring, tx_queue->tx_req->tp_block_nr - 1);
426 vec_foreach_index (i, tx_queue->tx_ring)
427 {
428 tx_queue->tx_ring[i] =
429 ring_addr + i * tx_queue->tx_req->tp_block_size;
430 }
431
432 tx_queue->next_tx_frame = 0;
433 tx_queue->queue_id = queue_id;
434 clib_spinlock_init (&tx_queue->lockp);
435 }
436
437 if (queue_id == 0)
Mohsin Kazmi61a2c892022-04-05 11:28:29 +0000438 apif->is_cksum_gso_enabled = is_cksum_gso_enabled;
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000439
440 return 0;
441error:
442 vlib_log_err (apm->log_class, "Failed to set queue %u error", queue_id);
Mohsin Kazmi3ce1d142022-04-05 10:46:39 +0000443 if (rx_queue)
444 vec_free (rx_queue->rx_req);
445 if (tx_queue)
446 vec_free (tx_queue->tx_req);
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000447 return ret;
448}
449
450int
451af_packet_device_init (vlib_main_t *vm, af_packet_if_t *apif,
452 af_packet_create_if_arg_t *args)
453{
454 af_packet_main_t *apm = &af_packet_main;
455 af_packet_queue_t *rx_queue = 0;
456 af_packet_queue_t *tx_queue = 0;
457 u16 nq = clib_min (args->num_rxqs, args->num_txqs);
458 u16 i = 0;
459 int ret = 0;
460 u8 is_fanout = (args->num_rxqs > 1) ? 1 : 0;
461
462 vec_validate (apif->rx_queues, args->num_rxqs - 1);
463 vec_validate (apif->tx_queues, args->num_txqs - 1);
464
465 for (; i < nq; i++)
466 {
467 rx_queue = vec_elt_at_index (apif->rx_queues, i);
468 tx_queue = vec_elt_at_index (apif->tx_queues, i);
469 ret = af_packet_queue_init (vm, apif, args, rx_queue, tx_queue, i,
470 is_fanout);
471 if (ret != 0)
472 goto error;
473 }
474
475 if (args->num_rxqs > args->num_txqs)
476 {
477 for (; i < args->num_rxqs; i++)
478 {
479 rx_queue = vec_elt_at_index (apif->rx_queues, i);
480 ret =
481 af_packet_queue_init (vm, apif, args, rx_queue, 0, i, is_fanout);
482 if (ret != 0)
483 goto error;
484 }
485 }
486 else if (args->num_txqs > args->num_rxqs)
487 {
488 for (; i < args->num_txqs; i++)
489 {
490 tx_queue = vec_elt_at_index (apif->tx_queues, i);
491 ret = af_packet_queue_init (vm, apif, args, 0, tx_queue, i, 0);
492 if (ret != 0)
493 goto error;
494 }
495 }
496
497 apif->num_rxqs = args->num_rxqs;
498 apif->num_txqs = args->num_txqs;
499
500 return 0;
501error:
502 vlib_log_err (apm->log_class, "Failed to init device error");
503 return ret;
504}
505
506int
Nathan Skrzypczak7d0e30b2021-06-23 11:28:39 +0200507af_packet_create_if (af_packet_create_if_arg_t *arg)
Damjan Marion83243a02016-02-29 13:09:30 +0100508{
Damjan Marion00a9dca2016-08-17 17:05:46 +0200509 af_packet_main_t *apm = &af_packet_main;
Nathan Skrzypczak7d0e30b2021-06-23 11:28:39 +0200510 vlib_main_t *vm = vlib_get_main ();
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000511 int fd2 = -1;
Mohsin Kazmia50a14c2018-04-25 15:58:05 +0200512 struct ifreq ifr;
Damjan Marion00a9dca2016-08-17 17:05:46 +0200513 af_packet_if_t *apif = 0;
Damjan Marion83243a02016-02-29 13:09:30 +0100514 u8 hw_addr[6];
Damjan Marion00a9dca2016-08-17 17:05:46 +0200515 vnet_sw_interface_t *sw;
516 vnet_main_t *vnm = vnet_get_main ();
Mohsin Kazmic1fd17b2022-03-22 21:40:04 +0000517 vnet_hw_if_caps_t caps = VNET_HW_IF_CAP_INT_MODE;
Damjan Marion00a9dca2016-08-17 17:05:46 +0200518 uword *p;
Damjan Marion83243a02016-02-29 13:09:30 +0100519 uword if_index;
jackiechen19857fa41602019-05-07 18:59:13 +0800520 u8 *host_if_name_dup = 0;
Ray Kinsellac855b732017-04-21 12:24:43 +0100521 int host_if_index = -1;
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000522 int ret = 0;
Damjan Marion83243a02016-02-29 13:09:30 +0100523
Nathan Skrzypczak7d0e30b2021-06-23 11:28:39 +0200524 p = mhash_get (&apm->if_index_by_host_if_name, arg->host_if_name);
Damjan Marion83243a02016-02-29 13:09:30 +0100525 if (p)
526 {
Mohsin Kazmi43fc6882018-03-22 23:45:23 +0100527 apif = vec_elt_at_index (apm->interfaces, p[0]);
Nathan Skrzypczak7d0e30b2021-06-23 11:28:39 +0200528 arg->sw_if_index = apif->sw_if_index;
Mohsin Kazmi43fc6882018-03-22 23:45:23 +0100529 return VNET_API_ERROR_IF_ALREADY_EXISTS;
Damjan Marion83243a02016-02-29 13:09:30 +0100530 }
531
Nathan Skrzypczak7d0e30b2021-06-23 11:28:39 +0200532 host_if_name_dup = vec_dup (arg->host_if_name);
533
Mohsin Kazmia50a14c2018-04-25 15:58:05 +0200534 /*
535 * make sure host side of interface is 'UP' before binding AF_PACKET
536 * socket on it.
537 */
538 if ((fd2 = socket (AF_UNIX, SOCK_DGRAM, 0)) < 0)
Ray Kinsellac855b732017-04-21 12:24:43 +0100539 {
jackiechen19857fa41602019-05-07 18:59:13 +0800540 vlib_log_debug (apm->log_class,
541 "Failed to create AF_UNIX socket: %s (errno %d)",
542 strerror (errno), errno);
Mohsin Kazmia50a14c2018-04-25 15:58:05 +0200543 ret = VNET_API_ERROR_SYSCALL_ERROR_1;
544 goto error;
545 }
546
Nathan Skrzypczak7d0e30b2021-06-23 11:28:39 +0200547 clib_memcpy (ifr.ifr_name, (const char *) arg->host_if_name,
548 vec_len (arg->host_if_name));
jackiechen19857fa41602019-05-07 18:59:13 +0800549 if (ioctl (fd2, SIOCGIFINDEX, &ifr) < 0)
Mohsin Kazmia50a14c2018-04-25 15:58:05 +0200550 {
Nathan Skrzypczak7d0e30b2021-06-23 11:28:39 +0200551 vlib_log_debug (
552 apm->log_class,
553 "Failed to retrieve the interface (%s) index: %s (errno %d)",
554 arg->host_if_name, strerror (errno), errno);
jackiechen19857fa41602019-05-07 18:59:13 +0800555 ret = VNET_API_ERROR_INVALID_INTERFACE;
556 goto error;
Ray Kinsellac855b732017-04-21 12:24:43 +0100557 }
558
Mohsin Kazmia50a14c2018-04-25 15:58:05 +0200559 host_if_index = ifr.ifr_ifindex;
jackiechen19857fa41602019-05-07 18:59:13 +0800560 if (ioctl (fd2, SIOCGIFFLAGS, &ifr) < 0)
Mohsin Kazmia50a14c2018-04-25 15:58:05 +0200561 {
jackiechen19857fa41602019-05-07 18:59:13 +0800562 vlib_log_debug (apm->log_class,
563 "Failed to get the active flag: %s (errno %d)",
564 strerror (errno), errno);
565 ret = VNET_API_ERROR_SYSCALL_ERROR_1;
Mohsin Kazmia50a14c2018-04-25 15:58:05 +0200566 goto error;
567 }
568
569 if (!(ifr.ifr_flags & IFF_UP))
570 {
571 ifr.ifr_flags |= IFF_UP;
jackiechen19857fa41602019-05-07 18:59:13 +0800572 if (ioctl (fd2, SIOCSIFFLAGS, &ifr) < 0)
Mohsin Kazmia50a14c2018-04-25 15:58:05 +0200573 {
jackiechen19857fa41602019-05-07 18:59:13 +0800574 vlib_log_debug (apm->log_class,
575 "Failed to set the active flag: %s (errno %d)",
576 strerror (errno), errno);
577 ret = VNET_API_ERROR_SYSCALL_ERROR_1;
Mohsin Kazmia50a14c2018-04-25 15:58:05 +0200578 goto error;
579 }
580 }
581
582 if (fd2 > -1)
jackiechen19857fa41602019-05-07 18:59:13 +0800583 {
584 close (fd2);
585 fd2 = -1;
586 }
Mohsin Kazmia50a14c2018-04-25 15:58:05 +0200587
Nathan Skrzypczak7d0e30b2021-06-23 11:28:39 +0200588 ret = is_bridge (arg->host_if_name);
Ray Kinsellac855b732017-04-21 12:24:43 +0100589 if (ret == 0) /* is a bridge, ignore state */
590 host_if_index = -1;
591
Damjan Marion83243a02016-02-29 13:09:30 +0100592 /* So far everything looks good, let's create interface */
Damjan Marion048ee2e2016-03-16 22:59:21 +0100593 pool_get (apm->interfaces, apif);
Damjan Marion83243a02016-02-29 13:09:30 +0100594 if_index = apif - apm->interfaces;
595
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000596 apif->dev_instance = if_index;
Ray Kinsellac855b732017-04-21 12:24:43 +0100597 apif->host_if_index = host_if_index;
Ivan Kellybfe737a2016-10-07 18:02:43 +0200598 apif->host_if_name = host_if_name_dup;
Dave Barach13f3c452016-03-29 11:56:41 -0400599 apif->per_interface_next_index = ~0;
Mohsin Kazmicae84fa2021-10-08 15:10:49 +0000600 apif->mode = arg->mode;
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000601
602 ret = af_packet_device_init (vm, apif, arg);
603 if (ret != 0)
604 goto error;
Damjan Marion83243a02016-02-29 13:09:30 +0100605
Nathan Skrzypczakffc6bdc2021-02-01 17:13:59 +0100606 ret = af_packet_read_mtu (apif);
607 if (ret != 0)
608 goto error;
609
Mohsin KAZMIcf751ec2017-01-18 11:59:45 +0100610
Nathan Skrzypczakb225b0a2021-10-26 16:11:38 +0200611 if (apif->mode != AF_PACKET_IF_MODE_IP)
Mohsin Kazmicae84fa2021-10-08 15:10:49 +0000612 {
Damjan Marion5c954c42022-01-06 20:36:14 +0100613 vnet_eth_interface_registration_t eir = {};
Mohsin Kazmicae84fa2021-10-08 15:10:49 +0000614 /*use configured or generate random MAC address */
615 if (arg->hw_addr)
616 clib_memcpy (hw_addr, arg->hw_addr, 6);
617 else
618 {
619 f64 now = vlib_time_now (vm);
620 u32 rnd;
621 rnd = (u32) (now * 1e6);
622 rnd = random_u32 (&rnd);
623
624 clib_memcpy (hw_addr + 2, &rnd, sizeof (rnd));
625 hw_addr[0] = 2;
626 hw_addr[1] = 0xfe;
627 }
628
Damjan Marion5c954c42022-01-06 20:36:14 +0100629 eir.dev_class_index = af_packet_device_class.index;
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000630 eir.dev_instance = apif->dev_instance;
Damjan Marion5c954c42022-01-06 20:36:14 +0100631 eir.address = hw_addr;
Damjan Marion1cd0e5d2022-01-17 14:49:17 +0100632 eir.cb.set_max_frame_size = af_packet_eth_set_max_frame_size;
Damjan Marion5c954c42022-01-06 20:36:14 +0100633 apif->hw_if_index = vnet_eth_register_interface (vnm, &eir);
Mohsin Kazmicae84fa2021-10-08 15:10:49 +0000634 }
Damjan Marion83243a02016-02-29 13:09:30 +0100635 else
636 {
Mohsin Kazmicae84fa2021-10-08 15:10:49 +0000637 apif->hw_if_index = vnet_register_interface (
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000638 vnm, af_packet_device_class.index, apif->dev_instance,
639 af_packet_ip_device_hw_interface_class.index, apif->dev_instance);
Damjan Marion83243a02016-02-29 13:09:30 +0100640 }
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000641
Damjan Marion83243a02016-02-29 13:09:30 +0100642 sw = vnet_get_hw_sw_interface (vnm, apif->hw_if_index);
643 apif->sw_if_index = sw->sw_if_index;
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000644
645 af_packet_set_rx_queues (vm, apif);
646 af_packet_set_tx_queues (vm, apif);
Damjan Marion44036902017-04-28 12:29:15 +0200647
Mohsin Kazmi2b6479c2022-04-05 12:03:47 +0000648 apif->is_qdisc_bypass_enabled =
649 (arg->flags & AF_PACKET_IF_FLAGS_QDISC_BYPASS);
650
Mohsin Kazmic1fd17b2022-03-22 21:40:04 +0000651 if (apif->is_cksum_gso_enabled)
652 caps |= VNET_HW_IF_CAP_TCP_GSO | VNET_HW_IF_CAP_TX_IP4_CKSUM |
653 VNET_HW_IF_CAP_TX_TCP_CKSUM | VNET_HW_IF_CAP_TX_UDP_CKSUM;
654
655 vnet_hw_if_set_caps (vnm, apif->hw_if_index, caps);
Damjan Marion83243a02016-02-29 13:09:30 +0100656 vnet_hw_interface_set_flags (vnm, apif->hw_if_index,
657 VNET_HW_INTERFACE_FLAG_LINK_UP);
658
Ivan Kellybfe737a2016-10-07 18:02:43 +0200659 mhash_set_mem (&apm->if_index_by_host_if_name, host_if_name_dup, &if_index,
660 0);
Nathan Skrzypczak7d0e30b2021-06-23 11:28:39 +0200661 arg->sw_if_index = apif->sw_if_index;
Mohsin KAZMIcf751ec2017-01-18 11:59:45 +0100662
Damjan Marion83243a02016-02-29 13:09:30 +0100663 return 0;
664
665error:
Mohsin Kazmia50a14c2018-04-25 15:58:05 +0200666 if (fd2 > -1)
jackiechen19857fa41602019-05-07 18:59:13 +0800667 {
668 close (fd2);
669 fd2 = -1;
670 }
Ivan Kellybfe737a2016-10-07 18:02:43 +0200671 vec_free (host_if_name_dup);
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000672 memset (apif, 0, sizeof (*apif));
673 pool_put (apm->interfaces, apif);
Damjan Marion83243a02016-02-29 13:09:30 +0100674 return ret;
675}
676
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000677static int
678af_packet_rx_queue_free (af_packet_if_t *apif, af_packet_queue_t *rx_queue)
679{
680 clib_file_del_by_index (&file_main, rx_queue->clib_file_index);
681 close (rx_queue->fd);
682 rx_queue->fd = -1;
683 rx_queue->rx_ring = NULL;
684 vec_free (rx_queue->rx_req);
685 rx_queue->rx_req = NULL;
686 return 0;
687}
688
689static int
690af_packet_tx_queue_free (af_packet_if_t *apif, af_packet_queue_t *tx_queue)
691{
692 close (tx_queue->fd);
693 tx_queue->fd = -1;
694 clib_spinlock_free (&tx_queue->lockp);
695 tx_queue->tx_ring = NULL;
696 vec_free (tx_queue->tx_req);
697 tx_queue->tx_req = NULL;
698 return 0;
699}
700
701static int
702af_packet_ring_free (af_packet_if_t *apif, af_packet_ring_t *ring)
703{
704 af_packet_main_t *apm = &af_packet_main;
705
706 if (ring)
707 {
708 // FIXME: unmap the memory
709 if (munmap (ring->ring_start_addr, ring->ring_size))
710 vlib_log_warn (apm->log_class,
711 "Host interface %s could not free ring %p of size %u",
712 apif->host_if_name, ring->ring_start_addr,
713 ring->ring_size);
714 else
715 ring->ring_start_addr = 0;
716 }
717
718 return 0;
719}
720
Peter Leidba76f22016-04-08 08:16:31 -0700721int
Nathan Skrzypczak7d0e30b2021-06-23 11:28:39 +0200722af_packet_delete_if (u8 *host_if_name)
Peter Leidba76f22016-04-08 08:16:31 -0700723{
Damjan Marion00a9dca2016-08-17 17:05:46 +0200724 vnet_main_t *vnm = vnet_get_main ();
Peter Leidba76f22016-04-08 08:16:31 -0700725 af_packet_main_t *apm = &af_packet_main;
726 af_packet_if_t *apif;
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000727 af_packet_queue_t *rx_queue;
728 af_packet_queue_t *tx_queue;
729 af_packet_ring_t *ring;
Peter Leidba76f22016-04-08 08:16:31 -0700730 uword *p;
Peter Leidba76f22016-04-08 08:16:31 -0700731
Damjan Marion00a9dca2016-08-17 17:05:46 +0200732 p = mhash_get (&apm->if_index_by_host_if_name, host_if_name);
733 if (p == NULL)
734 {
Mohsin Kazmiacba9f72018-05-17 15:42:27 +0200735 vlib_log_warn (apm->log_class, "Host interface %s does not exist",
736 host_if_name);
Damjan Marion00a9dca2016-08-17 17:05:46 +0200737 return VNET_API_ERROR_SYSCALL_ERROR_1;
738 }
739 apif = pool_elt_at_index (apm->interfaces, p[0]);
Peter Leidba76f22016-04-08 08:16:31 -0700740
741 /* bring down the interface */
Damjan Marion00a9dca2016-08-17 17:05:46 +0200742 vnet_hw_interface_set_flags (vnm, apif->hw_if_index, 0);
Peter Leidba76f22016-04-08 08:16:31 -0700743
744 /* clean up */
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000745 vec_foreach (rx_queue, apif->rx_queues)
746 af_packet_rx_queue_free (apif, rx_queue);
747 vec_foreach (tx_queue, apif->tx_queues)
748 af_packet_tx_queue_free (apif, tx_queue);
749 vec_foreach (ring, apif->rings)
750 af_packet_ring_free (apif, ring);
Eyal Barif298ecf2016-09-19 18:47:39 +0300751
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000752 vec_free (apif->rx_queues);
753 apif->rx_queues = NULL;
754 vec_free (apif->tx_queues);
755 apif->tx_queues = NULL;
756 vec_free (apif->rings);
757 apif->rings = NULL;
Peter Leidba76f22016-04-08 08:16:31 -0700758
Damjan Marion00a9dca2016-08-17 17:05:46 +0200759 vec_free (apif->host_if_name);
Peter Leidba76f22016-04-08 08:16:31 -0700760 apif->host_if_name = NULL;
Ray Kinsellac855b732017-04-21 12:24:43 +0100761 apif->host_if_index = -1;
Peter Leidba76f22016-04-08 08:16:31 -0700762
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000763 mhash_unset (&apm->if_index_by_host_if_name, host_if_name, p);
Peter Leidba76f22016-04-08 08:16:31 -0700764
Nathan Skrzypczakb225b0a2021-10-26 16:11:38 +0200765 if (apif->mode != AF_PACKET_IF_MODE_IP)
Mohsin Kazmicae84fa2021-10-08 15:10:49 +0000766 ethernet_delete_interface (vnm, apif->hw_if_index);
767 else
768 vnet_delete_hw_interface (vnm, apif->hw_if_index);
Peter Leidba76f22016-04-08 08:16:31 -0700769
Mohsin Kazmi5a7aa512022-03-25 14:27:45 +0000770 memset (apif, 0, sizeof (*apif));
Damjan Marion00a9dca2016-08-17 17:05:46 +0200771 pool_put (apm->interfaces, apif);
Peter Leidba76f22016-04-08 08:16:31 -0700772
773 return 0;
774}
775
Jakub Grajciar92b02752017-10-20 13:37:28 +0200776int
Nathan Skrzypczak7d0e30b2021-06-23 11:28:39 +0200777af_packet_set_l4_cksum_offload (u32 sw_if_index, u8 set)
Jakub Grajciar92b02752017-10-20 13:37:28 +0200778{
Mohsin Kazmic1fd17b2022-03-22 21:40:04 +0000779 // deprecated ...
Jakub Grajciar92b02752017-10-20 13:37:28 +0200780 return 0;
781}
782
Mohsin Kazmi04e0bb22018-05-28 18:55:37 +0200783int
784af_packet_dump_ifs (af_packet_if_detail_t ** out_af_packet_ifs)
785{
786 af_packet_main_t *apm = &af_packet_main;
787 af_packet_if_t *apif;
788 af_packet_if_detail_t *r_af_packet_ifs = NULL;
789 af_packet_if_detail_t *af_packet_if = NULL;
790
Damjan Marionb2c31b62020-12-13 21:47:40 +0100791 pool_foreach (apif, apm->interfaces)
792 {
Jakub Grajciar3b2db902019-08-26 11:25:52 +0200793 vec_add2 (r_af_packet_ifs, af_packet_if, 1);
794 af_packet_if->sw_if_index = apif->sw_if_index;
795 if (apif->host_if_name)
796 {
797 clib_memcpy (af_packet_if->host_if_name, apif->host_if_name,
798 MIN (ARRAY_LEN (af_packet_if->host_if_name) - 1,
799 strlen ((const char *) apif->host_if_name)));
800 }
Damjan Marionb2c31b62020-12-13 21:47:40 +0100801 }
Mohsin Kazmi04e0bb22018-05-28 18:55:37 +0200802
803 *out_af_packet_ifs = r_af_packet_ifs;
804
805 return 0;
806}
807
Damjan Marion83243a02016-02-29 13:09:30 +0100808static clib_error_t *
809af_packet_init (vlib_main_t * vm)
810{
Damjan Marion00a9dca2016-08-17 17:05:46 +0200811 af_packet_main_t *apm = &af_packet_main;
Damjan Marion553f6bd2016-09-07 11:54:22 +0200812 vlib_thread_main_t *tm = vlib_get_thread_main ();
Damjan Marion83243a02016-02-29 13:09:30 +0100813
Dave Barachb7b92992018-10-17 10:38:51 -0400814 clib_memset (apm, 0, sizeof (af_packet_main_t));
Damjan Marion83243a02016-02-29 13:09:30 +0100815
816 mhash_init_vec_string (&apm->if_index_by_host_if_name, sizeof (uword));
817
Damjan Marion553f6bd2016-09-07 11:54:22 +0200818 vec_validate_aligned (apm->rx_buffers, tm->n_vlib_mains - 1,
819 CLIB_CACHE_LINE_BYTES);
820
Mohsin Kazmiacba9f72018-05-17 15:42:27 +0200821 apm->log_class = vlib_log_register_class ("af_packet", 0);
822 vlib_log_debug (apm->log_class, "initialized");
823
Damjan Marion83243a02016-02-29 13:09:30 +0100824 return 0;
825}
826
827VLIB_INIT_FUNCTION (af_packet_init);
Damjan Marion00a9dca2016-08-17 17:05:46 +0200828
829/*
830 * fd.io coding-style-patch-verification: ON
831 *
832 * Local Variables:
833 * eval: (c-set-style "gnu")
834 * End:
835 */