blob: e814efa608a83ec6a0102ec15c467c5765d9159c [file] [log] [blame]
Ed Warnickecb9cada2015-12-08 15:45:58 -07001/*
2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15/*
16 * interface_output.c: interface output node
17 *
18 * Copyright (c) 2008 Eliot Dresselhaus
19 *
20 * Permission is hereby granted, free of charge, to any person obtaining
21 * a copy of this software and associated documentation files (the
22 * "Software"), to deal in the Software without restriction, including
23 * without limitation the rights to use, copy, modify, merge, publish,
24 * distribute, sublicense, and/or sell copies of the Software, and to
25 * permit persons to whom the Software is furnished to do so, subject to
26 * the following conditions:
27 *
28 * The above copyright notice and this permission notice shall be
29 * included in all copies or substantial portions of the Software.
30 *
31 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38 */
39
40#include <vnet/vnet.h>
Dave Barach2c0a4f42017-06-29 09:30:15 -040041#include <vnet/ip/icmp46_packet.h>
42#include <vnet/ip/ip4.h>
43#include <vnet/ip/ip6.h>
44#include <vnet/udp/udp_packet.h>
Damjan Marion152e21d2016-11-29 14:55:43 +010045#include <vnet/feature/feature.h>
Ed Warnickecb9cada2015-12-08 15:45:58 -070046
Dave Barachba868bb2016-08-08 09:51:21 -040047typedef struct
48{
Ed Warnickecb9cada2015-12-08 15:45:58 -070049 u32 sw_if_index;
Chris Luke1c9f6ce2016-04-26 10:55:52 -040050 u8 data[128 - sizeof (u32)];
Dave Barachba868bb2016-08-08 09:51:21 -040051}
52interface_output_trace_t;
Ed Warnickecb9cada2015-12-08 15:45:58 -070053
Dave Barachba868bb2016-08-08 09:51:21 -040054u8 *
55format_vnet_interface_output_trace (u8 * s, va_list * va)
Ed Warnickecb9cada2015-12-08 15:45:58 -070056{
57 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *);
Dave Barachba868bb2016-08-08 09:51:21 -040058 vlib_node_t *node = va_arg (*va, vlib_node_t *);
59 interface_output_trace_t *t = va_arg (*va, interface_output_trace_t *);
60 vnet_main_t *vnm = vnet_get_main ();
61 vnet_sw_interface_t *si;
Christophe Fontained3c008d2017-10-02 18:10:54 +020062 u32 indent;
Ed Warnickecb9cada2015-12-08 15:45:58 -070063
Dave Barachba868bb2016-08-08 09:51:21 -040064 if (t->sw_if_index != (u32) ~ 0)
Ed Warnickecb9cada2015-12-08 15:45:58 -070065 {
Ed Warnickecb9cada2015-12-08 15:45:58 -070066 indent = format_get_indent (s);
Dave Barachba868bb2016-08-08 09:51:21 -040067
Neale Ranns177bbdc2016-11-15 09:46:51 +000068 if (pool_is_free_index
69 (vnm->interface_main.sw_interfaces, t->sw_if_index))
70 {
71 /* the interface may have been deleted by the time the trace is printed */
72 s = format (s, "sw_if_index: %d\n%U%U",
73 t->sw_if_index,
74 format_white_space, indent,
75 node->format_buffer ? node->
76 format_buffer : format_hex_bytes, t->data,
77 sizeof (t->data));
78 }
79 else
80 {
81 si = vnet_get_sw_interface (vnm, t->sw_if_index);
82
83 s = format (s, "%U\n%U%U",
84 format_vnet_sw_interface_name, vnm, si,
85 format_white_space, indent,
86 node->format_buffer ? node->
87 format_buffer : format_hex_bytes, t->data,
88 sizeof (t->data));
89 }
Ed Warnickecb9cada2015-12-08 15:45:58 -070090 }
91 return s;
92}
93
94static void
95vnet_interface_output_trace (vlib_main_t * vm,
96 vlib_node_runtime_t * node,
Dave Barachba868bb2016-08-08 09:51:21 -040097 vlib_frame_t * frame, uword n_buffers)
Ed Warnickecb9cada2015-12-08 15:45:58 -070098{
Dave Barachba868bb2016-08-08 09:51:21 -040099 u32 n_left, *from;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700100
101 n_left = n_buffers;
Damjan Mariona3d59862018-11-10 10:23:00 +0100102 from = vlib_frame_vector_args (frame);
Dave Barachba868bb2016-08-08 09:51:21 -0400103
Ed Warnickecb9cada2015-12-08 15:45:58 -0700104 while (n_left >= 4)
105 {
106 u32 bi0, bi1;
Dave Barachba868bb2016-08-08 09:51:21 -0400107 vlib_buffer_t *b0, *b1;
108 interface_output_trace_t *t0, *t1;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700109
110 /* Prefetch next iteration. */
111 vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
112 vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
113
114 bi0 = from[0];
115 bi1 = from[1];
116
117 b0 = vlib_get_buffer (vm, bi0);
118 b1 = vlib_get_buffer (vm, bi1);
119
120 if (b0->flags & VLIB_BUFFER_IS_TRACED)
121 {
122 t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
123 t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX];
Dave Barach178cf492018-11-13 16:34:13 -0500124 clib_memcpy_fast (t0->data, vlib_buffer_get_current (b0),
125 sizeof (t0->data));
Ed Warnickecb9cada2015-12-08 15:45:58 -0700126 }
127 if (b1->flags & VLIB_BUFFER_IS_TRACED)
128 {
129 t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
130 t1->sw_if_index = vnet_buffer (b1)->sw_if_index[VLIB_TX];
Dave Barach178cf492018-11-13 16:34:13 -0500131 clib_memcpy_fast (t1->data, vlib_buffer_get_current (b1),
132 sizeof (t1->data));
Ed Warnickecb9cada2015-12-08 15:45:58 -0700133 }
134 from += 2;
135 n_left -= 2;
136 }
137
138 while (n_left >= 1)
139 {
140 u32 bi0;
Dave Barachba868bb2016-08-08 09:51:21 -0400141 vlib_buffer_t *b0;
142 interface_output_trace_t *t0;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700143
144 bi0 = from[0];
145
146 b0 = vlib_get_buffer (vm, bi0);
147
148 if (b0->flags & VLIB_BUFFER_IS_TRACED)
149 {
150 t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
151 t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX];
Dave Barach178cf492018-11-13 16:34:13 -0500152 clib_memcpy_fast (t0->data, vlib_buffer_get_current (b0),
153 sizeof (t0->data));
Ed Warnickecb9cada2015-12-08 15:45:58 -0700154 }
155 from += 1;
156 n_left -= 1;
157 }
158}
159
Dave Barach2c0a4f42017-06-29 09:30:15 -0400160static_always_inline void
161calc_checksums (vlib_main_t * vm, vlib_buffer_t * b)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700162{
Dave Barach2c0a4f42017-06-29 09:30:15 -0400163 ip4_header_t *ip4;
164 ip6_header_t *ip6;
165 tcp_header_t *th;
166 udp_header_t *uh;
167
168 int is_ip4 = (b->flags & VNET_BUFFER_F_IS_IP4) != 0;
169 int is_ip6 = (b->flags & VNET_BUFFER_F_IS_IP6) != 0;
170
171 ASSERT (!(is_ip4 && is_ip6));
172
173 ip4 = (ip4_header_t *) (b->data + vnet_buffer (b)->l3_hdr_offset);
174 ip6 = (ip6_header_t *) (b->data + vnet_buffer (b)->l3_hdr_offset);
175 th = (tcp_header_t *) (b->data + vnet_buffer (b)->l4_hdr_offset);
176 uh = (udp_header_t *) (b->data + vnet_buffer (b)->l4_hdr_offset);
177
178 if (is_ip4)
179 {
180 ip4 = (ip4_header_t *) (b->data + vnet_buffer (b)->l3_hdr_offset);
181 if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
182 ip4->checksum = ip4_header_checksum (ip4);
183 if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
Andrej Kozemcak9d7570c2019-01-07 08:39:22 +0100184 {
185 th->checksum = 0;
186 th->checksum = ip4_tcp_udp_compute_checksum (vm, b, ip4);
187 }
Dave Barach2c0a4f42017-06-29 09:30:15 -0400188 if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
189 uh->checksum = ip4_tcp_udp_compute_checksum (vm, b, ip4);
190 }
191 if (is_ip6)
192 {
193 int bogus;
Dave Barach2c0a4f42017-06-29 09:30:15 -0400194 if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
195 th->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ip6, &bogus);
196 if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
197 uh->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ip6, &bogus);
198 }
199
200 b->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
201 b->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
202 b->flags &= ~VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
203}
204
205static_always_inline uword
206vnet_interface_output_node_inline (vlib_main_t * vm,
207 vlib_node_runtime_t * node,
208 vlib_frame_t * frame, vnet_main_t * vnm,
209 vnet_hw_interface_t * hi,
210 int do_tx_offloads)
211{
Dave Barachba868bb2016-08-08 09:51:21 -0400212 vnet_interface_output_runtime_t *rt = (void *) node->runtime_data;
213 vnet_sw_interface_t *si;
Dave Barachba868bb2016-08-08 09:51:21 -0400214 u32 n_left_to_tx, *from, *from_end, *to_tx;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700215 u32 n_bytes, n_buffers, n_packets;
Damjan Marion363640d2017-03-06 11:53:10 +0100216 u32 n_bytes_b0, n_bytes_b1, n_bytes_b2, n_bytes_b3;
Damjan Marion586afd72017-04-05 19:18:20 +0200217 u32 thread_index = vm->thread_index;
Dave Barachba868bb2016-08-08 09:51:21 -0400218 vnet_interface_main_t *im = &vnm->interface_main;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700219 u32 next_index = VNET_INTERFACE_OUTPUT_NEXT_TX;
Damjan Marion152e21d2016-11-29 14:55:43 +0100220 u32 current_config_index = ~0;
221 u8 arc = im->output_feature_arc_index;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700222
223 n_buffers = frame->n_vectors;
224
225 if (node->flags & VLIB_NODE_FLAG_TRACE)
226 vnet_interface_output_trace (vm, node, frame, n_buffers);
227
Damjan Mariona3d59862018-11-10 10:23:00 +0100228 from = vlib_frame_vector_args (frame);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700229
230 if (rt->is_deleted)
Dave Barachba868bb2016-08-08 09:51:21 -0400231 return vlib_error_drop_buffers (vm, node, from,
Ed Warnickecb9cada2015-12-08 15:45:58 -0700232 /* buffer stride */ 1,
233 n_buffers,
234 VNET_INTERFACE_OUTPUT_NEXT_DROP,
235 node->node_index,
236 VNET_INTERFACE_OUTPUT_ERROR_INTERFACE_DELETED);
237
238 si = vnet_get_sw_interface (vnm, rt->sw_if_index);
239 hi = vnet_get_sup_hw_interface (vnm, rt->sw_if_index);
Steve Shin042a6212017-07-07 14:57:46 -0700240 if (!(si->flags & (VNET_SW_INTERFACE_FLAG_ADMIN_UP |
241 VNET_SW_INTERFACE_FLAG_BOND_SLAVE)) ||
Dave Barachba868bb2016-08-08 09:51:21 -0400242 !(hi->flags & VNET_HW_INTERFACE_FLAG_LINK_UP))
Ed Warnickecb9cada2015-12-08 15:45:58 -0700243 {
Dave Barachba868bb2016-08-08 09:51:21 -0400244 vlib_simple_counter_main_t *cm;
245
Ed Warnickecb9cada2015-12-08 15:45:58 -0700246 cm = vec_elt_at_index (vnm->interface_main.sw_if_counters,
Dave Barachba868bb2016-08-08 09:51:21 -0400247 VNET_INTERFACE_COUNTER_TX_ERROR);
Damjan Marion586afd72017-04-05 19:18:20 +0200248 vlib_increment_simple_counter (cm, thread_index,
Dave Barachba868bb2016-08-08 09:51:21 -0400249 rt->sw_if_index, n_buffers);
250
251 return vlib_error_drop_buffers (vm, node, from,
252 /* buffer stride */ 1,
253 n_buffers,
254 VNET_INTERFACE_OUTPUT_NEXT_DROP,
255 node->node_index,
256 VNET_INTERFACE_OUTPUT_ERROR_INTERFACE_DOWN);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700257 }
258
259 from_end = from + n_buffers;
260
261 /* Total byte count of all buffers. */
262 n_bytes = 0;
263 n_packets = 0;
264
Damjan Marion152e21d2016-11-29 14:55:43 +0100265 /* interface-output feature arc handling */
266 if (PREDICT_FALSE (vnet_have_features (arc, rt->sw_if_index)))
267 {
268 vnet_feature_config_main_t *fcm;
269 fcm = vnet_feature_get_config_main (arc);
270 current_config_index = vnet_get_feature_config_index (arc,
271 rt->sw_if_index);
272 vnet_get_config_data (&fcm->config_main, &current_config_index,
273 &next_index, 0);
274 }
275
Ed Warnickecb9cada2015-12-08 15:45:58 -0700276 while (from < from_end)
277 {
278 /* Get new next frame since previous incomplete frame may have less
Dave Barachba868bb2016-08-08 09:51:21 -0400279 than VNET_FRAME_SIZE vectors in it. */
280 vlib_get_new_next_frame (vm, node, next_index, to_tx, n_left_to_tx);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700281
Damjan Marion363640d2017-03-06 11:53:10 +0100282 while (from + 8 <= from_end && n_left_to_tx >= 4)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700283 {
Damjan Marion363640d2017-03-06 11:53:10 +0100284 u32 bi0, bi1, bi2, bi3;
285 vlib_buffer_t *b0, *b1, *b2, *b3;
286 u32 tx_swif0, tx_swif1, tx_swif2, tx_swif3;
Dave Barach2c0a4f42017-06-29 09:30:15 -0400287 u32 or_flags;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700288
289 /* Prefetch next iteration. */
Damjan Marion363640d2017-03-06 11:53:10 +0100290 vlib_prefetch_buffer_with_index (vm, from[4], LOAD);
291 vlib_prefetch_buffer_with_index (vm, from[5], LOAD);
292 vlib_prefetch_buffer_with_index (vm, from[6], LOAD);
293 vlib_prefetch_buffer_with_index (vm, from[7], LOAD);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700294
295 bi0 = from[0];
296 bi1 = from[1];
Damjan Marion363640d2017-03-06 11:53:10 +0100297 bi2 = from[2];
298 bi3 = from[3];
Ed Warnickecb9cada2015-12-08 15:45:58 -0700299 to_tx[0] = bi0;
300 to_tx[1] = bi1;
Damjan Marion363640d2017-03-06 11:53:10 +0100301 to_tx[2] = bi2;
302 to_tx[3] = bi3;
303 from += 4;
304 to_tx += 4;
305 n_left_to_tx -= 4;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700306
307 b0 = vlib_get_buffer (vm, bi0);
308 b1 = vlib_get_buffer (vm, bi1);
Damjan Marion363640d2017-03-06 11:53:10 +0100309 b2 = vlib_get_buffer (vm, bi2);
310 b3 = vlib_get_buffer (vm, bi3);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700311
312 /* Be grumpy about zero length buffers for benefit of
313 driver tx function. */
314 ASSERT (b0->current_length > 0);
315 ASSERT (b1->current_length > 0);
Damjan Marion363640d2017-03-06 11:53:10 +0100316 ASSERT (b2->current_length > 0);
317 ASSERT (b3->current_length > 0);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700318
Dave Barachba868bb2016-08-08 09:51:21 -0400319 n_bytes_b0 = vlib_buffer_length_in_chain (vm, b0);
320 n_bytes_b1 = vlib_buffer_length_in_chain (vm, b1);
Damjan Marion363640d2017-03-06 11:53:10 +0100321 n_bytes_b2 = vlib_buffer_length_in_chain (vm, b2);
322 n_bytes_b3 = vlib_buffer_length_in_chain (vm, b3);
Dave Barachba868bb2016-08-08 09:51:21 -0400323 tx_swif0 = vnet_buffer (b0)->sw_if_index[VLIB_TX];
324 tx_swif1 = vnet_buffer (b1)->sw_if_index[VLIB_TX];
Damjan Marion363640d2017-03-06 11:53:10 +0100325 tx_swif2 = vnet_buffer (b2)->sw_if_index[VLIB_TX];
326 tx_swif3 = vnet_buffer (b3)->sw_if_index[VLIB_TX];
Ed Warnickecb9cada2015-12-08 15:45:58 -0700327
328 n_bytes += n_bytes_b0 + n_bytes_b1;
Damjan Marion363640d2017-03-06 11:53:10 +0100329 n_bytes += n_bytes_b2 + n_bytes_b3;
330 n_packets += 4;
Dave Barachba868bb2016-08-08 09:51:21 -0400331
Damjan Marion152e21d2016-11-29 14:55:43 +0100332 if (PREDICT_FALSE (current_config_index != ~0))
333 {
Damjan Marionaa682a32018-04-26 22:45:40 +0200334 vnet_buffer (b0)->feature_arc_index = arc;
335 vnet_buffer (b1)->feature_arc_index = arc;
336 vnet_buffer (b2)->feature_arc_index = arc;
337 vnet_buffer (b3)->feature_arc_index = arc;
Damjan Marion152e21d2016-11-29 14:55:43 +0100338 b0->current_config_index = current_config_index;
339 b1->current_config_index = current_config_index;
Damjan Marion363640d2017-03-06 11:53:10 +0100340 b2->current_config_index = current_config_index;
341 b3->current_config_index = current_config_index;
Damjan Marion152e21d2016-11-29 14:55:43 +0100342 }
343
Damjan Marion363640d2017-03-06 11:53:10 +0100344 /* update vlan subif tx counts, if required */
Damjan Marion9c6ae5f2016-11-15 23:20:01 +0100345 if (PREDICT_FALSE (tx_swif0 != rt->sw_if_index))
Dave Barachba868bb2016-08-08 09:51:21 -0400346 {
Damjan Marion9c6ae5f2016-11-15 23:20:01 +0100347 vlib_increment_combined_counter (im->combined_sw_if_counters +
348 VNET_INTERFACE_COUNTER_TX,
Damjan Marion586afd72017-04-05 19:18:20 +0200349 thread_index, tx_swif0, 1,
Damjan Marion9c6ae5f2016-11-15 23:20:01 +0100350 n_bytes_b0);
Dave Barachba868bb2016-08-08 09:51:21 -0400351 }
Damjan Marion9c6ae5f2016-11-15 23:20:01 +0100352
Damjan Marion9c6ae5f2016-11-15 23:20:01 +0100353 if (PREDICT_FALSE (tx_swif1 != rt->sw_if_index))
354 {
355
356 vlib_increment_combined_counter (im->combined_sw_if_counters +
357 VNET_INTERFACE_COUNTER_TX,
Damjan Marion586afd72017-04-05 19:18:20 +0200358 thread_index, tx_swif1, 1,
Damjan Marion9c6ae5f2016-11-15 23:20:01 +0100359 n_bytes_b1);
360 }
Damjan Marion363640d2017-03-06 11:53:10 +0100361
362 if (PREDICT_FALSE (tx_swif2 != rt->sw_if_index))
363 {
364
365 vlib_increment_combined_counter (im->combined_sw_if_counters +
366 VNET_INTERFACE_COUNTER_TX,
Damjan Marion586afd72017-04-05 19:18:20 +0200367 thread_index, tx_swif2, 1,
Damjan Marion363640d2017-03-06 11:53:10 +0100368 n_bytes_b2);
369 }
370 if (PREDICT_FALSE (tx_swif3 != rt->sw_if_index))
371 {
372
373 vlib_increment_combined_counter (im->combined_sw_if_counters +
374 VNET_INTERFACE_COUNTER_TX,
Damjan Marion586afd72017-04-05 19:18:20 +0200375 thread_index, tx_swif3, 1,
Damjan Marion363640d2017-03-06 11:53:10 +0100376 n_bytes_b3);
377 }
Dave Barach2c0a4f42017-06-29 09:30:15 -0400378
379 or_flags = b0->flags | b1->flags | b2->flags | b3->flags;
380
381 if (do_tx_offloads)
382 {
383 if (or_flags &
384 (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
385 VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
386 VNET_BUFFER_F_OFFLOAD_IP_CKSUM))
387 {
388 calc_checksums (vm, b0);
389 calc_checksums (vm, b1);
390 calc_checksums (vm, b2);
391 calc_checksums (vm, b3);
392 }
393 }
Ed Warnickecb9cada2015-12-08 15:45:58 -0700394 }
395
396 while (from + 1 <= from_end && n_left_to_tx >= 1)
397 {
398 u32 bi0;
Dave Barachba868bb2016-08-08 09:51:21 -0400399 vlib_buffer_t *b0;
400 u32 tx_swif0;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700401
402 bi0 = from[0];
403 to_tx[0] = bi0;
404 from += 1;
405 to_tx += 1;
406 n_left_to_tx -= 1;
407
408 b0 = vlib_get_buffer (vm, bi0);
409
410 /* Be grumpy about zero length buffers for benefit of
411 driver tx function. */
412 ASSERT (b0->current_length > 0);
413
Dave Barachba868bb2016-08-08 09:51:21 -0400414 n_bytes_b0 = vlib_buffer_length_in_chain (vm, b0);
415 tx_swif0 = vnet_buffer (b0)->sw_if_index[VLIB_TX];
Ed Warnickecb9cada2015-12-08 15:45:58 -0700416 n_bytes += n_bytes_b0;
417 n_packets += 1;
418
Damjan Marion152e21d2016-11-29 14:55:43 +0100419 if (PREDICT_FALSE (current_config_index != ~0))
420 {
Damjan Marionaa682a32018-04-26 22:45:40 +0200421 vnet_buffer (b0)->feature_arc_index = arc;
Damjan Marion152e21d2016-11-29 14:55:43 +0100422 b0->current_config_index = current_config_index;
423 }
424
Damjan Marion9c6ae5f2016-11-15 23:20:01 +0100425 if (PREDICT_FALSE (tx_swif0 != rt->sw_if_index))
Dave Barachba868bb2016-08-08 09:51:21 -0400426 {
Ed Warnickecb9cada2015-12-08 15:45:58 -0700427
Damjan Marion9c6ae5f2016-11-15 23:20:01 +0100428 vlib_increment_combined_counter (im->combined_sw_if_counters +
429 VNET_INTERFACE_COUNTER_TX,
Damjan Marion586afd72017-04-05 19:18:20 +0200430 thread_index, tx_swif0, 1,
Damjan Marion9c6ae5f2016-11-15 23:20:01 +0100431 n_bytes_b0);
Dave Barachba868bb2016-08-08 09:51:21 -0400432 }
Dave Barach2c0a4f42017-06-29 09:30:15 -0400433
434 if (do_tx_offloads)
435 calc_checksums (vm, b0);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700436 }
437
Dave Barachba868bb2016-08-08 09:51:21 -0400438 vlib_put_next_frame (vm, node, next_index, n_left_to_tx);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700439 }
440
441 /* Update main interface stats. */
442 vlib_increment_combined_counter (im->combined_sw_if_counters
Dave Barachba868bb2016-08-08 09:51:21 -0400443 + VNET_INTERFACE_COUNTER_TX,
Damjan Marion586afd72017-04-05 19:18:20 +0200444 thread_index,
Dave Barachba868bb2016-08-08 09:51:21 -0400445 rt->sw_if_index, n_packets, n_bytes);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700446 return n_buffers;
447}
448
Damjan Marion2574d9f2018-06-26 14:45:49 +0200449static uword
Dave Barach2c0a4f42017-06-29 09:30:15 -0400450vnet_interface_output_node (vlib_main_t * vm, vlib_node_runtime_t * node,
451 vlib_frame_t * frame)
452{
453 vnet_main_t *vnm = vnet_get_main ();
454 vnet_hw_interface_t *hi;
455 vnet_interface_output_runtime_t *rt = (void *) node->runtime_data;
456 hi = vnet_get_sup_hw_interface (vnm, rt->sw_if_index);
457
458 if (hi->flags & VNET_HW_INTERFACE_FLAG_SUPPORTS_TX_L4_CKSUM_OFFLOAD)
459 return vnet_interface_output_node_inline (vm, node, frame, vnm, hi,
460 /* do_tx_offloads */ 0);
461 else
462 return vnet_interface_output_node_inline (vm, node, frame, vnm, hi,
463 /* do_tx_offloads */ 1);
464}
465
Damjan Marion8b2247d2016-12-02 08:09:45 +0100466VLIB_NODE_FUNCTION_MULTIARCH_CLONE (vnet_interface_output_node);
467CLIB_MULTIARCH_SELECT_FN (vnet_interface_output_node);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700468
469/* Use buffer's sw_if_index[VNET_TX] to choose output interface. */
470static uword
471vnet_per_buffer_interface_output (vlib_main_t * vm,
472 vlib_node_runtime_t * node,
473 vlib_frame_t * frame)
474{
Dave Barachba868bb2016-08-08 09:51:21 -0400475 vnet_main_t *vnm = vnet_get_main ();
476 u32 n_left_to_next, *from, *to_next;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700477 u32 n_left_from, next_index;
478
479 n_left_from = frame->n_vectors;
480
Damjan Mariona3d59862018-11-10 10:23:00 +0100481 from = vlib_frame_vector_args (frame);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700482 next_index = node->cached_next_index;
483
484 while (n_left_from > 0)
485 {
486 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
487
488 while (n_left_from >= 4 && n_left_to_next >= 2)
489 {
490 u32 bi0, bi1, next0, next1;
Dave Barachba868bb2016-08-08 09:51:21 -0400491 vlib_buffer_t *b0, *b1;
492 vnet_hw_interface_t *hi0, *hi1;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700493
494 /* Prefetch next iteration. */
495 vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
496 vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
497
498 bi0 = from[0];
499 bi1 = from[1];
500 to_next[0] = bi0;
501 to_next[1] = bi1;
502 from += 2;
503 to_next += 2;
504 n_left_to_next -= 2;
505 n_left_from -= 2;
506
507 b0 = vlib_get_buffer (vm, bi0);
508 b1 = vlib_get_buffer (vm, bi1);
509
Dave Barachba868bb2016-08-08 09:51:21 -0400510 hi0 =
511 vnet_get_sup_hw_interface (vnm,
512 vnet_buffer (b0)->sw_if_index
513 [VLIB_TX]);
514 hi1 =
515 vnet_get_sup_hw_interface (vnm,
516 vnet_buffer (b1)->sw_if_index
517 [VLIB_TX]);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700518
John Loe5453d02018-01-23 19:21:34 -0500519 next0 = hi0->output_node_next_index;
520 next1 = hi1->output_node_next_index;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700521
Dave Barachba868bb2016-08-08 09:51:21 -0400522 vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next,
523 n_left_to_next, bi0, bi1, next0,
524 next1);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700525 }
526
527 while (n_left_from > 0 && n_left_to_next > 0)
528 {
529 u32 bi0, next0;
Dave Barachba868bb2016-08-08 09:51:21 -0400530 vlib_buffer_t *b0;
531 vnet_hw_interface_t *hi0;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700532
533 bi0 = from[0];
534 to_next[0] = bi0;
535 from += 1;
536 to_next += 1;
537 n_left_to_next -= 1;
538 n_left_from -= 1;
539
540 b0 = vlib_get_buffer (vm, bi0);
541
Dave Barachba868bb2016-08-08 09:51:21 -0400542 hi0 =
543 vnet_get_sup_hw_interface (vnm,
544 vnet_buffer (b0)->sw_if_index
545 [VLIB_TX]);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700546
John Loe5453d02018-01-23 19:21:34 -0500547 next0 = hi0->output_node_next_index;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700548
Dave Barachba868bb2016-08-08 09:51:21 -0400549 vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
550 n_left_to_next, bi0, next0);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700551 }
552
553 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
554 }
555
556 return frame->n_vectors;
557}
558
559always_inline u32
560counter_index (vlib_main_t * vm, vlib_error_t e)
561{
Dave Barachba868bb2016-08-08 09:51:21 -0400562 vlib_node_t *n;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700563 u32 ci, ni;
564
565 ni = vlib_error_get_node (e);
566 n = vlib_get_node (vm, ni);
567
568 ci = vlib_error_get_code (e);
569 ASSERT (ci < n->n_errors);
570
571 ci += n->error_heap_index;
572
573 return ci;
574}
575
Dave Barachba868bb2016-08-08 09:51:21 -0400576static u8 *
577format_vnet_error_trace (u8 * s, va_list * va)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700578{
Dave Barachba868bb2016-08-08 09:51:21 -0400579 vlib_main_t *vm = va_arg (*va, vlib_main_t *);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700580 CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *);
Dave Barachba868bb2016-08-08 09:51:21 -0400581 vlib_error_t *e = va_arg (*va, vlib_error_t *);
582 vlib_node_t *error_node;
583 vlib_error_main_t *em = &vm->error_main;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700584 u32 i;
585
586 error_node = vlib_get_node (vm, vlib_error_get_node (e[0]));
587 i = counter_index (vm, e[0]);
588 s = format (s, "%v: %s", error_node->name, em->error_strings_heap[i]);
589
590 return s;
591}
592
593static void
594trace_errors_with_buffers (vlib_main_t * vm,
Dave Barachba868bb2016-08-08 09:51:21 -0400595 vlib_node_runtime_t * node, vlib_frame_t * frame)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700596{
Dave Barachba868bb2016-08-08 09:51:21 -0400597 u32 n_left, *buffers;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700598
599 buffers = vlib_frame_vector_args (frame);
600 n_left = frame->n_vectors;
Dave Barachba868bb2016-08-08 09:51:21 -0400601
Ed Warnickecb9cada2015-12-08 15:45:58 -0700602 while (n_left >= 4)
603 {
604 u32 bi0, bi1;
Dave Barachba868bb2016-08-08 09:51:21 -0400605 vlib_buffer_t *b0, *b1;
606 vlib_error_t *t0, *t1;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700607
608 /* Prefetch next iteration. */
609 vlib_prefetch_buffer_with_index (vm, buffers[2], LOAD);
610 vlib_prefetch_buffer_with_index (vm, buffers[3], LOAD);
611
612 bi0 = buffers[0];
613 bi1 = buffers[1];
614
615 b0 = vlib_get_buffer (vm, bi0);
616 b1 = vlib_get_buffer (vm, bi1);
617
618 if (b0->flags & VLIB_BUFFER_IS_TRACED)
619 {
620 t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
621 t0[0] = b0->error;
622 }
623 if (b1->flags & VLIB_BUFFER_IS_TRACED)
624 {
625 t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
626 t1[0] = b1->error;
627 }
628 buffers += 2;
629 n_left -= 2;
630 }
631
632 while (n_left >= 1)
633 {
634 u32 bi0;
Dave Barachba868bb2016-08-08 09:51:21 -0400635 vlib_buffer_t *b0;
636 vlib_error_t *t0;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700637
638 bi0 = buffers[0];
639
640 b0 = vlib_get_buffer (vm, bi0);
641
642 if (b0->flags & VLIB_BUFFER_IS_TRACED)
643 {
644 t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
645 t0[0] = b0->error;
646 }
647 buffers += 1;
648 n_left -= 1;
649 }
650}
651
652static u8 *
653validate_error (vlib_main_t * vm, vlib_error_t * e, u32 index)
654{
655 uword node_index = vlib_error_get_node (e[0]);
656 uword code = vlib_error_get_code (e[0]);
Dave Barachba868bb2016-08-08 09:51:21 -0400657 vlib_node_t *n;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700658
659 if (node_index >= vec_len (vm->node_main.nodes))
660 return format (0, "[%d], node index out of range 0x%x, error 0x%x",
661 index, node_index, e[0]);
662
663 n = vlib_get_node (vm, node_index);
664 if (code >= n->n_errors)
665 return format (0, "[%d], code %d out of range for node %v",
666 index, code, n->name);
667
668 return 0;
669}
670
671static u8 *
672validate_error_frame (vlib_main_t * vm,
Dave Barachba868bb2016-08-08 09:51:21 -0400673 vlib_node_runtime_t * node, vlib_frame_t * f)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700674{
Damjan Mariona3d59862018-11-10 10:23:00 +0100675 u32 *buffers = vlib_frame_vector_args (f);
Dave Barachba868bb2016-08-08 09:51:21 -0400676 vlib_buffer_t *b;
677 u8 *msg = 0;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700678 uword i;
679
680 for (i = 0; i < f->n_vectors; i++)
681 {
682 b = vlib_get_buffer (vm, buffers[i]);
683 msg = validate_error (vm, &b->error, i);
684 if (msg)
685 return msg;
686 }
687
688 return msg;
689}
690
Dave Barachba868bb2016-08-08 09:51:21 -0400691typedef enum
692{
Ed Warnickecb9cada2015-12-08 15:45:58 -0700693 VNET_ERROR_DISPOSITION_DROP,
694 VNET_ERROR_DISPOSITION_PUNT,
695 VNET_ERROR_N_DISPOSITION,
696} vnet_error_disposition_t;
697
698always_inline void
699do_packet (vlib_main_t * vm, vlib_error_t a)
700{
Dave Barachba868bb2016-08-08 09:51:21 -0400701 vlib_error_main_t *em = &vm->error_main;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700702 u32 i = counter_index (vm, a);
703 em->counters[i] += 1;
704 vlib_error_elog_count (vm, i, 1);
705}
Dave Barachba868bb2016-08-08 09:51:21 -0400706
Ed Warnickecb9cada2015-12-08 15:45:58 -0700707static_always_inline uword
708process_drop_punt (vlib_main_t * vm,
709 vlib_node_runtime_t * node,
Dave Barachba868bb2016-08-08 09:51:21 -0400710 vlib_frame_t * frame, vnet_error_disposition_t disposition)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700711{
Dave Barachba868bb2016-08-08 09:51:21 -0400712 vnet_main_t *vnm = vnet_get_main ();
713 vlib_error_main_t *em = &vm->error_main;
714 u32 *buffers, *first_buffer;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700715 vlib_error_t current_error;
716 u32 current_counter_index, n_errors_left;
717 u32 current_sw_if_index, n_errors_current_sw_if_index;
718 u64 current_counter;
Dave Barachba868bb2016-08-08 09:51:21 -0400719 vlib_simple_counter_main_t *cm;
Damjan Marion586afd72017-04-05 19:18:20 +0200720 u32 thread_index = vm->thread_index;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700721
722 static vlib_error_t memory[VNET_ERROR_N_DISPOSITION];
723 static char memory_init[VNET_ERROR_N_DISPOSITION];
724
Damjan Mariona3d59862018-11-10 10:23:00 +0100725 buffers = vlib_frame_vector_args (frame);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700726 first_buffer = buffers;
727
728 {
Dave Barachba868bb2016-08-08 09:51:21 -0400729 vlib_buffer_t *b = vlib_get_buffer (vm, first_buffer[0]);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700730
Dave Barachba868bb2016-08-08 09:51:21 -0400731 if (!memory_init[disposition])
Ed Warnickecb9cada2015-12-08 15:45:58 -0700732 {
733 memory_init[disposition] = 1;
734 memory[disposition] = b->error;
735 }
736
737 current_sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
738 n_errors_current_sw_if_index = 0;
739 }
740
741 current_error = memory[disposition];
742 current_counter_index = counter_index (vm, memory[disposition]);
743 current_counter = em->counters[current_counter_index];
744
745 if (node->flags & VLIB_NODE_FLAG_TRACE)
746 trace_errors_with_buffers (vm, node, frame);
Dave Barachba868bb2016-08-08 09:51:21 -0400747
Ed Warnickecb9cada2015-12-08 15:45:58 -0700748 n_errors_left = frame->n_vectors;
749 cm = vec_elt_at_index (vnm->interface_main.sw_if_counters,
750 (disposition == VNET_ERROR_DISPOSITION_PUNT
751 ? VNET_INTERFACE_COUNTER_PUNT
752 : VNET_INTERFACE_COUNTER_DROP));
753
754 while (n_errors_left >= 2)
755 {
Dave Barachba868bb2016-08-08 09:51:21 -0400756 vlib_buffer_t *b0, *b1;
757 vnet_sw_interface_t *sw_if0, *sw_if1;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700758 vlib_error_t e0, e1;
759 u32 bi0, bi1;
760 u32 sw_if_index0, sw_if_index1;
761
762 bi0 = buffers[0];
763 bi1 = buffers[1];
764
765 buffers += 2;
766 n_errors_left -= 2;
767
768 b0 = vlib_get_buffer (vm, bi0);
769 b1 = vlib_get_buffer (vm, bi1);
770
771 e0 = b0->error;
772 e1 = b1->error;
773
774 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
775 sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
776
777 /* Speculate that sw_if_index == sw_if_index[01]. */
778 n_errors_current_sw_if_index += 2;
779
780 /* Speculatively assume all 2 (node, code) pairs are equal
Dave Barachba868bb2016-08-08 09:51:21 -0400781 to current (node, code). */
Ed Warnickecb9cada2015-12-08 15:45:58 -0700782 current_counter += 2;
783
784 if (PREDICT_FALSE (e0 != current_error
785 || e1 != current_error
786 || sw_if_index0 != current_sw_if_index
787 || sw_if_index1 != current_sw_if_index))
788 {
789 current_counter -= 2;
790 n_errors_current_sw_if_index -= 2;
791
Damjan Marion586afd72017-04-05 19:18:20 +0200792 vlib_increment_simple_counter (cm, thread_index, sw_if_index0, 1);
793 vlib_increment_simple_counter (cm, thread_index, sw_if_index1, 1);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700794
795 /* Increment super-interface drop/punt counters for
796 sub-interfaces. */
797 sw_if0 = vnet_get_sw_interface (vnm, sw_if_index0);
798 vlib_increment_simple_counter
Damjan Marion586afd72017-04-05 19:18:20 +0200799 (cm, thread_index, sw_if0->sup_sw_if_index,
Dave Barachba868bb2016-08-08 09:51:21 -0400800 sw_if0->sup_sw_if_index != sw_if_index0);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700801
802 sw_if1 = vnet_get_sw_interface (vnm, sw_if_index1);
803 vlib_increment_simple_counter
Damjan Marion586afd72017-04-05 19:18:20 +0200804 (cm, thread_index, sw_if1->sup_sw_if_index,
Dave Barachba868bb2016-08-08 09:51:21 -0400805 sw_if1->sup_sw_if_index != sw_if_index1);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700806
807 em->counters[current_counter_index] = current_counter;
808 do_packet (vm, e0);
809 do_packet (vm, e1);
810
811 /* For 2 repeated errors, change current error. */
812 if (e0 == e1 && e1 != current_error)
813 {
814 current_error = e0;
815 current_counter_index = counter_index (vm, e0);
816 }
817 current_counter = em->counters[current_counter_index];
818 }
819 }
820
821 while (n_errors_left >= 1)
822 {
Dave Barachba868bb2016-08-08 09:51:21 -0400823 vlib_buffer_t *b0;
824 vnet_sw_interface_t *sw_if0;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700825 vlib_error_t e0;
826 u32 bi0, sw_if_index0;
827
828 bi0 = buffers[0];
829
830 buffers += 1;
831 n_errors_left -= 1;
832 current_counter += 1;
833
834 b0 = vlib_get_buffer (vm, bi0);
835 e0 = b0->error;
836
837 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
838
839 /* Increment drop/punt counters. */
Damjan Marion586afd72017-04-05 19:18:20 +0200840 vlib_increment_simple_counter (cm, thread_index, sw_if_index0, 1);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700841
842 /* Increment super-interface drop/punt counters for sub-interfaces. */
843 sw_if0 = vnet_get_sw_interface (vnm, sw_if_index0);
Damjan Marion586afd72017-04-05 19:18:20 +0200844 vlib_increment_simple_counter (cm, thread_index,
845 sw_if0->sup_sw_if_index,
Ed Warnickecb9cada2015-12-08 15:45:58 -0700846 sw_if0->sup_sw_if_index != sw_if_index0);
847
848 if (PREDICT_FALSE (e0 != current_error))
849 {
850 current_counter -= 1;
851
852 vlib_error_elog_count (vm, current_counter_index,
853 (current_counter
854 - em->counters[current_counter_index]));
Dave Barachba868bb2016-08-08 09:51:21 -0400855
Ed Warnickecb9cada2015-12-08 15:45:58 -0700856 em->counters[current_counter_index] = current_counter;
857
858 do_packet (vm, e0);
859 current_error = e0;
860 current_counter_index = counter_index (vm, e0);
861 current_counter = em->counters[current_counter_index];
862 }
863 }
864
865 if (n_errors_current_sw_if_index > 0)
866 {
Dave Barachba868bb2016-08-08 09:51:21 -0400867 vnet_sw_interface_t *si;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700868
Damjan Marion586afd72017-04-05 19:18:20 +0200869 vlib_increment_simple_counter (cm, thread_index, current_sw_if_index,
Ed Warnickecb9cada2015-12-08 15:45:58 -0700870 n_errors_current_sw_if_index);
871
872 si = vnet_get_sw_interface (vnm, current_sw_if_index);
873 if (si->sup_sw_if_index != current_sw_if_index)
Damjan Marion586afd72017-04-05 19:18:20 +0200874 vlib_increment_simple_counter (cm, thread_index, si->sup_sw_if_index,
Dave Barachba868bb2016-08-08 09:51:21 -0400875 n_errors_current_sw_if_index);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700876 }
877
878 vlib_error_elog_count (vm, current_counter_index,
879 (current_counter
880 - em->counters[current_counter_index]));
881
882 /* Return cached counter. */
883 em->counters[current_counter_index] = current_counter;
884
885 /* Save memory for next iteration. */
886 memory[disposition] = current_error;
887
Dave Barachba868bb2016-08-08 09:51:21 -0400888 if (disposition == VNET_ERROR_DISPOSITION_DROP || !vm->os_punt_frame)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700889 {
Dave Barachba868bb2016-08-08 09:51:21 -0400890 vlib_buffer_free (vm, first_buffer, frame->n_vectors);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700891
892 /* If there is no punt function, free the frame as well. */
Dave Barachba868bb2016-08-08 09:51:21 -0400893 if (disposition == VNET_ERROR_DISPOSITION_PUNT && !vm->os_punt_frame)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700894 vlib_frame_free (vm, node, frame);
895 }
896 else
897 vm->os_punt_frame (vm, node, frame);
898
899 return frame->n_vectors;
900}
901
Dave Barachba868bb2016-08-08 09:51:21 -0400902static inline void
903pcap_drop_trace (vlib_main_t * vm,
904 vnet_interface_main_t * im, vlib_frame_t * f)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700905{
Dave Barachba868bb2016-08-08 09:51:21 -0400906 u32 *from;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700907 u32 n_left = f->n_vectors;
Dave Barachba868bb2016-08-08 09:51:21 -0400908 vlib_buffer_t *b0, *p1;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700909 u32 bi0;
910 i16 save_current_data;
911 u16 save_current_length;
912
913 from = vlib_frame_vector_args (f);
914
915 while (n_left > 0)
916 {
917 if (PREDICT_TRUE (n_left > 1))
Dave Barachba868bb2016-08-08 09:51:21 -0400918 {
919 p1 = vlib_get_buffer (vm, from[1]);
920 vlib_prefetch_buffer_header (p1, LOAD);
921 }
922
Ed Warnickecb9cada2015-12-08 15:45:58 -0700923 bi0 = from[0];
924 b0 = vlib_get_buffer (vm, bi0);
925 from++;
926 n_left--;
Dave Barachba868bb2016-08-08 09:51:21 -0400927
Ed Warnickecb9cada2015-12-08 15:45:58 -0700928 /* See if we're pointedly ignoring this specific error */
Dave Barachba868bb2016-08-08 09:51:21 -0400929 if (im->pcap_drop_filter_hash
930 && hash_get (im->pcap_drop_filter_hash, b0->error))
931 continue;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700932
933 /* Trace all drops, or drops received on a specific interface */
934 if (im->pcap_sw_if_index == 0 ||
Dave Barachba868bb2016-08-08 09:51:21 -0400935 im->pcap_sw_if_index == vnet_buffer (b0)->sw_if_index[VLIB_RX])
936 {
937 save_current_data = b0->current_data;
938 save_current_length = b0->current_length;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700939
Dave Barachba868bb2016-08-08 09:51:21 -0400940 /*
941 * Typically, we'll need to rewind the buffer
942 */
943 if (b0->current_data > 0)
944 vlib_buffer_advance (b0, (word) - b0->current_data);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700945
Dave Barachba868bb2016-08-08 09:51:21 -0400946 pcap_add_buffer (&im->pcap_main, vm, bi0, 512);
947
948 b0->current_data = save_current_data;
949 b0->current_length = save_current_length;
950 }
Ed Warnickecb9cada2015-12-08 15:45:58 -0700951 }
952}
953
Dave Barachba868bb2016-08-08 09:51:21 -0400954void
955vnet_pcap_drop_trace_filter_add_del (u32 error_index, int is_add)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700956{
Dave Barachba868bb2016-08-08 09:51:21 -0400957 vnet_interface_main_t *im = &vnet_get_main ()->interface_main;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700958
959 if (im->pcap_drop_filter_hash == 0)
Dave Barachba868bb2016-08-08 09:51:21 -0400960 im->pcap_drop_filter_hash = hash_create (0, sizeof (uword));
Ed Warnickecb9cada2015-12-08 15:45:58 -0700961
962 if (is_add)
963 hash_set (im->pcap_drop_filter_hash, error_index, 1);
964 else
965 hash_unset (im->pcap_drop_filter_hash, error_index);
966}
967
968static uword
969process_drop (vlib_main_t * vm,
Dave Barachba868bb2016-08-08 09:51:21 -0400970 vlib_node_runtime_t * node, vlib_frame_t * frame)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700971{
Dave Barachba868bb2016-08-08 09:51:21 -0400972 vnet_interface_main_t *im = &vnet_get_main ()->interface_main;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700973
974 if (PREDICT_FALSE (im->drop_pcap_enable))
975 pcap_drop_trace (vm, im, frame);
976
977 return process_drop_punt (vm, node, frame, VNET_ERROR_DISPOSITION_DROP);
978}
979
980static uword
981process_punt (vlib_main_t * vm,
Dave Barachba868bb2016-08-08 09:51:21 -0400982 vlib_node_runtime_t * node, vlib_frame_t * frame)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700983{
984 return process_drop_punt (vm, node, frame, VNET_ERROR_DISPOSITION_PUNT);
985}
986
Dave Barachba868bb2016-08-08 09:51:21 -0400987/* *INDENT-OFF* */
Ed Warnickecb9cada2015-12-08 15:45:58 -0700988VLIB_REGISTER_NODE (drop_buffers,static) = {
989 .function = process_drop,
990 .name = "error-drop",
991 .flags = VLIB_NODE_FLAG_IS_DROP,
992 .vector_size = sizeof (u32),
993 .format_trace = format_vnet_error_trace,
994 .validate_frame = validate_error_frame,
995};
Dave Barachba868bb2016-08-08 09:51:21 -0400996/* *INDENT-ON* */
Ed Warnickecb9cada2015-12-08 15:45:58 -0700997
Dave Barachba868bb2016-08-08 09:51:21 -0400998VLIB_NODE_FUNCTION_MULTIARCH (drop_buffers, process_drop);
Damjan Marion1c80e832016-05-11 23:07:18 +0200999
Dave Barachba868bb2016-08-08 09:51:21 -04001000/* *INDENT-OFF* */
Ed Warnickecb9cada2015-12-08 15:45:58 -07001001VLIB_REGISTER_NODE (punt_buffers,static) = {
1002 .function = process_punt,
1003 .flags = (VLIB_NODE_FLAG_FRAME_NO_FREE_AFTER_DISPATCH
1004 | VLIB_NODE_FLAG_IS_PUNT),
1005 .name = "error-punt",
1006 .vector_size = sizeof (u32),
1007 .format_trace = format_vnet_error_trace,
1008 .validate_frame = validate_error_frame,
1009};
Dave Barachba868bb2016-08-08 09:51:21 -04001010/* *INDENT-ON* */
Ed Warnickecb9cada2015-12-08 15:45:58 -07001011
Dave Barachba868bb2016-08-08 09:51:21 -04001012VLIB_NODE_FUNCTION_MULTIARCH (punt_buffers, process_punt);
Damjan Marion1c80e832016-05-11 23:07:18 +02001013
Dave Barachba868bb2016-08-08 09:51:21 -04001014/* *INDENT-OFF* */
Ed Warnickecb9cada2015-12-08 15:45:58 -07001015VLIB_REGISTER_NODE (vnet_per_buffer_interface_output_node,static) = {
1016 .function = vnet_per_buffer_interface_output,
1017 .name = "interface-output",
1018 .vector_size = sizeof (u32),
1019};
Dave Barachba868bb2016-08-08 09:51:21 -04001020/* *INDENT-ON* */
Ed Warnickecb9cada2015-12-08 15:45:58 -07001021
Dave Barach87040972019-01-24 15:12:53 -05001022/* Convenience node to drop a vector of buffers with a "misc error". */
1023static uword
1024misc_drop_buffers (vlib_main_t * vm,
1025 vlib_node_runtime_t * node, vlib_frame_t * frame)
1026{
1027 return vlib_error_drop_buffers (vm, node, vlib_frame_vector_args (frame),
1028 /* buffer stride */ 1,
1029 frame->n_vectors,
1030 /* next */ 0,
1031 node->node_index,
1032 /* error */ 0);
1033}
1034
1035static char *misc_drop_buffers_error_strings[] = {
1036 [0] = "misc. errors",
1037};
1038
1039/* *INDENT-OFF* */
1040VLIB_REGISTER_NODE (misc_drop_buffers_node,static) = {
1041 .function = misc_drop_buffers,
1042 .name = "misc-drop-buffers",
1043 .vector_size = sizeof (u32),
1044 .n_errors = 1,
1045 .n_next_nodes = 1,
1046 .next_nodes = {
1047 "error-drop",
1048 },
1049 .error_strings = misc_drop_buffers_error_strings,
1050};
1051/* *INDENT-ON* */
1052
Dave Barachba868bb2016-08-08 09:51:21 -04001053VLIB_NODE_FUNCTION_MULTIARCH (vnet_per_buffer_interface_output_node,
1054 vnet_per_buffer_interface_output);
Damjan Marion1c80e832016-05-11 23:07:18 +02001055
Damjan Marion152e21d2016-11-29 14:55:43 +01001056static uword
1057interface_tx_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
1058 vlib_frame_t * from_frame)
1059{
1060 vnet_main_t *vnm = vnet_get_main ();
1061 u32 last_sw_if_index = ~0;
1062 vlib_frame_t *to_frame = 0;
1063 vnet_hw_interface_t *hw = 0;
1064 u32 *from, *to_next = 0;
1065 u32 n_left_from;
1066
1067 from = vlib_frame_vector_args (from_frame);
1068 n_left_from = from_frame->n_vectors;
1069 while (n_left_from > 0)
1070 {
1071 u32 bi0;
1072 vlib_buffer_t *b0;
1073 u32 sw_if_index0;
1074
1075 bi0 = from[0];
1076 from++;
1077 n_left_from--;
1078 b0 = vlib_get_buffer (vm, bi0);
1079 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_TX];
1080
1081 if (PREDICT_FALSE ((last_sw_if_index != sw_if_index0) || to_frame == 0))
1082 {
1083 if (to_frame)
1084 {
1085 hw = vnet_get_sup_hw_interface (vnm, last_sw_if_index);
1086 vlib_put_frame_to_node (vm, hw->tx_node_index, to_frame);
1087 }
1088 last_sw_if_index = sw_if_index0;
1089 hw = vnet_get_sup_hw_interface (vnm, sw_if_index0);
1090 to_frame = vlib_get_frame_to_node (vm, hw->tx_node_index);
1091 to_next = vlib_frame_vector_args (to_frame);
1092 }
1093
1094 to_next[0] = bi0;
1095 to_next++;
1096 to_frame->n_vectors++;
1097 }
1098 vlib_put_frame_to_node (vm, hw->tx_node_index, to_frame);
1099 return from_frame->n_vectors;
1100}
1101
1102/* *INDENT-OFF* */
1103VLIB_REGISTER_NODE (interface_tx, static) = {
1104 .function = interface_tx_node_fn,
1105 .name = "interface-tx",
1106 .vector_size = sizeof (u32),
1107 .n_next_nodes = 1,
1108 .next_nodes = {
1109 [0] = "error-drop",
1110 },
1111};
1112
1113VNET_FEATURE_ARC_INIT (interface_output, static) =
1114{
1115 .arc_name = "interface-output",
1116 .start_nodes = VNET_FEATURES (0),
Dave Baracha25def72018-11-26 11:04:45 -05001117 .last_in_arc = "interface-tx",
Damjan Marion152e21d2016-11-29 14:55:43 +01001118 .arc_index_ptr = &vnet_main.interface_main.output_feature_arc_index,
1119};
1120
Pavel Kotucek3a2a1c42016-12-06 10:10:10 +01001121VNET_FEATURE_INIT (span_tx, static) = {
1122 .arc_name = "interface-output",
1123 .node_name = "span-output",
1124 .runs_before = VNET_FEATURES ("interface-tx"),
1125};
1126
Matthew Smith537eeec2018-04-09 11:49:20 -05001127VNET_FEATURE_INIT (ipsec_if_tx, static) = {
1128 .arc_name = "interface-output",
1129 .node_name = "ipsec-if-output",
1130 .runs_before = VNET_FEATURES ("interface-tx"),
1131};
1132
Damjan Marion152e21d2016-11-29 14:55:43 +01001133VNET_FEATURE_INIT (interface_tx, static) = {
1134 .arc_name = "interface-output",
1135 .node_name = "interface-tx",
1136 .runs_before = 0,
1137};
1138/* *INDENT-ON* */
1139
Ed Warnickecb9cada2015-12-08 15:45:58 -07001140clib_error_t *
1141vnet_per_buffer_interface_output_hw_interface_add_del (vnet_main_t * vnm,
1142 u32 hw_if_index,
1143 u32 is_create)
1144{
Dave Barachba868bb2016-08-08 09:51:21 -04001145 vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001146 u32 next_index;
1147
John Loe5453d02018-01-23 19:21:34 -05001148 if (hi->output_node_index == 0)
1149 return 0;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001150
John Loe5453d02018-01-23 19:21:34 -05001151 next_index = vlib_node_add_next
1152 (vnm->vlib_main, vnet_per_buffer_interface_output_node.index,
1153 hi->output_node_index);
1154 hi->output_node_next_index = next_index;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001155
1156 return 0;
1157}
1158
Dave Barachba868bb2016-08-08 09:51:21 -04001159VNET_HW_INTERFACE_ADD_DEL_FUNCTION
1160 (vnet_per_buffer_interface_output_hw_interface_add_del);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001161
John Loe5453d02018-01-23 19:21:34 -05001162void
1163vnet_set_interface_output_node (vnet_main_t * vnm,
1164 u32 hw_if_index, u32 node_index)
1165{
1166 ASSERT (node_index);
1167 vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index);
1168 u32 next_index = vlib_node_add_next
1169 (vnm->vlib_main, vnet_per_buffer_interface_output_node.index, node_index);
1170 hi->output_node_next_index = next_index;
1171 hi->output_node_index = node_index;
1172}
1173
Ed Warnickecb9cada2015-12-08 15:45:58 -07001174static clib_error_t *
1175pcap_drop_trace_command_fn (vlib_main_t * vm,
Dave Barachba868bb2016-08-08 09:51:21 -04001176 unformat_input_t * input,
1177 vlib_cli_command_t * cmd)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001178{
Dave Barachba868bb2016-08-08 09:51:21 -04001179 vnet_main_t *vnm = vnet_get_main ();
1180 vnet_interface_main_t *im = &vnm->interface_main;
1181 u8 *filename;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001182 u32 max;
1183 int matched = 0;
Dave Barachba868bb2016-08-08 09:51:21 -04001184 clib_error_t *error = 0;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001185
Dave Barachba868bb2016-08-08 09:51:21 -04001186 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001187 {
1188 if (unformat (input, "on"))
Dave Barachba868bb2016-08-08 09:51:21 -04001189 {
1190 if (im->drop_pcap_enable == 0)
1191 {
1192 if (im->pcap_filename == 0)
1193 im->pcap_filename = format (0, "/tmp/drop.pcap%c", 0);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001194
Dave Barachb7b92992018-10-17 10:38:51 -04001195 clib_memset (&im->pcap_main, 0, sizeof (im->pcap_main));
Dave Barachba868bb2016-08-08 09:51:21 -04001196 im->pcap_main.file_name = (char *) im->pcap_filename;
1197 im->pcap_main.n_packets_to_capture = 100;
1198 if (im->pcap_pkts_to_capture)
1199 im->pcap_main.n_packets_to_capture = im->pcap_pkts_to_capture;
1200
1201 im->pcap_main.packet_type = PCAP_PACKET_TYPE_ethernet;
1202 im->drop_pcap_enable = 1;
1203 matched = 1;
1204 vlib_cli_output (vm, "pcap drop capture on...");
1205 }
1206 else
1207 {
1208 vlib_cli_output (vm, "pcap drop capture already on...");
1209 }
1210 matched = 1;
1211 }
Ed Warnickecb9cada2015-12-08 15:45:58 -07001212 else if (unformat (input, "off"))
Dave Barachba868bb2016-08-08 09:51:21 -04001213 {
1214 matched = 1;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001215
Dave Barachba868bb2016-08-08 09:51:21 -04001216 if (im->drop_pcap_enable)
1217 {
1218 vlib_cli_output (vm, "captured %d pkts...",
1219 im->pcap_main.n_packets_captured);
1220 if (im->pcap_main.n_packets_captured)
1221 {
1222 im->pcap_main.n_packets_to_capture =
1223 im->pcap_main.n_packets_captured;
1224 error = pcap_write (&im->pcap_main);
1225 if (error)
1226 clib_error_report (error);
1227 else
1228 vlib_cli_output (vm, "saved to %s...", im->pcap_filename);
1229 }
1230 }
1231 else
1232 {
1233 vlib_cli_output (vm, "pcap drop capture already off...");
1234 }
Ed Warnickecb9cada2015-12-08 15:45:58 -07001235
Dave Barachba868bb2016-08-08 09:51:21 -04001236 im->drop_pcap_enable = 0;
1237 }
Ed Warnickecb9cada2015-12-08 15:45:58 -07001238 else if (unformat (input, "max %d", &max))
Dave Barachba868bb2016-08-08 09:51:21 -04001239 {
1240 im->pcap_pkts_to_capture = max;
1241 matched = 1;
1242 }
Ed Warnickecb9cada2015-12-08 15:45:58 -07001243
Dave Barachba868bb2016-08-08 09:51:21 -04001244 else if (unformat (input, "intfc %U",
1245 unformat_vnet_sw_interface, vnm,
1246 &im->pcap_sw_if_index))
1247 matched = 1;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001248 else if (unformat (input, "intfc any"))
Dave Barachba868bb2016-08-08 09:51:21 -04001249 {
1250 im->pcap_sw_if_index = 0;
1251 matched = 1;
1252 }
Ed Warnickecb9cada2015-12-08 15:45:58 -07001253 else if (unformat (input, "file %s", &filename))
Dave Barachba868bb2016-08-08 09:51:21 -04001254 {
1255 u8 *chroot_filename;
1256 /* Brain-police user path input */
1257 if (strstr ((char *) filename, "..")
1258 || index ((char *) filename, '/'))
1259 {
1260 vlib_cli_output (vm, "illegal characters in filename '%s'",
1261 filename);
1262 continue;
1263 }
Ed Warnickecb9cada2015-12-08 15:45:58 -07001264
Dave Barachba868bb2016-08-08 09:51:21 -04001265 chroot_filename = format (0, "/tmp/%s%c", filename, 0);
1266 vec_free (filename);
1267
1268 if (im->pcap_filename)
1269 vec_free (im->pcap_filename);
Dave Barachba868bb2016-08-08 09:51:21 -04001270 im->pcap_filename = chroot_filename;
jerryian361abef2017-04-10 18:48:07 +08001271 im->pcap_main.file_name = (char *) im->pcap_filename;
Dave Barachba868bb2016-08-08 09:51:21 -04001272 matched = 1;
1273 }
Ed Warnickecb9cada2015-12-08 15:45:58 -07001274 else if (unformat (input, "status"))
Dave Barachba868bb2016-08-08 09:51:21 -04001275 {
1276 if (im->drop_pcap_enable == 0)
1277 {
1278 vlib_cli_output (vm, "pcap drop capture is off...");
1279 continue;
1280 }
Ed Warnickecb9cada2015-12-08 15:45:58 -07001281
Dave Barachba868bb2016-08-08 09:51:21 -04001282 vlib_cli_output (vm, "pcap drop capture: %d of %d pkts...",
1283 im->pcap_main.n_packets_captured,
1284 im->pcap_main.n_packets_to_capture);
1285 matched = 1;
1286 }
Ed Warnickecb9cada2015-12-08 15:45:58 -07001287
1288 else
Dave Barachba868bb2016-08-08 09:51:21 -04001289 break;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001290 }
1291
1292 if (matched == 0)
Dave Barachba868bb2016-08-08 09:51:21 -04001293 return clib_error_return (0, "unknown input `%U'",
1294 format_unformat_error, input);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001295
1296 return 0;
1297}
1298
Dave Barachba868bb2016-08-08 09:51:21 -04001299/* *INDENT-OFF* */
Ed Warnickecb9cada2015-12-08 15:45:58 -07001300VLIB_CLI_COMMAND (pcap_trace_command, static) = {
Dave Barachba868bb2016-08-08 09:51:21 -04001301 .path = "pcap drop trace",
1302 .short_help =
1303 "pcap drop trace on off max <nn> intfc <intfc> file <name> status",
1304 .function = pcap_drop_trace_command_fn,
Ed Warnickecb9cada2015-12-08 15:45:58 -07001305};
Dave Barachba868bb2016-08-08 09:51:21 -04001306/* *INDENT-ON* */
1307
1308/*
1309 * fd.io coding-style-patch-verification: ON
1310 *
1311 * Local Variables:
1312 * eval: (c-set-style "gnu")
1313 * End:
1314 */