Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2015 Cisco and/or its affiliates. |
| 3 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | * you may not use this file except in compliance with the License. |
| 5 | * You may obtain a copy of the License at: |
| 6 | * |
| 7 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | * |
| 9 | * Unless required by applicable law or agreed to in writing, software |
| 10 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | * See the License for the specific language governing permissions and |
| 13 | * limitations under the License. |
| 14 | */ |
| 15 | /* |
| 16 | * interface_output.c: interface output node |
| 17 | * |
| 18 | * Copyright (c) 2008 Eliot Dresselhaus |
| 19 | * |
| 20 | * Permission is hereby granted, free of charge, to any person obtaining |
| 21 | * a copy of this software and associated documentation files (the |
| 22 | * "Software"), to deal in the Software without restriction, including |
| 23 | * without limitation the rights to use, copy, modify, merge, publish, |
| 24 | * distribute, sublicense, and/or sell copies of the Software, and to |
| 25 | * permit persons to whom the Software is furnished to do so, subject to |
| 26 | * the following conditions: |
| 27 | * |
| 28 | * The above copyright notice and this permission notice shall be |
| 29 | * included in all copies or substantial portions of the Software. |
| 30 | * |
| 31 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 32 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 33 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 34 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE |
| 35 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION |
| 36 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION |
| 37 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
| 38 | */ |
| 39 | |
| 40 | #include <vnet/vnet.h> |
Dave Barach | 2c0a4f4 | 2017-06-29 09:30:15 -0400 | [diff] [blame] | 41 | #include <vnet/ip/icmp46_packet.h> |
Dave Barach | 4330c46 | 2020-06-10 17:07:32 -0400 | [diff] [blame] | 42 | #include <vnet/ethernet/packet.h> |
Florin Coras | b040f98 | 2020-10-20 14:59:43 -0700 | [diff] [blame] | 43 | #include <vnet/ip/format.h> |
Dave Barach | 2c0a4f4 | 2017-06-29 09:30:15 -0400 | [diff] [blame] | 44 | #include <vnet/ip/ip4.h> |
| 45 | #include <vnet/ip/ip6.h> |
| 46 | #include <vnet/udp/udp_packet.h> |
Damjan Marion | 152e21d | 2016-11-29 14:55:43 +0100 | [diff] [blame] | 47 | #include <vnet/feature/feature.h> |
Benoît Ganne | 30a8195 | 2021-02-26 13:47:41 +0100 | [diff] [blame] | 48 | #include <vnet/classify/pcap_classify.h> |
Dave Barach | 1bd2c01 | 2020-04-12 08:31:39 -0400 | [diff] [blame] | 49 | #include <vnet/interface_output.h> |
Damjan Marion | d154a17 | 2021-07-13 21:12:41 +0200 | [diff] [blame] | 50 | #include <vppinfra/vector/mask_compare.h> |
| 51 | #include <vppinfra/vector/compress.h> |
Damjan Marion | 4c276f0 | 2021-11-06 13:17:31 +0100 | [diff] [blame] | 52 | #include <vppinfra/vector/count_equal.h> |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 53 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 54 | typedef struct |
| 55 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 56 | u32 sw_if_index; |
Andrew Yourtchenko | 6a7cff7 | 2018-10-12 16:09:22 +0200 | [diff] [blame] | 57 | u32 flags; |
Mohsin Kazmi | 29467b5 | 2019-10-08 19:42:38 +0200 | [diff] [blame] | 58 | u8 data[128 - 2 * sizeof (u32)]; |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 59 | } |
| 60 | interface_output_trace_t; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 61 | |
Filip Tehlar | 6266877 | 2019-03-04 03:33:32 -0800 | [diff] [blame] | 62 | #ifndef CLIB_MARCH_VARIANT |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 63 | u8 * |
| 64 | format_vnet_interface_output_trace (u8 * s, va_list * va) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 65 | { |
| 66 | CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *); |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 67 | vlib_node_t *node = va_arg (*va, vlib_node_t *); |
| 68 | interface_output_trace_t *t = va_arg (*va, interface_output_trace_t *); |
| 69 | vnet_main_t *vnm = vnet_get_main (); |
| 70 | vnet_sw_interface_t *si; |
Christophe Fontaine | d3c008d | 2017-10-02 18:10:54 +0200 | [diff] [blame] | 71 | u32 indent; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 72 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 73 | if (t->sw_if_index != (u32) ~ 0) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 74 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 75 | indent = format_get_indent (s); |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 76 | |
Neale Ranns | 177bbdc | 2016-11-15 09:46:51 +0000 | [diff] [blame] | 77 | if (pool_is_free_index |
| 78 | (vnm->interface_main.sw_interfaces, t->sw_if_index)) |
| 79 | { |
| 80 | /* the interface may have been deleted by the time the trace is printed */ |
Andrew Yourtchenko | 6a7cff7 | 2018-10-12 16:09:22 +0200 | [diff] [blame] | 81 | s = format (s, "sw_if_index: %d ", t->sw_if_index); |
Neale Ranns | 177bbdc | 2016-11-15 09:46:51 +0000 | [diff] [blame] | 82 | } |
| 83 | else |
| 84 | { |
| 85 | si = vnet_get_sw_interface (vnm, t->sw_if_index); |
Andrew Yourtchenko | 6a7cff7 | 2018-10-12 16:09:22 +0200 | [diff] [blame] | 86 | s = |
| 87 | format (s, "%U ", format_vnet_sw_interface_name, vnm, si, |
| 88 | t->flags); |
Neale Ranns | 177bbdc | 2016-11-15 09:46:51 +0000 | [diff] [blame] | 89 | } |
Andrew Yourtchenko | 6a7cff7 | 2018-10-12 16:09:22 +0200 | [diff] [blame] | 90 | s = |
| 91 | format (s, "\n%U%U", format_white_space, indent, |
| 92 | node->format_buffer ? node->format_buffer : format_hex_bytes, |
| 93 | t->data, sizeof (t->data)); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 94 | } |
| 95 | return s; |
| 96 | } |
Benoît Ganne | 7d2094c | 2020-11-09 15:23:52 +0100 | [diff] [blame] | 97 | #endif /* CLIB_MARCH_VARIANT */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 98 | |
| 99 | static void |
| 100 | vnet_interface_output_trace (vlib_main_t * vm, |
| 101 | vlib_node_runtime_t * node, |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 102 | vlib_frame_t * frame, uword n_buffers) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 103 | { |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 104 | u32 n_left, *from; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 105 | |
| 106 | n_left = n_buffers; |
Damjan Marion | a3d5986 | 2018-11-10 10:23:00 +0100 | [diff] [blame] | 107 | from = vlib_frame_vector_args (frame); |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 108 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 109 | while (n_left >= 4) |
| 110 | { |
| 111 | u32 bi0, bi1; |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 112 | vlib_buffer_t *b0, *b1; |
| 113 | interface_output_trace_t *t0, *t1; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 114 | |
| 115 | /* Prefetch next iteration. */ |
| 116 | vlib_prefetch_buffer_with_index (vm, from[2], LOAD); |
| 117 | vlib_prefetch_buffer_with_index (vm, from[3], LOAD); |
| 118 | |
| 119 | bi0 = from[0]; |
| 120 | bi1 = from[1]; |
| 121 | |
| 122 | b0 = vlib_get_buffer (vm, bi0); |
| 123 | b1 = vlib_get_buffer (vm, bi1); |
| 124 | |
| 125 | if (b0->flags & VLIB_BUFFER_IS_TRACED) |
| 126 | { |
| 127 | t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0])); |
| 128 | t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX]; |
Andrew Yourtchenko | 6a7cff7 | 2018-10-12 16:09:22 +0200 | [diff] [blame] | 129 | t0->flags = b0->flags; |
Dave Barach | 178cf49 | 2018-11-13 16:34:13 -0500 | [diff] [blame] | 130 | clib_memcpy_fast (t0->data, vlib_buffer_get_current (b0), |
| 131 | sizeof (t0->data)); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 132 | } |
| 133 | if (b1->flags & VLIB_BUFFER_IS_TRACED) |
| 134 | { |
| 135 | t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0])); |
| 136 | t1->sw_if_index = vnet_buffer (b1)->sw_if_index[VLIB_TX]; |
Andrew Yourtchenko | 6a7cff7 | 2018-10-12 16:09:22 +0200 | [diff] [blame] | 137 | t1->flags = b1->flags; |
Dave Barach | 178cf49 | 2018-11-13 16:34:13 -0500 | [diff] [blame] | 138 | clib_memcpy_fast (t1->data, vlib_buffer_get_current (b1), |
| 139 | sizeof (t1->data)); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 140 | } |
| 141 | from += 2; |
| 142 | n_left -= 2; |
| 143 | } |
| 144 | |
| 145 | while (n_left >= 1) |
| 146 | { |
| 147 | u32 bi0; |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 148 | vlib_buffer_t *b0; |
| 149 | interface_output_trace_t *t0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 150 | |
| 151 | bi0 = from[0]; |
| 152 | |
| 153 | b0 = vlib_get_buffer (vm, bi0); |
| 154 | |
| 155 | if (b0->flags & VLIB_BUFFER_IS_TRACED) |
| 156 | { |
| 157 | t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0])); |
| 158 | t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX]; |
Andrew Yourtchenko | 6a7cff7 | 2018-10-12 16:09:22 +0200 | [diff] [blame] | 159 | t0->flags = b0->flags; |
Dave Barach | 178cf49 | 2018-11-13 16:34:13 -0500 | [diff] [blame] | 160 | clib_memcpy_fast (t0->data, vlib_buffer_get_current (b0), |
| 161 | sizeof (t0->data)); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 162 | } |
| 163 | from += 1; |
| 164 | n_left -= 1; |
| 165 | } |
| 166 | } |
| 167 | |
Damjan Marion | 4d2726e | 2021-03-23 21:05:18 +0100 | [diff] [blame] | 168 | static_always_inline void |
| 169 | vnet_interface_output_handle_offload (vlib_main_t *vm, vlib_buffer_t *b) |
Dave Barach | 2c0a4f4 | 2017-06-29 09:30:15 -0400 | [diff] [blame] | 170 | { |
Aloys Augustin | ac6c528 | 2021-04-15 18:12:51 +0200 | [diff] [blame] | 171 | vnet_calc_checksums_inline (vm, b, b->flags & VNET_BUFFER_F_IS_IP4, |
| 172 | b->flags & VNET_BUFFER_F_IS_IP6); |
Mohsin Kazmi | 376c210 | 2021-11-26 17:04:07 +0100 | [diff] [blame^] | 173 | vnet_calc_outer_checksums_inline (vm, b); |
Damjan Marion | 4d2726e | 2021-03-23 21:05:18 +0100 | [diff] [blame] | 174 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 175 | |
Damjan Marion | 4d2726e | 2021-03-23 21:05:18 +0100 | [diff] [blame] | 176 | static_always_inline uword |
| 177 | vnet_interface_output_node_inline (vlib_main_t *vm, u32 sw_if_index, |
| 178 | vlib_combined_counter_main_t *ccm, |
| 179 | vlib_buffer_t **b, u32 config_index, u8 arc, |
Mohsin Kazmi | 8f3415d | 2021-10-14 12:21:16 +0000 | [diff] [blame] | 180 | u32 n_left, int processing_level) |
Damjan Marion | 4d2726e | 2021-03-23 21:05:18 +0100 | [diff] [blame] | 181 | { |
| 182 | u32 n_bytes = 0; |
| 183 | u32 n_bytes0, n_bytes1, n_bytes2, n_bytes3; |
| 184 | u32 ti = vm->thread_index; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 185 | |
Damjan Marion | 4d2726e | 2021-03-23 21:05:18 +0100 | [diff] [blame] | 186 | while (n_left >= 8) |
Damjan Marion | 152e21d | 2016-11-29 14:55:43 +0100 | [diff] [blame] | 187 | { |
Damjan Marion | 4d2726e | 2021-03-23 21:05:18 +0100 | [diff] [blame] | 188 | u32 or_flags; |
Damjan Marion | 152e21d | 2016-11-29 14:55:43 +0100 | [diff] [blame] | 189 | |
Damjan Marion | 4d2726e | 2021-03-23 21:05:18 +0100 | [diff] [blame] | 190 | /* Prefetch next iteration. */ |
| 191 | vlib_prefetch_buffer_header (b[4], LOAD); |
| 192 | vlib_prefetch_buffer_header (b[5], LOAD); |
| 193 | vlib_prefetch_buffer_header (b[6], LOAD); |
| 194 | vlib_prefetch_buffer_header (b[7], LOAD); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 195 | |
Mohsin Kazmi | 8f3415d | 2021-10-14 12:21:16 +0000 | [diff] [blame] | 196 | if (processing_level >= 1) |
Damjan Marion | d4008cf | 2021-03-24 11:45:51 +0100 | [diff] [blame] | 197 | or_flags = b[0]->flags | b[1]->flags | b[2]->flags | b[3]->flags; |
Damjan Marion | 4d2726e | 2021-03-23 21:05:18 +0100 | [diff] [blame] | 198 | |
| 199 | /* Be grumpy about zero length buffers for benefit of |
| 200 | driver tx function. */ |
| 201 | ASSERT (b[0]->current_length > 0); |
| 202 | ASSERT (b[1]->current_length > 0); |
| 203 | ASSERT (b[2]->current_length > 0); |
| 204 | ASSERT (b[3]->current_length > 0); |
| 205 | |
| 206 | n_bytes += n_bytes0 = vlib_buffer_length_in_chain (vm, b[0]); |
| 207 | n_bytes += n_bytes1 = vlib_buffer_length_in_chain (vm, b[1]); |
| 208 | n_bytes += n_bytes2 = vlib_buffer_length_in_chain (vm, b[2]); |
| 209 | n_bytes += n_bytes3 = vlib_buffer_length_in_chain (vm, b[3]); |
| 210 | |
Mohsin Kazmi | 8f3415d | 2021-10-14 12:21:16 +0000 | [diff] [blame] | 211 | if (processing_level >= 2) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 212 | { |
Damjan Marion | d4008cf | 2021-03-24 11:45:51 +0100 | [diff] [blame] | 213 | u32 tx_swif0, tx_swif1, tx_swif2, tx_swif3; |
| 214 | tx_swif0 = vnet_buffer (b[0])->sw_if_index[VLIB_TX]; |
| 215 | tx_swif1 = vnet_buffer (b[1])->sw_if_index[VLIB_TX]; |
| 216 | tx_swif2 = vnet_buffer (b[2])->sw_if_index[VLIB_TX]; |
| 217 | tx_swif3 = vnet_buffer (b[3])->sw_if_index[VLIB_TX]; |
| 218 | |
| 219 | /* update vlan subif tx counts, if required */ |
| 220 | if (PREDICT_FALSE (tx_swif0 != sw_if_index)) |
| 221 | vlib_increment_combined_counter (ccm, ti, tx_swif0, 1, n_bytes0); |
| 222 | |
| 223 | if (PREDICT_FALSE (tx_swif1 != sw_if_index)) |
| 224 | vlib_increment_combined_counter (ccm, ti, tx_swif1, 1, n_bytes1); |
| 225 | |
| 226 | if (PREDICT_FALSE (tx_swif2 != sw_if_index)) |
| 227 | vlib_increment_combined_counter (ccm, ti, tx_swif2, 1, n_bytes2); |
| 228 | |
| 229 | if (PREDICT_FALSE (tx_swif3 != sw_if_index)) |
| 230 | vlib_increment_combined_counter (ccm, ti, tx_swif3, 1, n_bytes3); |
| 231 | |
| 232 | if (PREDICT_FALSE (config_index != ~0)) |
| 233 | { |
| 234 | vnet_buffer (b[0])->feature_arc_index = arc; |
| 235 | b[0]->current_config_index = config_index; |
| 236 | vnet_buffer (b[1])->feature_arc_index = arc; |
| 237 | b[1]->current_config_index = config_index; |
| 238 | vnet_buffer (b[2])->feature_arc_index = arc; |
| 239 | b[2]->current_config_index = config_index; |
| 240 | vnet_buffer (b[3])->feature_arc_index = arc; |
| 241 | b[3]->current_config_index = config_index; |
| 242 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 243 | } |
| 244 | |
Mohsin Kazmi | 8f3415d | 2021-10-14 12:21:16 +0000 | [diff] [blame] | 245 | if (processing_level >= 1 && (or_flags & VNET_BUFFER_F_OFFLOAD)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 246 | { |
Damjan Marion | 4d2726e | 2021-03-23 21:05:18 +0100 | [diff] [blame] | 247 | vnet_interface_output_handle_offload (vm, b[0]); |
| 248 | vnet_interface_output_handle_offload (vm, b[1]); |
| 249 | vnet_interface_output_handle_offload (vm, b[2]); |
| 250 | vnet_interface_output_handle_offload (vm, b[3]); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 251 | } |
| 252 | |
Damjan Marion | 4d2726e | 2021-03-23 21:05:18 +0100 | [diff] [blame] | 253 | n_left -= 4; |
| 254 | b += 4; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 255 | } |
| 256 | |
Damjan Marion | 4d2726e | 2021-03-23 21:05:18 +0100 | [diff] [blame] | 257 | while (n_left) |
| 258 | { |
Damjan Marion | 4d2726e | 2021-03-23 21:05:18 +0100 | [diff] [blame] | 259 | /* Be grumpy about zero length buffers for benefit of |
| 260 | driver tx function. */ |
| 261 | ASSERT (b[0]->current_length > 0); |
| 262 | |
| 263 | n_bytes += n_bytes0 = vlib_buffer_length_in_chain (vm, b[0]); |
Damjan Marion | 4d2726e | 2021-03-23 21:05:18 +0100 | [diff] [blame] | 264 | |
Mohsin Kazmi | 8f3415d | 2021-10-14 12:21:16 +0000 | [diff] [blame] | 265 | if (processing_level >= 2) |
Damjan Marion | 4d2726e | 2021-03-23 21:05:18 +0100 | [diff] [blame] | 266 | { |
Damjan Marion | d4008cf | 2021-03-24 11:45:51 +0100 | [diff] [blame] | 267 | u32 tx_swif0 = vnet_buffer (b[0])->sw_if_index[VLIB_TX]; |
Damjan Marion | 4d2726e | 2021-03-23 21:05:18 +0100 | [diff] [blame] | 268 | |
Damjan Marion | d4008cf | 2021-03-24 11:45:51 +0100 | [diff] [blame] | 269 | if (PREDICT_FALSE (config_index != ~0)) |
| 270 | { |
| 271 | vnet_buffer (b[0])->feature_arc_index = arc; |
| 272 | b[0]->current_config_index = config_index; |
| 273 | } |
| 274 | |
| 275 | if (PREDICT_FALSE (tx_swif0 != sw_if_index)) |
| 276 | vlib_increment_combined_counter (ccm, ti, tx_swif0, 1, n_bytes0); |
| 277 | } |
Damjan Marion | 4d2726e | 2021-03-23 21:05:18 +0100 | [diff] [blame] | 278 | |
Mohsin Kazmi | 8f3415d | 2021-10-14 12:21:16 +0000 | [diff] [blame] | 279 | if (processing_level >= 1) |
Damjan Marion | 4d2726e | 2021-03-23 21:05:18 +0100 | [diff] [blame] | 280 | vnet_interface_output_handle_offload (vm, b[0]); |
| 281 | |
| 282 | n_left -= 1; |
| 283 | b += 1; |
| 284 | } |
| 285 | |
| 286 | return n_bytes; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 287 | } |
| 288 | |
Benoît Ganne | 1433185 | 2021-07-15 19:21:31 +0200 | [diff] [blame] | 289 | static_always_inline void |
| 290 | vnet_interface_pcap_tx_trace (vlib_main_t *vm, vlib_node_runtime_t *node, |
| 291 | vlib_frame_t *frame, int in_interface_ouput) |
Dave Barach | 5ecd5a5 | 2019-02-25 15:27:28 -0500 | [diff] [blame] | 292 | { |
Damjan Marion | 8fb5add | 2021-03-04 18:41:59 +0100 | [diff] [blame] | 293 | vnet_main_t *vnm = vnet_get_main (); |
Dave Barach | 5ecd5a5 | 2019-02-25 15:27:28 -0500 | [diff] [blame] | 294 | u32 n_left_from, *from; |
Benoît Ganne | 1433185 | 2021-07-15 19:21:31 +0200 | [diff] [blame] | 295 | u32 sw_if_index = ~0, hw_if_index = ~0; |
Damjan Marion | 8fb5add | 2021-03-04 18:41:59 +0100 | [diff] [blame] | 296 | vnet_pcap_t *pp = &vnm->pcap; |
Dave Barach | 5ecd5a5 | 2019-02-25 15:27:28 -0500 | [diff] [blame] | 297 | |
Dave Barach | 3390977 | 2019-09-23 10:27:27 -0400 | [diff] [blame] | 298 | if (PREDICT_TRUE (pp->pcap_tx_enable == 0)) |
Dave Barach | 5ecd5a5 | 2019-02-25 15:27:28 -0500 | [diff] [blame] | 299 | return; |
| 300 | |
Benoît Ganne | 1433185 | 2021-07-15 19:21:31 +0200 | [diff] [blame] | 301 | if (in_interface_ouput) |
| 302 | { |
| 303 | /* interface-output is called right before interface-output-template. |
| 304 | * We only want to capture packets here if there is a per-interface |
| 305 | * filter, in case it matches the sub-interface sw_if_index. |
| 306 | * If there is no per-interface filter configured, let the |
| 307 | * interface-output-template node deal with it */ |
| 308 | if (pp->pcap_sw_if_index == 0) |
| 309 | return; |
| 310 | } |
| 311 | else |
Dave Barach | 5ecd5a5 | 2019-02-25 15:27:28 -0500 | [diff] [blame] | 312 | { |
| 313 | vnet_interface_output_runtime_t *rt = (void *) node->runtime_data; |
| 314 | sw_if_index = rt->sw_if_index; |
| 315 | } |
Dave Barach | 5ecd5a5 | 2019-02-25 15:27:28 -0500 | [diff] [blame] | 316 | |
| 317 | n_left_from = frame->n_vectors; |
| 318 | from = vlib_frame_vector_args (frame); |
| 319 | |
| 320 | while (n_left_from > 0) |
| 321 | { |
| 322 | u32 bi0 = from[0]; |
| 323 | vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0); |
Dave Barach | 9137e54 | 2019-09-13 17:47:50 -0400 | [diff] [blame] | 324 | from++; |
| 325 | n_left_from--; |
| 326 | |
Benoît Ganne | 1433185 | 2021-07-15 19:21:31 +0200 | [diff] [blame] | 327 | if (in_interface_ouput) |
| 328 | { |
| 329 | const u32 sii = vnet_buffer (b0)->sw_if_index[VLIB_TX]; |
| 330 | if (PREDICT_FALSE (sii != sw_if_index)) |
| 331 | { |
| 332 | const vnet_hw_interface_t *hi = |
| 333 | vnet_get_sup_hw_interface (vnm, sii); |
| 334 | hw_if_index = hi->sw_if_index; |
| 335 | sw_if_index = sii; |
| 336 | } |
| 337 | if (hw_if_index == sw_if_index) |
| 338 | continue; /* defer to interface-output-template */ |
| 339 | } |
Dave Barach | 5ecd5a5 | 2019-02-25 15:27:28 -0500 | [diff] [blame] | 340 | |
Benoît Ganne | 30a8195 | 2021-02-26 13:47:41 +0100 | [diff] [blame] | 341 | if (vnet_is_packet_pcaped (pp, b0, sw_if_index)) |
| 342 | pcap_add_buffer (&pp->pcap_main, vm, bi0, pp->max_bytes_per_pkt); |
Dave Barach | 5ecd5a5 | 2019-02-25 15:27:28 -0500 | [diff] [blame] | 343 | } |
| 344 | } |
| 345 | |
Damjan Marion | 1bd6cbb | 2021-04-15 13:12:51 +0200 | [diff] [blame] | 346 | static_always_inline void |
| 347 | store_tx_frame_scalar_data (vnet_hw_if_output_node_runtime_t *r, |
| 348 | vnet_hw_if_tx_frame_t *tf) |
| 349 | { |
| 350 | if (r) |
| 351 | clib_memcpy_fast (tf, &r->frame, sizeof (vnet_hw_if_tx_frame_t)); |
| 352 | } |
| 353 | |
| 354 | static_always_inline void |
| 355 | enqueu_to_tx_node (vlib_main_t *vm, vlib_node_runtime_t *node, |
| 356 | vnet_hw_interface_t *hi, u32 *from, u32 n_vectors) |
| 357 | { |
| 358 | u32 next_index = VNET_INTERFACE_OUTPUT_NEXT_TX; |
| 359 | vnet_hw_if_output_node_runtime_t *r = 0; |
| 360 | u32 n_free, n_copy, *to; |
| 361 | vnet_hw_if_tx_frame_t *tf; |
| 362 | vlib_frame_t *f; |
| 363 | |
| 364 | ASSERT (n_vectors <= VLIB_FRAME_SIZE); |
| 365 | |
| 366 | if (hi->output_node_thread_runtimes) |
| 367 | r = vec_elt_at_index (hi->output_node_thread_runtimes, vm->thread_index); |
| 368 | |
| 369 | f = vlib_get_next_frame_internal (vm, node, next_index, 0); |
| 370 | tf = vlib_frame_scalar_args (f); |
| 371 | |
| 372 | if (f->n_vectors > 0 && (r == 0 || tf->queue_id == r->frame.queue_id)) |
| 373 | { |
| 374 | /* append current next frame */ |
| 375 | n_free = VLIB_FRAME_SIZE - f->n_vectors; |
| 376 | n_copy = clib_min (n_vectors, n_free); |
| 377 | n_vectors -= n_copy; |
| 378 | to = vlib_frame_vector_args (f); |
| 379 | to += f->n_vectors; |
| 380 | } |
| 381 | else |
| 382 | { |
| 383 | if (f->n_vectors > 0) |
| 384 | { |
| 385 | /* current frame doesn't fit - grab empty one */ |
| 386 | f = vlib_get_next_frame_internal (vm, node, next_index, 1); |
| 387 | tf = vlib_frame_scalar_args (f); |
| 388 | } |
| 389 | |
| 390 | /* empty frame - store scalar data */ |
| 391 | store_tx_frame_scalar_data (r, tf); |
| 392 | to = vlib_frame_vector_args (f); |
| 393 | n_free = VLIB_FRAME_SIZE; |
| 394 | n_copy = n_vectors; |
| 395 | n_vectors = 0; |
| 396 | } |
| 397 | |
| 398 | vlib_buffer_copy_indices (to, from, n_copy); |
| 399 | vlib_put_next_frame (vm, node, next_index, n_free - n_copy); |
| 400 | |
| 401 | if (n_vectors == 0) |
| 402 | return; |
| 403 | |
| 404 | /* we have more indices to store, take empty frame */ |
| 405 | from += n_copy; |
| 406 | f = vlib_get_next_frame_internal (vm, node, next_index, 1); |
| 407 | store_tx_frame_scalar_data (r, vlib_frame_scalar_args (f)); |
| 408 | vlib_buffer_copy_indices (vlib_frame_vector_args (f), from, n_vectors); |
| 409 | vlib_put_next_frame (vm, node, next_index, VLIB_FRAME_SIZE - n_vectors); |
| 410 | } |
| 411 | |
Damjan Marion | f91098e | 2021-03-09 16:28:15 +0100 | [diff] [blame] | 412 | VLIB_NODE_FN (vnet_interface_output_node) |
| 413 | (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame) |
Dave Barach | 2c0a4f4 | 2017-06-29 09:30:15 -0400 | [diff] [blame] | 414 | { |
| 415 | vnet_main_t *vnm = vnet_get_main (); |
Damjan Marion | 4d2726e | 2021-03-23 21:05:18 +0100 | [diff] [blame] | 416 | vnet_interface_main_t *im = &vnm->interface_main; |
| 417 | vlib_combined_counter_main_t *ccm; |
Dave Barach | 2c0a4f4 | 2017-06-29 09:30:15 -0400 | [diff] [blame] | 418 | vnet_hw_interface_t *hi; |
Damjan Marion | 3853b26 | 2021-03-23 18:47:34 +0100 | [diff] [blame] | 419 | vnet_sw_interface_t *si; |
Dave Barach | 2c0a4f4 | 2017-06-29 09:30:15 -0400 | [diff] [blame] | 420 | vnet_interface_output_runtime_t *rt = (void *) node->runtime_data; |
Damjan Marion | 3853b26 | 2021-03-23 18:47:34 +0100 | [diff] [blame] | 421 | vlib_buffer_t *bufs[VLIB_FRAME_SIZE]; |
Damjan Marion | 4d2726e | 2021-03-23 21:05:18 +0100 | [diff] [blame] | 422 | u32 n_bytes, n_buffers = frame->n_vectors; |
| 423 | u32 config_index = ~0; |
| 424 | u32 sw_if_index = rt->sw_if_index; |
| 425 | u32 next_index = VNET_INTERFACE_OUTPUT_NEXT_TX; |
| 426 | u32 ti = vm->thread_index; |
| 427 | u8 arc = im->output_feature_arc_index; |
Damjan Marion | d4008cf | 2021-03-24 11:45:51 +0100 | [diff] [blame] | 428 | int arc_or_subif = 0; |
| 429 | int do_tx_offloads = 0; |
Damjan Marion | 3853b26 | 2021-03-23 18:47:34 +0100 | [diff] [blame] | 430 | u32 *from; |
| 431 | |
| 432 | if (node->flags & VLIB_NODE_FLAG_TRACE) |
| 433 | vnet_interface_output_trace (vm, node, frame, n_buffers); |
| 434 | |
| 435 | from = vlib_frame_vector_args (frame); |
| 436 | |
| 437 | if (rt->is_deleted) |
| 438 | return vlib_error_drop_buffers ( |
| 439 | vm, node, from, |
| 440 | /* buffer stride */ 1, n_buffers, VNET_INTERFACE_OUTPUT_NEXT_DROP, |
| 441 | node->node_index, VNET_INTERFACE_OUTPUT_ERROR_INTERFACE_DELETED); |
Dave Barach | 2c0a4f4 | 2017-06-29 09:30:15 -0400 | [diff] [blame] | 442 | |
Benoît Ganne | 1433185 | 2021-07-15 19:21:31 +0200 | [diff] [blame] | 443 | vnet_interface_pcap_tx_trace (vm, node, frame, 0 /* in_interface_ouput */); |
Dave Barach | 5ecd5a5 | 2019-02-25 15:27:28 -0500 | [diff] [blame] | 444 | |
Damjan Marion | 3853b26 | 2021-03-23 18:47:34 +0100 | [diff] [blame] | 445 | vlib_get_buffers (vm, from, bufs, n_buffers); |
| 446 | |
Damjan Marion | 4d2726e | 2021-03-23 21:05:18 +0100 | [diff] [blame] | 447 | si = vnet_get_sw_interface (vnm, sw_if_index); |
| 448 | hi = vnet_get_sup_hw_interface (vnm, sw_if_index); |
Damjan Marion | 3853b26 | 2021-03-23 18:47:34 +0100 | [diff] [blame] | 449 | |
| 450 | if (!(si->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) || |
| 451 | !(hi->flags & VNET_HW_INTERFACE_FLAG_LINK_UP)) |
| 452 | { |
| 453 | vlib_simple_counter_main_t *cm; |
| 454 | |
| 455 | cm = vec_elt_at_index (vnm->interface_main.sw_if_counters, |
| 456 | VNET_INTERFACE_COUNTER_TX_ERROR); |
Damjan Marion | 4d2726e | 2021-03-23 21:05:18 +0100 | [diff] [blame] | 457 | vlib_increment_simple_counter (cm, ti, sw_if_index, n_buffers); |
Damjan Marion | 3853b26 | 2021-03-23 18:47:34 +0100 | [diff] [blame] | 458 | |
| 459 | return vlib_error_drop_buffers ( |
| 460 | vm, node, from, |
| 461 | /* buffer stride */ 1, n_buffers, VNET_INTERFACE_OUTPUT_NEXT_DROP, |
| 462 | node->node_index, VNET_INTERFACE_OUTPUT_ERROR_INTERFACE_DOWN); |
| 463 | } |
| 464 | |
Damjan Marion | 4d2726e | 2021-03-23 21:05:18 +0100 | [diff] [blame] | 465 | /* interface-output feature arc handling */ |
| 466 | if (PREDICT_FALSE (vnet_have_features (arc, sw_if_index))) |
| 467 | { |
| 468 | vnet_feature_config_main_t *fcm; |
| 469 | fcm = vnet_feature_get_config_main (arc); |
| 470 | config_index = vnet_get_feature_config_index (arc, sw_if_index); |
| 471 | vnet_get_config_data (&fcm->config_main, &config_index, &next_index, 0); |
Damjan Marion | d4008cf | 2021-03-24 11:45:51 +0100 | [diff] [blame] | 472 | arc_or_subif = 1; |
Damjan Marion | 4d2726e | 2021-03-23 21:05:18 +0100 | [diff] [blame] | 473 | } |
Damjan Marion | d4008cf | 2021-03-24 11:45:51 +0100 | [diff] [blame] | 474 | else if (hash_elts (hi->sub_interface_sw_if_index_by_id)) |
| 475 | arc_or_subif = 1; |
Damjan Marion | 4d2726e | 2021-03-23 21:05:18 +0100 | [diff] [blame] | 476 | |
| 477 | ccm = im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX; |
| 478 | |
Nathan Skrzypczak | 2e55823 | 2021-05-27 19:34:59 +0200 | [diff] [blame] | 479 | /* if not all three flags IP4_,TCP_,UDP_CKSUM set, do compute them |
| 480 | * here before sending to the interface */ |
| 481 | if ((hi->caps & VNET_HW_INTERFACE_CAP_SUPPORTS_TX_CKSUM) != |
| 482 | VNET_HW_INTERFACE_CAP_SUPPORTS_TX_CKSUM) |
Damjan Marion | d4008cf | 2021-03-24 11:45:51 +0100 | [diff] [blame] | 483 | do_tx_offloads = 1; |
| 484 | |
| 485 | if (do_tx_offloads == 0 && arc_or_subif == 0) |
| 486 | n_bytes = vnet_interface_output_node_inline ( |
Mohsin Kazmi | 8f3415d | 2021-10-14 12:21:16 +0000 | [diff] [blame] | 487 | vm, sw_if_index, ccm, bufs, config_index, arc, n_buffers, 0); |
Damjan Marion | 4f81d44 | 2021-03-30 14:34:44 +0200 | [diff] [blame] | 488 | else if (do_tx_offloads == 1 && arc_or_subif == 0) |
Damjan Marion | d4008cf | 2021-03-24 11:45:51 +0100 | [diff] [blame] | 489 | n_bytes = vnet_interface_output_node_inline ( |
Mohsin Kazmi | 8f3415d | 2021-10-14 12:21:16 +0000 | [diff] [blame] | 490 | vm, sw_if_index, ccm, bufs, config_index, arc, n_buffers, 1); |
Dave Barach | 2c0a4f4 | 2017-06-29 09:30:15 -0400 | [diff] [blame] | 491 | else |
Damjan Marion | d4008cf | 2021-03-24 11:45:51 +0100 | [diff] [blame] | 492 | n_bytes = vnet_interface_output_node_inline ( |
Mohsin Kazmi | 8f3415d | 2021-10-14 12:21:16 +0000 | [diff] [blame] | 493 | vm, sw_if_index, ccm, bufs, config_index, arc, n_buffers, 2); |
Damjan Marion | 4d2726e | 2021-03-23 21:05:18 +0100 | [diff] [blame] | 494 | |
Damjan Marion | 1bd6cbb | 2021-04-15 13:12:51 +0200 | [diff] [blame] | 495 | from = vlib_frame_vector_args (frame); |
| 496 | if (PREDICT_TRUE (next_index == VNET_INTERFACE_OUTPUT_NEXT_TX)) |
| 497 | { |
| 498 | enqueu_to_tx_node (vm, node, hi, from, frame->n_vectors); |
| 499 | } |
| 500 | else |
| 501 | { |
| 502 | vlib_buffer_enqueue_to_single_next (vm, node, from, next_index, |
| 503 | frame->n_vectors); |
| 504 | } |
Damjan Marion | 4d2726e | 2021-03-23 21:05:18 +0100 | [diff] [blame] | 505 | |
| 506 | /* Update main interface stats. */ |
| 507 | vlib_increment_combined_counter (ccm, ti, sw_if_index, n_buffers, n_bytes); |
| 508 | return n_buffers; |
Dave Barach | 2c0a4f4 | 2017-06-29 09:30:15 -0400 | [diff] [blame] | 509 | } |
Benoît Ganne | 7d2094c | 2020-11-09 15:23:52 +0100 | [diff] [blame] | 510 | |
Damjan Marion | f91098e | 2021-03-09 16:28:15 +0100 | [diff] [blame] | 511 | VLIB_REGISTER_NODE (vnet_interface_output_node) = { |
| 512 | .name = "interface-output-template", |
| 513 | .vector_size = sizeof (u32), |
| 514 | }; |
Dave Barach | 2c0a4f4 | 2017-06-29 09:30:15 -0400 | [diff] [blame] | 515 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 516 | /* Use buffer's sw_if_index[VNET_TX] to choose output interface. */ |
Filip Tehlar | 6266877 | 2019-03-04 03:33:32 -0800 | [diff] [blame] | 517 | VLIB_NODE_FN (vnet_per_buffer_interface_output_node) (vlib_main_t * vm, |
| 518 | vlib_node_runtime_t * |
| 519 | node, |
| 520 | vlib_frame_t * frame) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 521 | { |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 522 | vnet_main_t *vnm = vnet_get_main (); |
| 523 | u32 n_left_to_next, *from, *to_next; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 524 | u32 n_left_from, next_index; |
| 525 | |
Benoît Ganne | 1433185 | 2021-07-15 19:21:31 +0200 | [diff] [blame] | 526 | vnet_interface_pcap_tx_trace (vm, node, frame, 1 /* in_interface_ouput */); |
Dave Barach | 5ecd5a5 | 2019-02-25 15:27:28 -0500 | [diff] [blame] | 527 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 528 | n_left_from = frame->n_vectors; |
| 529 | |
Damjan Marion | a3d5986 | 2018-11-10 10:23:00 +0100 | [diff] [blame] | 530 | from = vlib_frame_vector_args (frame); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 531 | next_index = node->cached_next_index; |
| 532 | |
| 533 | while (n_left_from > 0) |
| 534 | { |
| 535 | vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); |
| 536 | |
| 537 | while (n_left_from >= 4 && n_left_to_next >= 2) |
| 538 | { |
| 539 | u32 bi0, bi1, next0, next1; |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 540 | vlib_buffer_t *b0, *b1; |
| 541 | vnet_hw_interface_t *hi0, *hi1; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 542 | |
| 543 | /* Prefetch next iteration. */ |
| 544 | vlib_prefetch_buffer_with_index (vm, from[2], LOAD); |
| 545 | vlib_prefetch_buffer_with_index (vm, from[3], LOAD); |
| 546 | |
| 547 | bi0 = from[0]; |
| 548 | bi1 = from[1]; |
| 549 | to_next[0] = bi0; |
| 550 | to_next[1] = bi1; |
| 551 | from += 2; |
| 552 | to_next += 2; |
| 553 | n_left_to_next -= 2; |
| 554 | n_left_from -= 2; |
| 555 | |
| 556 | b0 = vlib_get_buffer (vm, bi0); |
| 557 | b1 = vlib_get_buffer (vm, bi1); |
| 558 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 559 | hi0 = |
| 560 | vnet_get_sup_hw_interface (vnm, |
| 561 | vnet_buffer (b0)->sw_if_index |
| 562 | [VLIB_TX]); |
| 563 | hi1 = |
| 564 | vnet_get_sup_hw_interface (vnm, |
| 565 | vnet_buffer (b1)->sw_if_index |
| 566 | [VLIB_TX]); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 567 | |
John Lo | e5453d0 | 2018-01-23 19:21:34 -0500 | [diff] [blame] | 568 | next0 = hi0->output_node_next_index; |
| 569 | next1 = hi1->output_node_next_index; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 570 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 571 | vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next, |
| 572 | n_left_to_next, bi0, bi1, next0, |
| 573 | next1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 574 | } |
| 575 | |
| 576 | while (n_left_from > 0 && n_left_to_next > 0) |
| 577 | { |
| 578 | u32 bi0, next0; |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 579 | vlib_buffer_t *b0; |
| 580 | vnet_hw_interface_t *hi0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 581 | |
| 582 | bi0 = from[0]; |
| 583 | to_next[0] = bi0; |
| 584 | from += 1; |
| 585 | to_next += 1; |
| 586 | n_left_to_next -= 1; |
| 587 | n_left_from -= 1; |
| 588 | |
| 589 | b0 = vlib_get_buffer (vm, bi0); |
| 590 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 591 | hi0 = |
| 592 | vnet_get_sup_hw_interface (vnm, |
| 593 | vnet_buffer (b0)->sw_if_index |
| 594 | [VLIB_TX]); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 595 | |
John Lo | e5453d0 | 2018-01-23 19:21:34 -0500 | [diff] [blame] | 596 | next0 = hi0->output_node_next_index; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 597 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 598 | vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, |
| 599 | n_left_to_next, bi0, next0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 600 | } |
| 601 | |
| 602 | vlib_put_next_frame (vm, node, next_index, n_left_to_next); |
| 603 | } |
| 604 | |
| 605 | return frame->n_vectors; |
| 606 | } |
| 607 | |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 608 | typedef struct vnet_error_trace_t_ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 609 | { |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 610 | u32 sw_if_index; |
Dave Barach | 4330c46 | 2020-06-10 17:07:32 -0400 | [diff] [blame] | 611 | i8 details_valid; |
| 612 | u8 is_ip6; |
| 613 | u8 pad[2]; |
| 614 | u16 mactype; |
| 615 | ip46_address_t src, dst; |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 616 | } vnet_error_trace_t; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 617 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 618 | static u8 * |
| 619 | format_vnet_error_trace (u8 * s, va_list * va) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 620 | { |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 621 | CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 622 | CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *); |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 623 | vnet_error_trace_t *t = va_arg (*va, vnet_error_trace_t *); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 624 | |
Dave Barach | 4330c46 | 2020-06-10 17:07:32 -0400 | [diff] [blame] | 625 | /* Normal, non-catchup trace */ |
| 626 | if (t->details_valid == 0) |
| 627 | { |
| 628 | s = format (s, "rx:%U", format_vnet_sw_if_index_name, |
| 629 | vnet_get_main (), t->sw_if_index); |
| 630 | } |
| 631 | else if (t->details_valid == 1) |
| 632 | { |
| 633 | /* The trace capture code didn't understant the mactype */ |
| 634 | s = format (s, "mactype 0x%4x (not decoded)", t->mactype); |
| 635 | } |
| 636 | else if (t->details_valid == 2) |
| 637 | { |
| 638 | /* Dump the src/dst addresses */ |
| 639 | if (t->is_ip6 == 0) |
| 640 | s = format (s, "IP4: %U -> %U", |
| 641 | format_ip4_address, &t->src.ip4, |
| 642 | format_ip4_address, &t->dst.ip4); |
| 643 | else |
| 644 | s = format (s, "IP6: %U -> %U", |
| 645 | format_ip6_address, &t->src.ip6, |
| 646 | format_ip6_address, &t->dst.ip6); |
| 647 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 648 | return s; |
| 649 | } |
| 650 | |
| 651 | static void |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 652 | interface_trace_buffers (vlib_main_t * vm, |
| 653 | vlib_node_runtime_t * node, vlib_frame_t * frame) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 654 | { |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 655 | u32 n_left, *buffers; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 656 | |
| 657 | buffers = vlib_frame_vector_args (frame); |
| 658 | n_left = frame->n_vectors; |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 659 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 660 | while (n_left >= 4) |
| 661 | { |
| 662 | u32 bi0, bi1; |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 663 | vlib_buffer_t *b0, *b1; |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 664 | vnet_error_trace_t *t0, *t1; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 665 | |
| 666 | /* Prefetch next iteration. */ |
| 667 | vlib_prefetch_buffer_with_index (vm, buffers[2], LOAD); |
| 668 | vlib_prefetch_buffer_with_index (vm, buffers[3], LOAD); |
| 669 | |
| 670 | bi0 = buffers[0]; |
| 671 | bi1 = buffers[1]; |
| 672 | |
| 673 | b0 = vlib_get_buffer (vm, bi0); |
| 674 | b1 = vlib_get_buffer (vm, bi1); |
| 675 | |
| 676 | if (b0->flags & VLIB_BUFFER_IS_TRACED) |
| 677 | { |
Dave Barach | 4330c46 | 2020-06-10 17:07:32 -0400 | [diff] [blame] | 678 | t0 = vlib_add_trace (vm, node, b0, |
| 679 | STRUCT_OFFSET_OF (vnet_error_trace_t, pad)); |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 680 | t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX]; |
Dave Barach | 4330c46 | 2020-06-10 17:07:32 -0400 | [diff] [blame] | 681 | t0->details_valid = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 682 | } |
| 683 | if (b1->flags & VLIB_BUFFER_IS_TRACED) |
| 684 | { |
Dave Barach | 4330c46 | 2020-06-10 17:07:32 -0400 | [diff] [blame] | 685 | t1 = vlib_add_trace (vm, node, b1, |
| 686 | STRUCT_OFFSET_OF (vnet_error_trace_t, pad)); |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 687 | t1->sw_if_index = vnet_buffer (b1)->sw_if_index[VLIB_RX]; |
Dave Barach | 4330c46 | 2020-06-10 17:07:32 -0400 | [diff] [blame] | 688 | t1->details_valid = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 689 | } |
| 690 | buffers += 2; |
| 691 | n_left -= 2; |
| 692 | } |
| 693 | |
| 694 | while (n_left >= 1) |
| 695 | { |
| 696 | u32 bi0; |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 697 | vlib_buffer_t *b0; |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 698 | vnet_error_trace_t *t0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 699 | |
| 700 | bi0 = buffers[0]; |
| 701 | |
| 702 | b0 = vlib_get_buffer (vm, bi0); |
| 703 | |
| 704 | if (b0->flags & VLIB_BUFFER_IS_TRACED) |
| 705 | { |
Dave Barach | 4330c46 | 2020-06-10 17:07:32 -0400 | [diff] [blame] | 706 | t0 = vlib_add_trace (vm, node, b0, |
| 707 | STRUCT_OFFSET_OF (vnet_error_trace_t, pad)); |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 708 | t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX]; |
Dave Barach | 4330c46 | 2020-06-10 17:07:32 -0400 | [diff] [blame] | 709 | t0->details_valid = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 710 | } |
| 711 | buffers += 1; |
| 712 | n_left -= 1; |
| 713 | } |
| 714 | } |
| 715 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 716 | typedef enum |
| 717 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 718 | VNET_ERROR_DISPOSITION_DROP, |
| 719 | VNET_ERROR_DISPOSITION_PUNT, |
| 720 | VNET_ERROR_N_DISPOSITION, |
| 721 | } vnet_error_disposition_t; |
| 722 | |
Dave Barach | 4330c46 | 2020-06-10 17:07:32 -0400 | [diff] [blame] | 723 | static void |
| 724 | drop_catchup_trace (vlib_main_t * vm, |
| 725 | vlib_node_runtime_t * node, vlib_buffer_t * b) |
| 726 | { |
| 727 | /* Can we safely rewind the buffer? If not, fagedaboudit */ |
| 728 | if (b->flags & VNET_BUFFER_F_L2_HDR_OFFSET_VALID) |
| 729 | { |
| 730 | vnet_error_trace_t *t; |
| 731 | ip4_header_t *ip4; |
| 732 | ip6_header_t *ip6; |
| 733 | ethernet_header_t *eh; |
| 734 | i16 delta; |
| 735 | |
| 736 | t = vlib_add_trace (vm, node, b, sizeof (*t)); |
| 737 | delta = vnet_buffer (b)->l2_hdr_offset - b->current_data; |
| 738 | vlib_buffer_advance (b, delta); |
| 739 | |
| 740 | eh = vlib_buffer_get_current (b); |
| 741 | /* Save mactype */ |
| 742 | t->mactype = clib_net_to_host_u16 (eh->type); |
| 743 | t->details_valid = 1; |
| 744 | switch (t->mactype) |
| 745 | { |
| 746 | case ETHERNET_TYPE_IP4: |
| 747 | ip4 = (void *) (eh + 1); |
| 748 | t->details_valid = 2; |
| 749 | t->is_ip6 = 0; |
| 750 | t->src.ip4.as_u32 = ip4->src_address.as_u32; |
| 751 | t->dst.ip4.as_u32 = ip4->dst_address.as_u32; |
| 752 | break; |
| 753 | |
| 754 | case ETHERNET_TYPE_IP6: |
| 755 | ip6 = (void *) (eh + 1); |
| 756 | t->details_valid = 2; |
| 757 | t->is_ip6 = 1; |
| 758 | clib_memcpy_fast (t->src.as_u8, ip6->src_address.as_u8, |
| 759 | sizeof (ip6_address_t)); |
| 760 | clib_memcpy_fast (t->dst.as_u8, ip6->dst_address.as_u8, |
| 761 | sizeof (ip6_address_t)); |
| 762 | break; |
| 763 | |
| 764 | default: |
| 765 | /* Dunno, do nothing, leave details_valid alone */ |
| 766 | break; |
| 767 | } |
| 768 | /* Restore current data (probably unnecessary) */ |
| 769 | vlib_buffer_advance (b, -delta); |
| 770 | } |
| 771 | } |
| 772 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 773 | static_always_inline uword |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 774 | interface_drop_punt (vlib_main_t * vm, |
| 775 | vlib_node_runtime_t * node, |
| 776 | vlib_frame_t * frame, |
| 777 | vnet_error_disposition_t disposition) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 778 | { |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 779 | u32 *from, n_left, thread_index, *sw_if_index; |
| 780 | vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b; |
| 781 | u32 sw_if_indices[VLIB_FRAME_SIZE]; |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 782 | vlib_simple_counter_main_t *cm; |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 783 | u16 nexts[VLIB_FRAME_SIZE]; |
Dave Barach | 4330c46 | 2020-06-10 17:07:32 -0400 | [diff] [blame] | 784 | u32 n_trace; |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 785 | vnet_main_t *vnm; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 786 | |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 787 | vnm = vnet_get_main (); |
| 788 | thread_index = vm->thread_index; |
| 789 | from = vlib_frame_vector_args (frame); |
| 790 | n_left = frame->n_vectors; |
| 791 | b = bufs; |
| 792 | sw_if_index = sw_if_indices; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 793 | |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 794 | vlib_get_buffers (vm, from, bufs, n_left); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 795 | |
Dave Barach | 4330c46 | 2020-06-10 17:07:32 -0400 | [diff] [blame] | 796 | /* "trace add error-drop NNN?" */ |
| 797 | if (PREDICT_FALSE ((n_trace = vlib_get_trace_count (vm, node)))) |
| 798 | { |
| 799 | /* If pkts aren't otherwise traced... */ |
| 800 | if ((node->flags & VLIB_NODE_FLAG_TRACE) == 0) |
| 801 | { |
| 802 | /* Trace them from here */ |
| 803 | node->flags |= VLIB_NODE_FLAG_TRACE; |
| 804 | while (n_trace && n_left) |
| 805 | { |
Benoît Ganne | 9a3973e | 2020-10-02 19:36:57 +0200 | [diff] [blame] | 806 | if (PREDICT_TRUE |
| 807 | (vlib_trace_buffer (vm, node, 0 /* next_index */ , b[0], |
| 808 | 0 /* follow chain */ ))) |
| 809 | { |
| 810 | /* |
| 811 | * Here we have a wireshark dissector problem. |
| 812 | * Packets may be well-formed, or not. We |
| 813 | * must not blow chunks in any case. |
| 814 | * |
| 815 | * Try to produce trace records which will help |
| 816 | * folks understand what's going on. |
| 817 | */ |
| 818 | drop_catchup_trace (vm, node, b[0]); |
| 819 | n_trace--; |
| 820 | } |
Dave Barach | 4330c46 | 2020-06-10 17:07:32 -0400 | [diff] [blame] | 821 | n_left--; |
| 822 | b++; |
| 823 | } |
| 824 | } |
| 825 | |
| 826 | vlib_set_trace_count (vm, node, n_trace); |
| 827 | b = bufs; |
| 828 | n_left = frame->n_vectors; |
| 829 | } |
| 830 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 831 | if (node->flags & VLIB_NODE_FLAG_TRACE) |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 832 | interface_trace_buffers (vm, node, frame); |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 833 | |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 834 | /* All going to drop regardless, this is just a counting exercise */ |
| 835 | clib_memset (nexts, 0, sizeof (nexts)); |
| 836 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 837 | cm = vec_elt_at_index (vnm->interface_main.sw_if_counters, |
| 838 | (disposition == VNET_ERROR_DISPOSITION_PUNT |
| 839 | ? VNET_INTERFACE_COUNTER_PUNT |
| 840 | : VNET_INTERFACE_COUNTER_DROP)); |
| 841 | |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 842 | /* collect the array of interfaces first ... */ |
| 843 | while (n_left >= 4) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 844 | { |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 845 | if (n_left >= 12) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 846 | { |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 847 | /* Prefetch 8 ahead - there's not much going on in each iteration */ |
| 848 | vlib_prefetch_buffer_header (b[4], LOAD); |
| 849 | vlib_prefetch_buffer_header (b[5], LOAD); |
| 850 | vlib_prefetch_buffer_header (b[6], LOAD); |
| 851 | vlib_prefetch_buffer_header (b[7], LOAD); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 852 | } |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 853 | sw_if_index[0] = vnet_buffer (b[0])->sw_if_index[VLIB_RX]; |
| 854 | sw_if_index[1] = vnet_buffer (b[1])->sw_if_index[VLIB_RX]; |
| 855 | sw_if_index[2] = vnet_buffer (b[2])->sw_if_index[VLIB_RX]; |
| 856 | sw_if_index[3] = vnet_buffer (b[3])->sw_if_index[VLIB_RX]; |
| 857 | |
| 858 | sw_if_index += 4; |
| 859 | n_left -= 4; |
| 860 | b += 4; |
| 861 | } |
| 862 | while (n_left) |
| 863 | { |
| 864 | sw_if_index[0] = vnet_buffer (b[0])->sw_if_index[VLIB_RX]; |
| 865 | |
| 866 | sw_if_index += 1; |
| 867 | n_left -= 1; |
| 868 | b += 1; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 869 | } |
| 870 | |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 871 | /* ... then count against them in blocks */ |
| 872 | n_left = frame->n_vectors; |
| 873 | |
| 874 | while (n_left) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 875 | { |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 876 | vnet_sw_interface_t *sw_if0; |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 877 | u16 off, count; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 878 | |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 879 | off = frame->n_vectors - n_left; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 880 | |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 881 | sw_if_index = sw_if_indices + off; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 882 | |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 883 | count = clib_count_equal_u32 (sw_if_index, n_left); |
| 884 | n_left -= count; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 885 | |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 886 | vlib_increment_simple_counter (cm, thread_index, sw_if_index[0], count); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 887 | |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 888 | /* Increment super-interface drop/punt counters for |
| 889 | sub-interfaces. */ |
| 890 | sw_if0 = vnet_get_sw_interface (vnm, sw_if_index[0]); |
| 891 | if (sw_if0->sup_sw_if_index != sw_if_index[0]) |
| 892 | vlib_increment_simple_counter |
| 893 | (cm, thread_index, sw_if0->sup_sw_if_index, count); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 894 | } |
| 895 | |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 896 | vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 897 | |
| 898 | return frame->n_vectors; |
| 899 | } |
| 900 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 901 | static inline void |
| 902 | pcap_drop_trace (vlib_main_t * vm, |
Dave Barach | 3390977 | 2019-09-23 10:27:27 -0400 | [diff] [blame] | 903 | vnet_interface_main_t * im, |
| 904 | vnet_pcap_t * pp, vlib_frame_t * f) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 905 | { |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 906 | u32 *from; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 907 | u32 n_left = f->n_vectors; |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 908 | vlib_buffer_t *b0, *p1; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 909 | u32 bi0; |
| 910 | i16 save_current_data; |
| 911 | u16 save_current_length; |
Dave Barach | 9382ad9 | 2019-09-23 16:03:49 -0400 | [diff] [blame] | 912 | vlib_error_main_t *em = &vm->error_main; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 913 | |
| 914 | from = vlib_frame_vector_args (f); |
| 915 | |
| 916 | while (n_left > 0) |
| 917 | { |
| 918 | if (PREDICT_TRUE (n_left > 1)) |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 919 | { |
| 920 | p1 = vlib_get_buffer (vm, from[1]); |
| 921 | vlib_prefetch_buffer_header (p1, LOAD); |
| 922 | } |
| 923 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 924 | bi0 = from[0]; |
| 925 | b0 = vlib_get_buffer (vm, bi0); |
| 926 | from++; |
| 927 | n_left--; |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 928 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 929 | /* See if we're pointedly ignoring this specific error */ |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 930 | if (im->pcap_drop_filter_hash |
| 931 | && hash_get (im->pcap_drop_filter_hash, b0->error)) |
| 932 | continue; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 933 | |
Benoît Ganne | 30a8195 | 2021-02-26 13:47:41 +0100 | [diff] [blame] | 934 | if (!vnet_is_packet_pcaped (pp, b0, ~0)) |
| 935 | continue; /* not matching, skip */ |
Dave Barach | f5667c3 | 2019-09-25 11:27:46 -0400 | [diff] [blame] | 936 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 937 | /* Trace all drops, or drops received on a specific interface */ |
Benoît Ganne | 30a8195 | 2021-02-26 13:47:41 +0100 | [diff] [blame] | 938 | save_current_data = b0->current_data; |
| 939 | save_current_length = b0->current_length; |
| 940 | |
| 941 | /* |
| 942 | * Typically, we'll need to rewind the buffer |
| 943 | * if l2_hdr_offset is valid, make sure to rewind to the start of |
| 944 | * the L2 header. This may not be the buffer start in case we pop-ed |
| 945 | * vlan tags. |
| 946 | * Otherwise, rewind to buffer start and hope for the best. |
| 947 | */ |
| 948 | if (b0->flags & VNET_BUFFER_F_L2_HDR_OFFSET_VALID) |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 949 | { |
Benoît Ganne | 30a8195 | 2021-02-26 13:47:41 +0100 | [diff] [blame] | 950 | if (b0->current_data > vnet_buffer (b0)->l2_hdr_offset) |
| 951 | vlib_buffer_advance (b0, vnet_buffer (b0)->l2_hdr_offset - |
| 952 | b0->current_data); |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 953 | } |
Benoît Ganne | 30a8195 | 2021-02-26 13:47:41 +0100 | [diff] [blame] | 954 | else if (b0->current_data > 0) |
| 955 | { |
| 956 | vlib_buffer_advance (b0, (word) -b0->current_data); |
| 957 | } |
| 958 | |
| 959 | { |
| 960 | vlib_buffer_t *last = b0; |
| 961 | u32 error_node_index; |
| 962 | int drop_string_len; |
| 963 | vlib_node_t *n; |
| 964 | /* Length of the error string */ |
| 965 | int error_string_len = |
| 966 | clib_strnlen (em->counters_heap[b0->error].name, 128); |
| 967 | |
| 968 | /* Dig up the drop node */ |
| 969 | error_node_index = vm->node_main.node_by_error[b0->error]; |
| 970 | n = vlib_get_node (vm, error_node_index); |
| 971 | |
| 972 | /* Length of full drop string, w/ "nodename: " prepended */ |
| 973 | drop_string_len = error_string_len + vec_len (n->name) + 2; |
| 974 | |
| 975 | /* Find the last buffer in the chain */ |
| 976 | while (last->flags & VLIB_BUFFER_NEXT_PRESENT) |
| 977 | last = vlib_get_buffer (vm, last->next_buffer); |
| 978 | |
| 979 | /* |
| 980 | * Append <nodename>: <error-string> to the capture, |
| 981 | * only if we can do that without allocating a new buffer. |
| 982 | */ |
| 983 | if (PREDICT_TRUE ((last->current_data + last->current_length) < |
| 984 | (VLIB_BUFFER_DEFAULT_DATA_SIZE - drop_string_len))) |
| 985 | { |
| 986 | clib_memcpy_fast (last->data + last->current_data + |
| 987 | last->current_length, |
| 988 | n->name, vec_len (n->name)); |
| 989 | clib_memcpy_fast (last->data + last->current_data + |
| 990 | last->current_length + vec_len (n->name), |
| 991 | ": ", 2); |
| 992 | clib_memcpy_fast (last->data + last->current_data + |
| 993 | last->current_length + vec_len (n->name) + 2, |
| 994 | em->counters_heap[b0->error].name, |
| 995 | error_string_len); |
| 996 | last->current_length += drop_string_len; |
| 997 | b0->flags &= ~(VLIB_BUFFER_TOTAL_LENGTH_VALID); |
| 998 | pcap_add_buffer (&pp->pcap_main, vm, bi0, pp->max_bytes_per_pkt); |
| 999 | last->current_length -= drop_string_len; |
| 1000 | b0->current_data = save_current_data; |
| 1001 | b0->current_length = save_current_length; |
| 1002 | continue; |
| 1003 | } |
| 1004 | } |
| 1005 | |
| 1006 | /* |
| 1007 | * Didn't have space in the last buffer, here's the dropped |
| 1008 | * packet as-is |
| 1009 | */ |
| 1010 | pcap_add_buffer (&pp->pcap_main, vm, bi0, pp->max_bytes_per_pkt); |
| 1011 | |
| 1012 | b0->current_data = save_current_data; |
| 1013 | b0->current_length = save_current_length; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1014 | } |
| 1015 | } |
| 1016 | |
Filip Tehlar | 6266877 | 2019-03-04 03:33:32 -0800 | [diff] [blame] | 1017 | #ifndef CLIB_MARCH_VARIANT |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 1018 | void |
| 1019 | vnet_pcap_drop_trace_filter_add_del (u32 error_index, int is_add) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1020 | { |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 1021 | vnet_interface_main_t *im = &vnet_get_main ()->interface_main; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1022 | |
| 1023 | if (im->pcap_drop_filter_hash == 0) |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 1024 | im->pcap_drop_filter_hash = hash_create (0, sizeof (uword)); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1025 | |
| 1026 | if (is_add) |
| 1027 | hash_set (im->pcap_drop_filter_hash, error_index, 1); |
| 1028 | else |
| 1029 | hash_unset (im->pcap_drop_filter_hash, error_index); |
| 1030 | } |
Filip Tehlar | 6266877 | 2019-03-04 03:33:32 -0800 | [diff] [blame] | 1031 | #endif /* CLIB_MARCH_VARIANT */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1032 | |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 1033 | VLIB_NODE_FN (interface_drop) (vlib_main_t * vm, |
| 1034 | vlib_node_runtime_t * node, |
| 1035 | vlib_frame_t * frame) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1036 | { |
Damjan Marion | 8fb5add | 2021-03-04 18:41:59 +0100 | [diff] [blame] | 1037 | vnet_main_t *vnm = vnet_get_main (); |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 1038 | vnet_interface_main_t *im = &vnet_get_main ()->interface_main; |
Damjan Marion | 8fb5add | 2021-03-04 18:41:59 +0100 | [diff] [blame] | 1039 | vnet_pcap_t *pp = &vnm->pcap; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1040 | |
Dave Barach | 3390977 | 2019-09-23 10:27:27 -0400 | [diff] [blame] | 1041 | if (PREDICT_FALSE (pp->pcap_drop_enable)) |
| 1042 | pcap_drop_trace (vm, im, pp, frame); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1043 | |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 1044 | return interface_drop_punt (vm, node, frame, VNET_ERROR_DISPOSITION_DROP); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1045 | } |
| 1046 | |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 1047 | VLIB_NODE_FN (interface_punt) (vlib_main_t * vm, |
| 1048 | vlib_node_runtime_t * node, |
| 1049 | vlib_frame_t * frame) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1050 | { |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 1051 | return interface_drop_punt (vm, node, frame, VNET_ERROR_DISPOSITION_PUNT); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1052 | } |
| 1053 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 1054 | /* *INDENT-OFF* */ |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 1055 | VLIB_REGISTER_NODE (interface_drop) = { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1056 | .name = "error-drop", |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1057 | .vector_size = sizeof (u32), |
| 1058 | .format_trace = format_vnet_error_trace, |
Dave Barach | 4330c46 | 2020-06-10 17:07:32 -0400 | [diff] [blame] | 1059 | .flags = VLIB_NODE_FLAG_TRACE_SUPPORTED, |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 1060 | .n_next_nodes = 1, |
| 1061 | .next_nodes = { |
| 1062 | [0] = "drop", |
| 1063 | }, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1064 | }; |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 1065 | /* *INDENT-ON* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1066 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 1067 | /* *INDENT-OFF* */ |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 1068 | VLIB_REGISTER_NODE (interface_punt) = { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1069 | .name = "error-punt", |
| 1070 | .vector_size = sizeof (u32), |
| 1071 | .format_trace = format_vnet_error_trace, |
Dave Barach | 4330c46 | 2020-06-10 17:07:32 -0400 | [diff] [blame] | 1072 | .flags = VLIB_NODE_FLAG_TRACE_SUPPORTED, |
Neale Ranns | 22e1f1d | 2019-03-01 15:53:11 +0000 | [diff] [blame] | 1073 | .n_next_nodes = 1, |
| 1074 | .next_nodes = { |
| 1075 | [0] = "punt", |
| 1076 | }, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1077 | }; |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 1078 | /* *INDENT-ON* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1079 | |
Filip Tehlar | 6266877 | 2019-03-04 03:33:32 -0800 | [diff] [blame] | 1080 | VLIB_REGISTER_NODE (vnet_per_buffer_interface_output_node) = { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1081 | .name = "interface-output", |
| 1082 | .vector_size = sizeof (u32), |
| 1083 | }; |
| 1084 | |
Damjan Marion | 8932e45 | 2021-04-16 11:49:26 +0200 | [diff] [blame] | 1085 | VLIB_NODE_FN (vnet_interface_output_arc_end_node) |
| 1086 | (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame) |
Damjan Marion | 152e21d | 2016-11-29 14:55:43 +0100 | [diff] [blame] | 1087 | { |
| 1088 | vnet_main_t *vnm = vnet_get_main (); |
Damjan Marion | 8932e45 | 2021-04-16 11:49:26 +0200 | [diff] [blame] | 1089 | vnet_interface_main_t *im = &vnm->interface_main; |
Damjan Marion | 1bd6cbb | 2021-04-15 13:12:51 +0200 | [diff] [blame] | 1090 | vnet_hw_if_output_node_runtime_t *r = 0; |
| 1091 | vnet_hw_interface_t *hi; |
| 1092 | vnet_hw_if_tx_frame_t *tf; |
Damjan Marion | 8932e45 | 2021-04-16 11:49:26 +0200 | [diff] [blame] | 1093 | vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs; |
Damjan Marion | 1bd6cbb | 2021-04-15 13:12:51 +0200 | [diff] [blame] | 1094 | u32 sw_if_indices[VLIB_FRAME_SIZE], *sw_if_index = sw_if_indices; |
| 1095 | u64 used_elts[VLIB_FRAME_SIZE / 64] = {}; |
| 1096 | u64 mask[VLIB_FRAME_SIZE / 64] = {}; |
| 1097 | u32 *tmp, *from, n_left, n_free, n_comp, *to, swif, off; |
| 1098 | u16 next_index; |
| 1099 | vlib_frame_t *f; |
Damjan Marion | 152e21d | 2016-11-29 14:55:43 +0100 | [diff] [blame] | 1100 | |
Damjan Marion | 8932e45 | 2021-04-16 11:49:26 +0200 | [diff] [blame] | 1101 | from = vlib_frame_vector_args (frame); |
| 1102 | n_left = frame->n_vectors; |
| 1103 | vlib_get_buffers (vm, from, bufs, n_left); |
| 1104 | |
| 1105 | while (n_left >= 8) |
Damjan Marion | 152e21d | 2016-11-29 14:55:43 +0100 | [diff] [blame] | 1106 | { |
Damjan Marion | 8932e45 | 2021-04-16 11:49:26 +0200 | [diff] [blame] | 1107 | vlib_prefetch_buffer_header (b[4], LOAD); |
| 1108 | vlib_prefetch_buffer_header (b[5], LOAD); |
| 1109 | vlib_prefetch_buffer_header (b[6], LOAD); |
| 1110 | vlib_prefetch_buffer_header (b[7], LOAD); |
Damjan Marion | 1bd6cbb | 2021-04-15 13:12:51 +0200 | [diff] [blame] | 1111 | sw_if_index[0] = vnet_buffer (b[0])->sw_if_index[VLIB_TX]; |
| 1112 | sw_if_index[1] = vnet_buffer (b[1])->sw_if_index[VLIB_TX]; |
| 1113 | sw_if_index[2] = vnet_buffer (b[2])->sw_if_index[VLIB_TX]; |
| 1114 | sw_if_index[3] = vnet_buffer (b[3])->sw_if_index[VLIB_TX]; |
Damjan Marion | 152e21d | 2016-11-29 14:55:43 +0100 | [diff] [blame] | 1115 | |
Damjan Marion | 8932e45 | 2021-04-16 11:49:26 +0200 | [diff] [blame] | 1116 | b += 4; |
Damjan Marion | 1bd6cbb | 2021-04-15 13:12:51 +0200 | [diff] [blame] | 1117 | sw_if_index += 4; |
Damjan Marion | 8932e45 | 2021-04-16 11:49:26 +0200 | [diff] [blame] | 1118 | n_left -= 4; |
Damjan Marion | 152e21d | 2016-11-29 14:55:43 +0100 | [diff] [blame] | 1119 | } |
Damjan Marion | 8932e45 | 2021-04-16 11:49:26 +0200 | [diff] [blame] | 1120 | |
| 1121 | while (n_left) |
| 1122 | { |
Damjan Marion | 1bd6cbb | 2021-04-15 13:12:51 +0200 | [diff] [blame] | 1123 | sw_if_index[0] = vnet_buffer (b[0])->sw_if_index[VLIB_TX]; |
Damjan Marion | 8932e45 | 2021-04-16 11:49:26 +0200 | [diff] [blame] | 1124 | b++; |
Damjan Marion | 1bd6cbb | 2021-04-15 13:12:51 +0200 | [diff] [blame] | 1125 | sw_if_index++; |
Damjan Marion | 8932e45 | 2021-04-16 11:49:26 +0200 | [diff] [blame] | 1126 | n_left--; |
| 1127 | } |
| 1128 | |
Damjan Marion | 1bd6cbb | 2021-04-15 13:12:51 +0200 | [diff] [blame] | 1129 | n_left = frame->n_vectors; |
| 1130 | swif = sw_if_indices[0]; |
| 1131 | off = 0; |
| 1132 | |
| 1133 | /* a bit ugly but it allows us to reuse stack space for temporary store |
| 1134 | * which may also improve memory latency */ |
| 1135 | tmp = (u32 *) bufs; |
| 1136 | |
| 1137 | more: |
| 1138 | next_index = vec_elt (im->if_out_arc_end_next_index_by_sw_if_index, swif); |
| 1139 | hi = vnet_get_sup_hw_interface (vnm, swif); |
| 1140 | if (hi->output_node_thread_runtimes) |
| 1141 | r = vec_elt_at_index (hi->output_node_thread_runtimes, vm->thread_index); |
| 1142 | f = vlib_get_next_frame_internal (vm, node, next_index, 0); |
| 1143 | tf = vlib_frame_scalar_args (f); |
| 1144 | |
| 1145 | if (f->n_vectors > 0 && (r == 0 || r->frame.queue_id == tf->queue_id)) |
| 1146 | { |
| 1147 | /* append frame */ |
| 1148 | n_free = VLIB_FRAME_SIZE - f->n_vectors; |
| 1149 | if (n_free >= f->n_vectors) |
| 1150 | to = (u32 *) vlib_frame_vector_args (f) + f->n_vectors; |
| 1151 | else |
| 1152 | to = tmp; |
| 1153 | } |
| 1154 | else |
| 1155 | { |
| 1156 | if (f->n_vectors > 0) |
| 1157 | { |
| 1158 | /* current frame doesn't fit - grab empty one */ |
| 1159 | f = vlib_get_next_frame_internal (vm, node, next_index, 1); |
| 1160 | tf = vlib_frame_scalar_args (f); |
| 1161 | } |
| 1162 | |
| 1163 | /* empty frame - store scalar data */ |
| 1164 | store_tx_frame_scalar_data (r, tf); |
| 1165 | n_free = VLIB_FRAME_SIZE; |
| 1166 | to = vlib_frame_vector_args (f); |
| 1167 | } |
| 1168 | |
| 1169 | /* compare and compress based on comparison mask */ |
| 1170 | clib_mask_compare_u32 (swif, sw_if_indices, mask, frame->n_vectors); |
| 1171 | n_comp = clib_compress_u32 (to, from, mask, frame->n_vectors); |
| 1172 | |
| 1173 | if (tmp != to) |
| 1174 | { |
| 1175 | /* indices already written to frame, just close it */ |
| 1176 | vlib_put_next_frame (vm, node, next_index, n_free - n_comp); |
| 1177 | } |
| 1178 | else if (n_free >= n_comp) |
| 1179 | { |
| 1180 | /* enough space in the existing frame */ |
| 1181 | to = (u32 *) vlib_frame_vector_args (f) + f->n_vectors; |
| 1182 | vlib_buffer_copy_indices (to, tmp, n_comp); |
| 1183 | vlib_put_next_frame (vm, node, next_index, n_free - n_comp); |
| 1184 | } |
| 1185 | else |
| 1186 | { |
| 1187 | /* full frame */ |
| 1188 | to = (u32 *) vlib_frame_vector_args (f) + f->n_vectors; |
| 1189 | vlib_buffer_copy_indices (to, tmp, n_free); |
| 1190 | vlib_put_next_frame (vm, node, next_index, 0); |
| 1191 | |
| 1192 | /* second frame */ |
| 1193 | u32 n_frame2 = n_comp - n_free; |
| 1194 | f = vlib_get_next_frame_internal (vm, node, next_index, 1); |
| 1195 | to = vlib_frame_vector_args (f); |
| 1196 | vlib_buffer_copy_indices (to, tmp + n_free, n_frame2); |
| 1197 | tf = vlib_frame_scalar_args (f); |
| 1198 | store_tx_frame_scalar_data (r, tf); |
| 1199 | vlib_put_next_frame (vm, node, next_index, VLIB_FRAME_SIZE - n_frame2); |
| 1200 | } |
| 1201 | |
| 1202 | n_left -= n_comp; |
| 1203 | if (n_left) |
| 1204 | { |
| 1205 | /* store comparison mask so we can find next unused element */ |
| 1206 | for (int i = 0; i < ARRAY_LEN (used_elts); i++) |
| 1207 | used_elts[i] |= mask[i]; |
| 1208 | |
| 1209 | /* fine first unused sw_if_index by scanning trough used_elts bitmap */ |
| 1210 | while (PREDICT_FALSE (used_elts[off] == ~0)) |
| 1211 | off++; |
| 1212 | |
| 1213 | swif = |
| 1214 | sw_if_indices[(off << 6) + count_trailing_zeros (~used_elts[off])]; |
| 1215 | goto more; |
| 1216 | } |
| 1217 | |
Damjan Marion | 8932e45 | 2021-04-16 11:49:26 +0200 | [diff] [blame] | 1218 | return frame->n_vectors; |
Damjan Marion | 152e21d | 2016-11-29 14:55:43 +0100 | [diff] [blame] | 1219 | } |
| 1220 | |
Damjan Marion | 8932e45 | 2021-04-16 11:49:26 +0200 | [diff] [blame] | 1221 | VLIB_REGISTER_NODE (vnet_interface_output_arc_end_node) = { |
| 1222 | .name = "interface-output-arc-end", |
Damjan Marion | 152e21d | 2016-11-29 14:55:43 +0100 | [diff] [blame] | 1223 | .vector_size = sizeof (u32), |
| 1224 | .n_next_nodes = 1, |
| 1225 | .next_nodes = { |
| 1226 | [0] = "error-drop", |
| 1227 | }, |
| 1228 | }; |
| 1229 | |
Damjan Marion | 8932e45 | 2021-04-16 11:49:26 +0200 | [diff] [blame] | 1230 | VNET_FEATURE_ARC_INIT (interface_output, static) = { |
| 1231 | .arc_name = "interface-output", |
Damjan Marion | 152e21d | 2016-11-29 14:55:43 +0100 | [diff] [blame] | 1232 | .start_nodes = VNET_FEATURES (0), |
Damjan Marion | 8932e45 | 2021-04-16 11:49:26 +0200 | [diff] [blame] | 1233 | .last_in_arc = "interface-output-arc-end", |
Damjan Marion | 152e21d | 2016-11-29 14:55:43 +0100 | [diff] [blame] | 1234 | .arc_index_ptr = &vnet_main.interface_main.output_feature_arc_index, |
| 1235 | }; |
| 1236 | |
Pavel Kotucek | 3a2a1c4 | 2016-12-06 10:10:10 +0100 | [diff] [blame] | 1237 | VNET_FEATURE_INIT (span_tx, static) = { |
| 1238 | .arc_name = "interface-output", |
| 1239 | .node_name = "span-output", |
Damjan Marion | 8932e45 | 2021-04-16 11:49:26 +0200 | [diff] [blame] | 1240 | .runs_before = VNET_FEATURES ("interface-output-arc-end"), |
Pavel Kotucek | 3a2a1c4 | 2016-12-06 10:10:10 +0100 | [diff] [blame] | 1241 | }; |
| 1242 | |
Matthew Smith | 537eeec | 2018-04-09 11:49:20 -0500 | [diff] [blame] | 1243 | VNET_FEATURE_INIT (ipsec_if_tx, static) = { |
| 1244 | .arc_name = "interface-output", |
| 1245 | .node_name = "ipsec-if-output", |
Damjan Marion | 8932e45 | 2021-04-16 11:49:26 +0200 | [diff] [blame] | 1246 | .runs_before = VNET_FEATURES ("interface-output-arc-end"), |
Matthew Smith | 537eeec | 2018-04-09 11:49:20 -0500 | [diff] [blame] | 1247 | }; |
| 1248 | |
Damjan Marion | 8932e45 | 2021-04-16 11:49:26 +0200 | [diff] [blame] | 1249 | VNET_FEATURE_INIT (interface_output_arc_end, static) = { |
Damjan Marion | 152e21d | 2016-11-29 14:55:43 +0100 | [diff] [blame] | 1250 | .arc_name = "interface-output", |
Damjan Marion | 8932e45 | 2021-04-16 11:49:26 +0200 | [diff] [blame] | 1251 | .node_name = "interface-output-arc-end", |
Damjan Marion | 152e21d | 2016-11-29 14:55:43 +0100 | [diff] [blame] | 1252 | .runs_before = 0, |
| 1253 | }; |
Damjan Marion | 152e21d | 2016-11-29 14:55:43 +0100 | [diff] [blame] | 1254 | |
Filip Tehlar | 6266877 | 2019-03-04 03:33:32 -0800 | [diff] [blame] | 1255 | #ifndef CLIB_MARCH_VARIANT |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1256 | clib_error_t * |
| 1257 | vnet_per_buffer_interface_output_hw_interface_add_del (vnet_main_t * vnm, |
| 1258 | u32 hw_if_index, |
| 1259 | u32 is_create) |
| 1260 | { |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 1261 | vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1262 | u32 next_index; |
| 1263 | |
John Lo | e5453d0 | 2018-01-23 19:21:34 -0500 | [diff] [blame] | 1264 | if (hi->output_node_index == 0) |
| 1265 | return 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1266 | |
John Lo | e5453d0 | 2018-01-23 19:21:34 -0500 | [diff] [blame] | 1267 | next_index = vlib_node_add_next |
| 1268 | (vnm->vlib_main, vnet_per_buffer_interface_output_node.index, |
| 1269 | hi->output_node_index); |
| 1270 | hi->output_node_next_index = next_index; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1271 | |
| 1272 | return 0; |
| 1273 | } |
| 1274 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 1275 | VNET_HW_INTERFACE_ADD_DEL_FUNCTION |
| 1276 | (vnet_per_buffer_interface_output_hw_interface_add_del); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1277 | |
John Lo | e5453d0 | 2018-01-23 19:21:34 -0500 | [diff] [blame] | 1278 | void |
| 1279 | vnet_set_interface_output_node (vnet_main_t * vnm, |
| 1280 | u32 hw_if_index, u32 node_index) |
| 1281 | { |
| 1282 | ASSERT (node_index); |
| 1283 | vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index); |
| 1284 | u32 next_index = vlib_node_add_next |
| 1285 | (vnm->vlib_main, vnet_per_buffer_interface_output_node.index, node_index); |
| 1286 | hi->output_node_next_index = next_index; |
| 1287 | hi->output_node_index = node_index; |
| 1288 | } |
Filip Tehlar | 6266877 | 2019-03-04 03:33:32 -0800 | [diff] [blame] | 1289 | #endif /* CLIB_MARCH_VARIANT */ |
John Lo | e5453d0 | 2018-01-23 19:21:34 -0500 | [diff] [blame] | 1290 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 1291 | /* |
| 1292 | * fd.io coding-style-patch-verification: ON |
| 1293 | * |
| 1294 | * Local Variables: |
| 1295 | * eval: (c-set-style "gnu") |
| 1296 | * End: |
| 1297 | */ |