Eyal Bari | 3ce0b08 | 2018-01-17 12:06:32 +0200 | [diff] [blame] | 1 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 2 | /* |
| 3 | * Copyright (c) 2015 Cisco and/or its affiliates. |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at: |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | #include <vppinfra/error.h> |
| 17 | #include <vppinfra/hash.h> |
| 18 | #include <vnet/vnet.h> |
| 19 | #include <vnet/ip/ip.h> |
| 20 | #include <vnet/ethernet/ethernet.h> |
| 21 | #include <vnet/vxlan/vxlan.h> |
Igor Mikhailov (imichail) | 6636440 | 2018-04-24 21:53:00 -0700 | [diff] [blame] | 22 | #include <vnet/qos/qos_types.h> |
eyal bari | 82e21d7 | 2018-04-26 13:14:55 +0300 | [diff] [blame] | 23 | #include <vnet/adj/rewrite.h> |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 24 | |
| 25 | /* Statistics (not all errors) */ |
| 26 | #define foreach_vxlan_encap_error \ |
John Lo | 3ef822e | 2016-06-07 09:14:07 -0400 | [diff] [blame] | 27 | _(ENCAPSULATED, "good packets encapsulated") |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 28 | |
| 29 | static char * vxlan_encap_error_strings[] = { |
| 30 | #define _(sym,string) string, |
| 31 | foreach_vxlan_encap_error |
| 32 | #undef _ |
| 33 | }; |
| 34 | |
| 35 | typedef enum { |
| 36 | #define _(sym,str) VXLAN_ENCAP_ERROR_##sym, |
| 37 | foreach_vxlan_encap_error |
| 38 | #undef _ |
| 39 | VXLAN_ENCAP_N_ERROR, |
| 40 | } vxlan_encap_error_t; |
| 41 | |
| 42 | typedef enum { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 43 | VXLAN_ENCAP_NEXT_DROP, |
| 44 | VXLAN_ENCAP_N_NEXT, |
| 45 | } vxlan_encap_next_t; |
| 46 | |
| 47 | typedef struct { |
| 48 | u32 tunnel_index; |
| 49 | u32 vni; |
| 50 | } vxlan_encap_trace_t; |
| 51 | |
Filip Tehlar | e1714d3 | 2019-03-05 03:01:43 -0800 | [diff] [blame] | 52 | #ifndef CLIB_MARCH_VARIANT |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 53 | u8 * format_vxlan_encap_trace (u8 * s, va_list * args) |
| 54 | { |
| 55 | CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); |
| 56 | CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); |
| 57 | vxlan_encap_trace_t * t |
| 58 | = va_arg (*args, vxlan_encap_trace_t *); |
| 59 | |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 60 | s = format (s, "VXLAN encap to vxlan_tunnel%d vni %d", |
| 61 | t->tunnel_index, t->vni); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 62 | return s; |
| 63 | } |
Filip Tehlar | e1714d3 | 2019-03-05 03:01:43 -0800 | [diff] [blame] | 64 | #endif |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 65 | |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 66 | always_inline uword |
| 67 | vxlan_encap_inline (vlib_main_t * vm, |
| 68 | vlib_node_runtime_t * node, |
| 69 | vlib_frame_t * from_frame, |
John Lo | f4215a6 | 2017-09-18 00:20:05 -0400 | [diff] [blame] | 70 | u8 is_ip4, u8 csum_offload) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 71 | { |
| 72 | u32 n_left_from, next_index, * from, * to_next; |
| 73 | vxlan_main_t * vxm = &vxlan_main; |
| 74 | vnet_main_t * vnm = vxm->vnet_main; |
| 75 | vnet_interface_main_t * im = &vnm->interface_main; |
John Lo | 25d417f | 2018-02-15 15:47:53 -0500 | [diff] [blame] | 76 | vlib_combined_counter_main_t * tx_counter = |
| 77 | im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 78 | u32 pkts_encapsulated = 0; |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 79 | u32 thread_index = vlib_get_thread_index(); |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 80 | u32 sw_if_index0 = 0, sw_if_index1 = 0; |
| 81 | u32 next0 = 0, next1 = 0; |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 82 | vxlan_tunnel_t * t0 = NULL, * t1 = NULL; |
John Lo | 25d417f | 2018-02-15 15:47:53 -0500 | [diff] [blame] | 83 | index_t dpoi_idx0 = INDEX_INVALID, dpoi_idx1 = INDEX_INVALID; |
Zhiyong Yang | 102dd1c | 2020-03-27 13:04:38 +0000 | [diff] [blame^] | 84 | vlib_buffer_t *bufs[VLIB_FRAME_SIZE]; |
| 85 | vlib_buffer_t **b = bufs; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 86 | |
| 87 | from = vlib_frame_vector_args (from_frame); |
| 88 | n_left_from = from_frame->n_vectors; |
| 89 | |
| 90 | next_index = node->cached_next_index; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 91 | |
Eyal Bari | 3ce0b08 | 2018-01-17 12:06:32 +0200 | [diff] [blame] | 92 | STATIC_ASSERT_SIZEOF(ip6_vxlan_header_t, 56); |
| 93 | STATIC_ASSERT_SIZEOF(ip4_vxlan_header_t, 36); |
| 94 | |
eyal bari | 82e21d7 | 2018-04-26 13:14:55 +0300 | [diff] [blame] | 95 | u8 const underlay_hdr_len = is_ip4 ? |
Eyal Bari | 3ce0b08 | 2018-01-17 12:06:32 +0200 | [diff] [blame] | 96 | sizeof(ip4_vxlan_header_t) : sizeof(ip6_vxlan_header_t); |
| 97 | u16 const l3_len = is_ip4 ? sizeof(ip4_header_t) : sizeof(ip6_header_t); |
John Lo | b1b98f5 | 2019-08-03 18:25:49 -0400 | [diff] [blame] | 98 | u32 const csum_flags = is_ip4 ? VNET_BUFFER_F_OFFLOAD_IP_CKSUM | |
| 99 | VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM : |
| 100 | VNET_BUFFER_F_IS_IP6 | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; |
Zhiyong Yang | 102dd1c | 2020-03-27 13:04:38 +0000 | [diff] [blame^] | 101 | vlib_get_buffers (vm, from, bufs, n_left_from); |
Eyal Bari | 3ce0b08 | 2018-01-17 12:06:32 +0200 | [diff] [blame] | 102 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 103 | while (n_left_from > 0) |
| 104 | { |
| 105 | u32 n_left_to_next; |
| 106 | |
| 107 | vlib_get_next_frame (vm, node, next_index, |
| 108 | to_next, n_left_to_next); |
| 109 | |
| 110 | while (n_left_from >= 4 && n_left_to_next >= 2) |
| 111 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 112 | /* Prefetch next iteration. */ |
| 113 | { |
Zhiyong Yang | 102dd1c | 2020-03-27 13:04:38 +0000 | [diff] [blame^] | 114 | vlib_prefetch_buffer_header (b[2], LOAD); |
| 115 | vlib_prefetch_buffer_header (b[3], LOAD); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 116 | |
Zhiyong Yang | 102dd1c | 2020-03-27 13:04:38 +0000 | [diff] [blame^] | 117 | CLIB_PREFETCH (b[2]->data - CLIB_CACHE_LINE_BYTES, 2 * CLIB_CACHE_LINE_BYTES, LOAD); |
| 118 | CLIB_PREFETCH (b[3]->data - CLIB_CACHE_LINE_BYTES, 2 * CLIB_CACHE_LINE_BYTES, LOAD); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 119 | } |
| 120 | |
eyal bari | 82e21d7 | 2018-04-26 13:14:55 +0300 | [diff] [blame] | 121 | u32 bi0 = to_next[0] = from[0]; |
| 122 | u32 bi1 = to_next[1] = from[1]; |
| 123 | from += 2; |
| 124 | to_next += 2; |
| 125 | n_left_to_next -= 2; |
| 126 | n_left_from -= 2; |
Eyal Bari | 3ce0b08 | 2018-01-17 12:06:32 +0200 | [diff] [blame] | 127 | |
Zhiyong Yang | 102dd1c | 2020-03-27 13:04:38 +0000 | [diff] [blame^] | 128 | vlib_buffer_t * b0 = b[0]; |
| 129 | vlib_buffer_t * b1 = b[1]; |
| 130 | b += 2; |
| 131 | |
Eyal Bari | 3ce0b08 | 2018-01-17 12:06:32 +0200 | [diff] [blame] | 132 | u32 flow_hash0 = vnet_l2_compute_flow_hash (b0); |
| 133 | u32 flow_hash1 = vnet_l2_compute_flow_hash (b1); |
| 134 | |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 135 | /* Get next node index and adj index from tunnel next_dpo */ |
| 136 | if (sw_if_index0 != vnet_buffer(b0)->sw_if_index[VLIB_TX]) |
| 137 | { |
| 138 | sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_TX]; |
John Lo | 25d417f | 2018-02-15 15:47:53 -0500 | [diff] [blame] | 139 | vnet_hw_interface_t *hi0 = |
| 140 | vnet_get_sup_hw_interface (vnm, sw_if_index0); |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 141 | t0 = &vxm->tunnels[hi0->dev_instance]; |
John Lo | 25d417f | 2018-02-15 15:47:53 -0500 | [diff] [blame] | 142 | /* Note: change to always set next0 if it may set to drop */ |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 143 | next0 = t0->next_dpo.dpoi_next_node; |
John Lo | 25d417f | 2018-02-15 15:47:53 -0500 | [diff] [blame] | 144 | dpoi_idx0 = t0->next_dpo.dpoi_index; |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 145 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 146 | |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 147 | /* Get next node index and adj index from tunnel next_dpo */ |
| 148 | if (sw_if_index1 != vnet_buffer(b1)->sw_if_index[VLIB_TX]) |
| 149 | { |
John Lo | 25d417f | 2018-02-15 15:47:53 -0500 | [diff] [blame] | 150 | if (sw_if_index0 == vnet_buffer(b1)->sw_if_index[VLIB_TX]) |
| 151 | { |
| 152 | sw_if_index1 = sw_if_index0; |
| 153 | t1 = t0; |
| 154 | next1 = next0; |
| 155 | dpoi_idx1 = dpoi_idx0; |
| 156 | } |
| 157 | else |
| 158 | { |
| 159 | sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_TX]; |
| 160 | vnet_hw_interface_t *hi1 = |
| 161 | vnet_get_sup_hw_interface (vnm, sw_if_index1); |
| 162 | t1 = &vxm->tunnels[hi1->dev_instance]; |
| 163 | /* Note: change to always set next1 if it may set to drop */ |
| 164 | next1 = t1->next_dpo.dpoi_next_node; |
| 165 | dpoi_idx1 = t1->next_dpo.dpoi_index; |
| 166 | } |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 167 | } |
John Lo | 25d417f | 2018-02-15 15:47:53 -0500 | [diff] [blame] | 168 | |
| 169 | vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpoi_idx0; |
| 170 | vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpoi_idx1; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 171 | |
eyal bari | 82e21d7 | 2018-04-26 13:14:55 +0300 | [diff] [blame] | 172 | ASSERT(t0->rewrite_header.data_bytes == underlay_hdr_len); |
| 173 | ASSERT(t1->rewrite_header.data_bytes == underlay_hdr_len); |
Benoît Ganne | 4af1a7f | 2019-03-01 14:14:10 +0100 | [diff] [blame] | 174 | vnet_rewrite_two_headers(*t0, *t1, vlib_buffer_get_current(b0), vlib_buffer_get_current(b1), underlay_hdr_len); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 175 | |
Eyal Bari | 3ce0b08 | 2018-01-17 12:06:32 +0200 | [diff] [blame] | 176 | vlib_buffer_advance (b0, -underlay_hdr_len); |
| 177 | vlib_buffer_advance (b1, -underlay_hdr_len); |
| 178 | |
| 179 | u32 len0 = vlib_buffer_length_in_chain (vm, b0); |
| 180 | u32 len1 = vlib_buffer_length_in_chain (vm, b1); |
| 181 | u16 payload_l0 = clib_host_to_net_u16 (len0 - l3_len); |
| 182 | u16 payload_l1 = clib_host_to_net_u16 (len1 - l3_len); |
| 183 | |
eyal bari | 82e21d7 | 2018-04-26 13:14:55 +0300 | [diff] [blame] | 184 | void * underlay0 = vlib_buffer_get_current(b0); |
| 185 | void * underlay1 = vlib_buffer_get_current(b1); |
| 186 | |
Eyal Bari | 3ce0b08 | 2018-01-17 12:06:32 +0200 | [diff] [blame] | 187 | ip4_header_t * ip4_0, * ip4_1; |
Igor Mikhailov (imichail) | 6636440 | 2018-04-24 21:53:00 -0700 | [diff] [blame] | 188 | qos_bits_t ip4_0_tos = 0, ip4_1_tos = 0; |
Eyal Bari | 3ce0b08 | 2018-01-17 12:06:32 +0200 | [diff] [blame] | 189 | ip6_header_t * ip6_0, * ip6_1; |
| 190 | udp_header_t * udp0, * udp1; |
| 191 | u8 * l3_0, * l3_1; |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 192 | if (is_ip4) |
| 193 | { |
eyal bari | 82e21d7 | 2018-04-26 13:14:55 +0300 | [diff] [blame] | 194 | ip4_vxlan_header_t * hdr0 = underlay0; |
| 195 | ip4_vxlan_header_t * hdr1 = underlay1; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 196 | |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 197 | /* Fix the IP4 checksum and length */ |
Eyal Bari | 3ce0b08 | 2018-01-17 12:06:32 +0200 | [diff] [blame] | 198 | ip4_0 = &hdr0->ip4; |
| 199 | ip4_1 = &hdr1->ip4; |
| 200 | ip4_0->length = clib_host_to_net_u16 (len0); |
| 201 | ip4_1->length = clib_host_to_net_u16 (len1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 202 | |
Igor Mikhailov (imichail) | 6636440 | 2018-04-24 21:53:00 -0700 | [diff] [blame] | 203 | if (PREDICT_FALSE (b0->flags & VNET_BUFFER_F_QOS_DATA_VALID)) |
| 204 | { |
| 205 | ip4_0_tos = vnet_buffer2 (b0)->qos.bits; |
| 206 | ip4_0->tos = ip4_0_tos; |
| 207 | } |
| 208 | if (PREDICT_FALSE (b1->flags & VNET_BUFFER_F_QOS_DATA_VALID)) |
| 209 | { |
| 210 | ip4_1_tos = vnet_buffer2 (b1)->qos.bits; |
| 211 | ip4_1->tos = ip4_1_tos; |
| 212 | } |
| 213 | |
Eyal Bari | 3ce0b08 | 2018-01-17 12:06:32 +0200 | [diff] [blame] | 214 | l3_0 = (u8 *)ip4_0; |
| 215 | l3_1 = (u8 *)ip4_1; |
| 216 | udp0 = &hdr0->udp; |
| 217 | udp1 = &hdr1->udp; |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 218 | } |
| 219 | else /* ipv6 */ |
| 220 | { |
eyal bari | 82e21d7 | 2018-04-26 13:14:55 +0300 | [diff] [blame] | 221 | ip6_vxlan_header_t * hdr0 = underlay0; |
| 222 | ip6_vxlan_header_t * hdr1 = underlay1; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 223 | |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 224 | /* Fix IP6 payload length */ |
Eyal Bari | 3ce0b08 | 2018-01-17 12:06:32 +0200 | [diff] [blame] | 225 | ip6_0 = &hdr0->ip6; |
| 226 | ip6_1 = &hdr1->ip6; |
| 227 | ip6_0->payload_length = payload_l0; |
| 228 | ip6_1->payload_length = payload_l1; |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 229 | |
Eyal Bari | 3ce0b08 | 2018-01-17 12:06:32 +0200 | [diff] [blame] | 230 | l3_0 = (u8 *)ip6_0; |
| 231 | l3_1 = (u8 *)ip6_1; |
| 232 | udp0 = &hdr0->udp; |
| 233 | udp1 = &hdr1->udp; |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 234 | } |
Chris Luke | 99cb335 | 2016-04-26 10:49:53 -0400 | [diff] [blame] | 235 | |
Eyal Bari | 3ce0b08 | 2018-01-17 12:06:32 +0200 | [diff] [blame] | 236 | /* Fix UDP length and set source port */ |
| 237 | udp0->length = payload_l0; |
| 238 | udp0->src_port = flow_hash0; |
| 239 | udp1->length = payload_l1; |
| 240 | udp1->src_port = flow_hash1; |
| 241 | |
| 242 | if (csum_offload) |
| 243 | { |
| 244 | b0->flags |= csum_flags; |
| 245 | vnet_buffer (b0)->l3_hdr_offset = l3_0 - b0->data; |
| 246 | vnet_buffer (b0)->l4_hdr_offset = (u8 *) udp0 - b0->data; |
| 247 | b1->flags |= csum_flags; |
| 248 | vnet_buffer (b1)->l3_hdr_offset = l3_1 - b1->data; |
| 249 | vnet_buffer (b1)->l4_hdr_offset = (u8 *) udp1 - b1->data; |
| 250 | } |
| 251 | /* IPv4 UDP checksum only if checksum offload is used */ |
| 252 | else if (is_ip4) |
| 253 | { |
| 254 | ip_csum_t sum0 = ip4_0->checksum; |
| 255 | sum0 = ip_csum_update (sum0, 0, ip4_0->length, ip4_header_t, |
| 256 | length /* changed member */); |
Igor Mikhailov (imichail) | 6636440 | 2018-04-24 21:53:00 -0700 | [diff] [blame] | 257 | if (PREDICT_FALSE (ip4_0_tos)) |
| 258 | { |
| 259 | sum0 = ip_csum_update (sum0, 0, ip4_0_tos, ip4_header_t, |
| 260 | tos /* changed member */); |
| 261 | } |
Eyal Bari | 3ce0b08 | 2018-01-17 12:06:32 +0200 | [diff] [blame] | 262 | ip4_0->checksum = ip_csum_fold (sum0); |
| 263 | ip_csum_t sum1 = ip4_1->checksum; |
| 264 | sum1 = ip_csum_update (sum1, 0, ip4_1->length, ip4_header_t, |
| 265 | length /* changed member */); |
Igor Mikhailov (imichail) | 6636440 | 2018-04-24 21:53:00 -0700 | [diff] [blame] | 266 | if (PREDICT_FALSE (ip4_1_tos)) |
| 267 | { |
| 268 | sum1 = ip_csum_update (sum1, 0, ip4_1_tos, ip4_header_t, |
| 269 | tos /* changed member */); |
| 270 | } |
Eyal Bari | 3ce0b08 | 2018-01-17 12:06:32 +0200 | [diff] [blame] | 271 | ip4_1->checksum = ip_csum_fold (sum1); |
| 272 | } |
| 273 | /* IPv6 UDP checksum is mandatory */ |
| 274 | else |
| 275 | { |
| 276 | int bogus = 0; |
| 277 | |
| 278 | udp0->checksum = ip6_tcp_udp_icmp_compute_checksum |
| 279 | (vm, b0, ip6_0, &bogus); |
| 280 | ASSERT(bogus == 0); |
| 281 | if (udp0->checksum == 0) |
| 282 | udp0->checksum = 0xffff; |
| 283 | udp1->checksum = ip6_tcp_udp_icmp_compute_checksum |
| 284 | (vm, b1, ip6_1, &bogus); |
| 285 | ASSERT(bogus == 0); |
| 286 | if (udp1->checksum == 0) |
| 287 | udp1->checksum = 0xffff; |
| 288 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 289 | |
Shawn Ji | 623b4f8 | 2019-12-18 10:10:54 +0800 | [diff] [blame] | 290 | /* save inner packet flow_hash for load-balance node */ |
| 291 | vnet_buffer (b0)->ip.flow_hash = flow_hash0; |
| 292 | vnet_buffer (b1)->ip.flow_hash = flow_hash1; |
| 293 | |
Zhiyong Yang | 70fda4c | 2018-09-04 22:45:17 -0400 | [diff] [blame] | 294 | if (sw_if_index0 == sw_if_index1) |
| 295 | { |
| 296 | vlib_increment_combined_counter (tx_counter, thread_index, |
| 297 | sw_if_index0, 2, len0 + len1); |
| 298 | } |
| 299 | else |
| 300 | { |
Eyal Bari | 0f4b184 | 2018-04-12 12:39:51 +0300 | [diff] [blame] | 301 | vlib_increment_combined_counter (tx_counter, thread_index, |
| 302 | sw_if_index0, 1, len0); |
| 303 | vlib_increment_combined_counter (tx_counter, thread_index, |
| 304 | sw_if_index1, 1, len1); |
Zhiyong Yang | 70fda4c | 2018-09-04 22:45:17 -0400 | [diff] [blame] | 305 | } |
Eyal Bari | 3ce0b08 | 2018-01-17 12:06:32 +0200 | [diff] [blame] | 306 | pkts_encapsulated += 2; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 307 | |
| 308 | if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) |
| 309 | { |
| 310 | vxlan_encap_trace_t *tr = |
| 311 | vlib_add_trace (vm, node, b0, sizeof (*tr)); |
| 312 | tr->tunnel_index = t0 - vxm->tunnels; |
| 313 | tr->vni = t0->vni; |
| 314 | } |
| 315 | |
| 316 | if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED)) |
| 317 | { |
| 318 | vxlan_encap_trace_t *tr = |
| 319 | vlib_add_trace (vm, node, b1, sizeof (*tr)); |
| 320 | tr->tunnel_index = t1 - vxm->tunnels; |
| 321 | tr->vni = t1->vni; |
| 322 | } |
| 323 | |
| 324 | vlib_validate_buffer_enqueue_x2 (vm, node, next_index, |
| 325 | to_next, n_left_to_next, |
| 326 | bi0, bi1, next0, next1); |
| 327 | } |
| 328 | |
| 329 | while (n_left_from > 0 && n_left_to_next > 0) |
| 330 | { |
eyal bari | 82e21d7 | 2018-04-26 13:14:55 +0300 | [diff] [blame] | 331 | u32 bi0 = to_next[0] = from[0]; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 332 | from += 1; |
| 333 | to_next += 1; |
| 334 | n_left_from -= 1; |
| 335 | n_left_to_next -= 1; |
| 336 | |
Zhiyong Yang | 102dd1c | 2020-03-27 13:04:38 +0000 | [diff] [blame^] | 337 | vlib_buffer_t * b0 = b[0]; |
| 338 | b += 1; |
| 339 | |
eyal bari | 82e21d7 | 2018-04-26 13:14:55 +0300 | [diff] [blame] | 340 | u32 flow_hash0 = vnet_l2_compute_flow_hash(b0); |
| 341 | |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 342 | /* Get next node index and adj index from tunnel next_dpo */ |
| 343 | if (sw_if_index0 != vnet_buffer(b0)->sw_if_index[VLIB_TX]) |
| 344 | { |
| 345 | sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_TX]; |
John Lo | 25d417f | 2018-02-15 15:47:53 -0500 | [diff] [blame] | 346 | vnet_hw_interface_t *hi0 = |
| 347 | vnet_get_sup_hw_interface (vnm, sw_if_index0); |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 348 | t0 = &vxm->tunnels[hi0->dev_instance]; |
| 349 | /* Note: change to always set next0 if it may be set to drop */ |
| 350 | next0 = t0->next_dpo.dpoi_next_node; |
John Lo | 25d417f | 2018-02-15 15:47:53 -0500 | [diff] [blame] | 351 | dpoi_idx0 = t0->next_dpo.dpoi_index; |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 352 | } |
John Lo | 25d417f | 2018-02-15 15:47:53 -0500 | [diff] [blame] | 353 | vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpoi_idx0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 354 | |
eyal bari | 82e21d7 | 2018-04-26 13:14:55 +0300 | [diff] [blame] | 355 | ASSERT(t0->rewrite_header.data_bytes == underlay_hdr_len); |
Benoît Ganne | 4af1a7f | 2019-03-01 14:14:10 +0100 | [diff] [blame] | 356 | vnet_rewrite_one_header(*t0, vlib_buffer_get_current(b0), underlay_hdr_len); |
eyal bari | 82e21d7 | 2018-04-26 13:14:55 +0300 | [diff] [blame] | 357 | |
Eyal Bari | 3ce0b08 | 2018-01-17 12:06:32 +0200 | [diff] [blame] | 358 | vlib_buffer_advance (b0, -underlay_hdr_len); |
eyal bari | 82e21d7 | 2018-04-26 13:14:55 +0300 | [diff] [blame] | 359 | void * underlay0 = vlib_buffer_get_current(b0); |
| 360 | |
Eyal Bari | 3ce0b08 | 2018-01-17 12:06:32 +0200 | [diff] [blame] | 361 | u32 len0 = vlib_buffer_length_in_chain (vm, b0); |
| 362 | u16 payload_l0 = clib_host_to_net_u16 (len0 - l3_len); |
| 363 | |
| 364 | udp_header_t * udp0; |
| 365 | ip4_header_t * ip4_0; |
Igor Mikhailov (imichail) | 6636440 | 2018-04-24 21:53:00 -0700 | [diff] [blame] | 366 | qos_bits_t ip4_0_tos = 0; |
Eyal Bari | 3ce0b08 | 2018-01-17 12:06:32 +0200 | [diff] [blame] | 367 | ip6_header_t * ip6_0; |
| 368 | u8 * l3_0; |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 369 | if (is_ip4) |
| 370 | { |
eyal bari | 82e21d7 | 2018-04-26 13:14:55 +0300 | [diff] [blame] | 371 | ip4_vxlan_header_t * hdr = underlay0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 372 | |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 373 | /* Fix the IP4 checksum and length */ |
Eyal Bari | 3ce0b08 | 2018-01-17 12:06:32 +0200 | [diff] [blame] | 374 | ip4_0 = &hdr->ip4; |
| 375 | ip4_0->length = clib_host_to_net_u16 (len0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 376 | |
Igor Mikhailov (imichail) | 6636440 | 2018-04-24 21:53:00 -0700 | [diff] [blame] | 377 | if (PREDICT_FALSE (b0->flags & VNET_BUFFER_F_QOS_DATA_VALID)) |
| 378 | { |
| 379 | ip4_0_tos = vnet_buffer2 (b0)->qos.bits; |
| 380 | ip4_0->tos = ip4_0_tos; |
| 381 | } |
| 382 | |
Eyal Bari | 3ce0b08 | 2018-01-17 12:06:32 +0200 | [diff] [blame] | 383 | l3_0 = (u8*)ip4_0; |
| 384 | udp0 = &hdr->udp; |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 385 | } |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 386 | else /* ip6 path */ |
| 387 | { |
eyal bari | 82e21d7 | 2018-04-26 13:14:55 +0300 | [diff] [blame] | 388 | ip6_vxlan_header_t * hdr = underlay0; |
Chris Luke | 99cb335 | 2016-04-26 10:49:53 -0400 | [diff] [blame] | 389 | |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 390 | /* Fix IP6 payload length */ |
Eyal Bari | 3ce0b08 | 2018-01-17 12:06:32 +0200 | [diff] [blame] | 391 | ip6_0 = &hdr->ip6; |
| 392 | ip6_0->payload_length = payload_l0; |
Chris Luke | 99cb335 | 2016-04-26 10:49:53 -0400 | [diff] [blame] | 393 | |
Eyal Bari | 3ce0b08 | 2018-01-17 12:06:32 +0200 | [diff] [blame] | 394 | l3_0 = (u8 *)ip6_0; |
| 395 | udp0 = &hdr->udp; |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 396 | } |
| 397 | |
Eyal Bari | 3ce0b08 | 2018-01-17 12:06:32 +0200 | [diff] [blame] | 398 | /* Fix UDP length and set source port */ |
| 399 | udp0->length = payload_l0; |
| 400 | udp0->src_port = flow_hash0; |
| 401 | |
| 402 | if (csum_offload) |
| 403 | { |
| 404 | b0->flags |= csum_flags; |
| 405 | vnet_buffer (b0)->l3_hdr_offset = l3_0 - b0->data; |
| 406 | vnet_buffer (b0)->l4_hdr_offset = (u8 *) udp0 - b0->data; |
| 407 | } |
| 408 | /* IPv4 UDP checksum only if checksum offload is used */ |
| 409 | else if (is_ip4) |
| 410 | { |
| 411 | ip_csum_t sum0 = ip4_0->checksum; |
| 412 | sum0 = ip_csum_update (sum0, 0, ip4_0->length, ip4_header_t, |
| 413 | length /* changed member */); |
Igor Mikhailov (imichail) | 6636440 | 2018-04-24 21:53:00 -0700 | [diff] [blame] | 414 | if (PREDICT_FALSE (ip4_0_tos)) |
| 415 | { |
| 416 | sum0 = ip_csum_update (sum0, 0, ip4_0_tos, ip4_header_t, |
| 417 | tos /* changed member */); |
| 418 | } |
Eyal Bari | 3ce0b08 | 2018-01-17 12:06:32 +0200 | [diff] [blame] | 419 | ip4_0->checksum = ip_csum_fold (sum0); |
| 420 | } |
| 421 | /* IPv6 UDP checksum is mandatory */ |
| 422 | else |
| 423 | { |
| 424 | int bogus = 0; |
| 425 | |
| 426 | udp0->checksum = ip6_tcp_udp_icmp_compute_checksum |
| 427 | (vm, b0, ip6_0, &bogus); |
| 428 | ASSERT(bogus == 0); |
| 429 | if (udp0->checksum == 0) |
| 430 | udp0->checksum = 0xffff; |
| 431 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 432 | |
Shawn Ji | 623b4f8 | 2019-12-18 10:10:54 +0800 | [diff] [blame] | 433 | /* reuse inner packet flow_hash for load-balance node */ |
| 434 | vnet_buffer (b0)->ip.flow_hash = flow_hash0; |
| 435 | |
Eyal Bari | 0f4b184 | 2018-04-12 12:39:51 +0300 | [diff] [blame] | 436 | vlib_increment_combined_counter (tx_counter, thread_index, |
| 437 | sw_if_index0, 1, len0); |
Eyal Bari | 3ce0b08 | 2018-01-17 12:06:32 +0200 | [diff] [blame] | 438 | pkts_encapsulated ++; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 439 | |
| 440 | if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) |
| 441 | { |
| 442 | vxlan_encap_trace_t *tr = |
| 443 | vlib_add_trace (vm, node, b0, sizeof (*tr)); |
| 444 | tr->tunnel_index = t0 - vxm->tunnels; |
| 445 | tr->vni = t0->vni; |
| 446 | } |
| 447 | vlib_validate_buffer_enqueue_x1 (vm, node, next_index, |
| 448 | to_next, n_left_to_next, |
| 449 | bi0, next0); |
| 450 | } |
| 451 | |
| 452 | vlib_put_next_frame (vm, node, next_index, n_left_to_next); |
| 453 | } |
| 454 | |
| 455 | /* Do we still need this now that tunnel tx stats is kept? */ |
| 456 | vlib_node_increment_counter (vm, node->node_index, |
| 457 | VXLAN_ENCAP_ERROR_ENCAPSULATED, |
| 458 | pkts_encapsulated); |
| 459 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 460 | return from_frame->n_vectors; |
| 461 | } |
| 462 | |
Filip Tehlar | e1714d3 | 2019-03-05 03:01:43 -0800 | [diff] [blame] | 463 | VLIB_NODE_FN (vxlan4_encap_node) (vlib_main_t * vm, |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 464 | vlib_node_runtime_t * node, |
| 465 | vlib_frame_t * from_frame) |
| 466 | { |
John Lo | f4215a6 | 2017-09-18 00:20:05 -0400 | [diff] [blame] | 467 | /* Disable chksum offload as setup overhead in tx node is not worthwhile |
| 468 | for ip4 header checksum only, unless udp checksum is also required */ |
| 469 | return vxlan_encap_inline (vm, node, from_frame, /* is_ip4 */ 1, |
| 470 | /* csum_offload */ 0); |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 471 | } |
| 472 | |
Filip Tehlar | e1714d3 | 2019-03-05 03:01:43 -0800 | [diff] [blame] | 473 | VLIB_NODE_FN (vxlan6_encap_node) (vlib_main_t * vm, |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 474 | vlib_node_runtime_t * node, |
| 475 | vlib_frame_t * from_frame) |
| 476 | { |
John Lo | f4215a6 | 2017-09-18 00:20:05 -0400 | [diff] [blame] | 477 | /* Enable checksum offload for ip6 as udp checksum is mandatory, */ |
| 478 | return vxlan_encap_inline (vm, node, from_frame, /* is_ip4 */ 0, |
| 479 | /* csum_offload */ 1); |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 480 | } |
| 481 | |
| 482 | VLIB_REGISTER_NODE (vxlan4_encap_node) = { |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 483 | .name = "vxlan4-encap", |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 484 | .vector_size = sizeof (u32), |
| 485 | .format_trace = format_vxlan_encap_trace, |
| 486 | .type = VLIB_NODE_TYPE_INTERNAL, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 487 | .n_errors = ARRAY_LEN(vxlan_encap_error_strings), |
| 488 | .error_strings = vxlan_encap_error_strings, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 489 | .n_next_nodes = VXLAN_ENCAP_N_NEXT, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 490 | .next_nodes = { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 491 | [VXLAN_ENCAP_NEXT_DROP] = "error-drop", |
| 492 | }, |
| 493 | }; |
Damjan Marion | 1c80e83 | 2016-05-11 23:07:18 +0200 | [diff] [blame] | 494 | |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 495 | VLIB_REGISTER_NODE (vxlan6_encap_node) = { |
John Lo | c42912d | 2016-11-07 18:30:47 -0500 | [diff] [blame] | 496 | .name = "vxlan6-encap", |
| 497 | .vector_size = sizeof (u32), |
| 498 | .format_trace = format_vxlan_encap_trace, |
| 499 | .type = VLIB_NODE_TYPE_INTERNAL, |
| 500 | .n_errors = ARRAY_LEN(vxlan_encap_error_strings), |
| 501 | .error_strings = vxlan_encap_error_strings, |
| 502 | .n_next_nodes = VXLAN_ENCAP_N_NEXT, |
| 503 | .next_nodes = { |
| 504 | [VXLAN_ENCAP_NEXT_DROP] = "error-drop", |
| 505 | }, |
| 506 | }; |
| 507 | |