| /* |
| * Copyright (c) 2021 Cisco and/or its affiliates. |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at: |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| #include <vlib/vlib.h> |
| #include <vnet/ethernet/ethernet.h> |
| #include <vnet/hash/hash.h> |
| #include <vnet/ip/ip4_packet.h> |
| #include <vnet/ip/ip6_packet.h> |
| #include <vnet/mpls/packet.h> |
| #include <vppinfra/crc32.h> |
| #include <vppinfra/xxhash.h> |
| |
| always_inline u32 |
| ho_hash (u64 key) |
| { |
| #ifdef clib_crc32c_uses_intrinsics |
| return clib_crc32c ((u8 *) &key, sizeof (key)); |
| #else |
| return clib_xxhash (key); |
| #endif |
| } |
| |
| static inline u64 |
| ipv4_get_key (ip4_header_t * ip) |
| { |
| u64 hash_key; |
| |
| hash_key = *((u64 *) (&ip->address_pair)) ^ ip->protocol; |
| |
| return hash_key; |
| } |
| |
| static inline u64 |
| ipv6_get_key (ip6_header_t * ip) |
| { |
| u64 hash_key; |
| |
| hash_key = ip->src_address.as_u64[0] ^ |
| rotate_left (ip->src_address.as_u64[1], 13) ^ |
| rotate_left (ip->dst_address.as_u64[0], 26) ^ |
| rotate_left (ip->dst_address.as_u64[1], 39) ^ ip->protocol; |
| |
| return hash_key; |
| } |
| |
| #define MPLS_BOTTOM_OF_STACK_BIT_MASK 0x00000100U |
| #define MPLS_LABEL_MASK 0xFFFFF000U |
| |
| static inline u64 |
| mpls_get_key (mpls_unicast_header_t * m) |
| { |
| u64 hash_key; |
| u8 ip_ver; |
| |
| |
| /* find the bottom of the MPLS label stack. */ |
| if (PREDICT_TRUE (m->label_exp_s_ttl & |
| clib_net_to_host_u32 (MPLS_BOTTOM_OF_STACK_BIT_MASK))) |
| { |
| goto bottom_lbl_found; |
| } |
| m++; |
| |
| if (PREDICT_TRUE (m->label_exp_s_ttl & |
| clib_net_to_host_u32 (MPLS_BOTTOM_OF_STACK_BIT_MASK))) |
| { |
| goto bottom_lbl_found; |
| } |
| m++; |
| |
| if (m->label_exp_s_ttl & |
| clib_net_to_host_u32 (MPLS_BOTTOM_OF_STACK_BIT_MASK)) |
| { |
| goto bottom_lbl_found; |
| } |
| m++; |
| |
| if (m->label_exp_s_ttl & |
| clib_net_to_host_u32 (MPLS_BOTTOM_OF_STACK_BIT_MASK)) |
| { |
| goto bottom_lbl_found; |
| } |
| m++; |
| |
| if (m->label_exp_s_ttl & |
| clib_net_to_host_u32 (MPLS_BOTTOM_OF_STACK_BIT_MASK)) |
| { |
| goto bottom_lbl_found; |
| } |
| |
| /* the bottom label was not found - use the last label */ |
| hash_key = m->label_exp_s_ttl & clib_net_to_host_u32 (MPLS_LABEL_MASK); |
| |
| return hash_key; |
| |
| bottom_lbl_found: |
| m++; |
| ip_ver = (*((u8 *) m) >> 4); |
| |
| /* find out if it is IPV4 or IPV6 header */ |
| if (PREDICT_TRUE (ip_ver == 4)) |
| { |
| hash_key = ipv4_get_key ((ip4_header_t *) m); |
| } |
| else if (PREDICT_TRUE (ip_ver == 6)) |
| { |
| hash_key = ipv6_get_key ((ip6_header_t *) m); |
| } |
| else |
| { |
| /* use the bottom label */ |
| hash_key = |
| (m - 1)->label_exp_s_ttl & clib_net_to_host_u32 (MPLS_LABEL_MASK); |
| } |
| |
| return hash_key; |
| |
| } |
| |
| static inline u64 |
| eth_get_sym_key (ethernet_header_t * h0) |
| { |
| u64 hash_key; |
| |
| if (PREDICT_TRUE (h0->type) == clib_host_to_net_u16 (ETHERNET_TYPE_IP4)) |
| { |
| ip4_header_t *ip = (ip4_header_t *) (h0 + 1); |
| hash_key = |
| (u64) (ip->src_address.as_u32 ^ |
| ip->dst_address.as_u32 ^ ip->protocol); |
| } |
| else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6)) |
| { |
| ip6_header_t *ip = (ip6_header_t *) (h0 + 1); |
| hash_key = (u64) (ip->src_address.as_u64[0] ^ |
| ip->src_address.as_u64[1] ^ |
| ip->dst_address.as_u64[0] ^ |
| ip->dst_address.as_u64[1] ^ ip->protocol); |
| } |
| else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS)) |
| { |
| hash_key = mpls_get_key ((mpls_unicast_header_t *) (h0 + 1)); |
| } |
| else |
| if (PREDICT_FALSE |
| ((h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_VLAN)) |
| || (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_DOT1AD)))) |
| { |
| ethernet_vlan_header_t *outer = (ethernet_vlan_header_t *) (h0 + 1); |
| |
| outer = (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_VLAN)) ? |
| outer + 1 : outer; |
| if (PREDICT_TRUE (outer->type) == |
| clib_host_to_net_u16 (ETHERNET_TYPE_IP4)) |
| { |
| ip4_header_t *ip = (ip4_header_t *) (outer + 1); |
| hash_key = |
| (u64) (ip->src_address.as_u32 ^ |
| ip->dst_address.as_u32 ^ ip->protocol); |
| } |
| else if (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6)) |
| { |
| ip6_header_t *ip = (ip6_header_t *) (outer + 1); |
| hash_key = |
| (u64) (ip->src_address.as_u64[0] ^ ip->src_address.as_u64[1] ^ |
| ip->dst_address.as_u64[0] ^ |
| ip->dst_address.as_u64[1] ^ ip->protocol); |
| } |
| else if (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS)) |
| { |
| hash_key = mpls_get_key ((mpls_unicast_header_t *) (outer + 1)); |
| } |
| else |
| { |
| hash_key = outer->type; |
| } |
| } |
| else |
| { |
| hash_key = 0; |
| } |
| |
| return hash_key; |
| } |
| |
| static inline u64 |
| eth_get_key (ethernet_header_t * h0) |
| { |
| u64 hash_key; |
| |
| if (PREDICT_TRUE (h0->type) == clib_host_to_net_u16 (ETHERNET_TYPE_IP4)) |
| { |
| hash_key = ipv4_get_key ((ip4_header_t *) (h0 + 1)); |
| } |
| else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6)) |
| { |
| hash_key = ipv6_get_key ((ip6_header_t *) (h0 + 1)); |
| } |
| else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS)) |
| { |
| hash_key = mpls_get_key ((mpls_unicast_header_t *) (h0 + 1)); |
| } |
| else if ((h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_VLAN)) || |
| (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_DOT1AD))) |
| { |
| ethernet_vlan_header_t *outer = (ethernet_vlan_header_t *) (h0 + 1); |
| |
| outer = (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_VLAN)) ? |
| outer + 1 : outer; |
| if (PREDICT_TRUE (outer->type) == |
| clib_host_to_net_u16 (ETHERNET_TYPE_IP4)) |
| { |
| hash_key = ipv4_get_key ((ip4_header_t *) (outer + 1)); |
| } |
| else if (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6)) |
| { |
| hash_key = ipv6_get_key ((ip6_header_t *) (outer + 1)); |
| } |
| else if (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS)) |
| { |
| hash_key = mpls_get_key ((mpls_unicast_header_t *) (outer + 1)); |
| } |
| else |
| { |
| hash_key = outer->type; |
| } |
| } |
| else |
| { |
| hash_key = 0; |
| } |
| |
| return hash_key; |
| } |
| |
| void |
| handoff_eth_func (void **p, u32 *hash, u32 n_packets) |
| { |
| u32 n_left_from = n_packets; |
| |
| while (n_left_from >= 8) |
| { |
| u64 key[4] = {}; |
| |
| clib_prefetch_load (p[4]); |
| clib_prefetch_load (p[5]); |
| clib_prefetch_load (p[6]); |
| clib_prefetch_load (p[7]); |
| |
| key[0] = eth_get_key ((ethernet_header_t *) p[0]); |
| key[1] = eth_get_key ((ethernet_header_t *) p[1]); |
| key[2] = eth_get_key ((ethernet_header_t *) p[2]); |
| key[3] = eth_get_key ((ethernet_header_t *) p[3]); |
| |
| hash[0] = ho_hash (key[0]); |
| hash[1] = ho_hash (key[1]); |
| hash[2] = ho_hash (key[2]); |
| hash[3] = ho_hash (key[3]); |
| |
| hash += 4; |
| n_left_from -= 4; |
| p += 4; |
| } |
| |
| while (n_left_from > 0) |
| { |
| u64 key; |
| |
| key = eth_get_key ((ethernet_header_t *) p[0]); |
| hash[0] = ho_hash (key); |
| |
| hash += 1; |
| n_left_from -= 1; |
| p += 1; |
| } |
| } |
| |
| VNET_REGISTER_HASH_FUNCTION (handoff_eth, static) = { |
| .name = "handoff-eth", |
| .description = "Ethernet/IPv4/IPv6/MPLS headers", |
| .priority = 2, |
| .function[VNET_HASH_FN_TYPE_ETHERNET] = handoff_eth_func, |
| }; |
| |
| void |
| handoff_eth_sym_func (void **p, u32 *hash, u32 n_packets) |
| { |
| u32 n_left_from = n_packets; |
| |
| while (n_left_from >= 8) |
| { |
| u64 key[4] = {}; |
| |
| clib_prefetch_load (p[4]); |
| clib_prefetch_load (p[5]); |
| clib_prefetch_load (p[6]); |
| clib_prefetch_load (p[7]); |
| |
| key[0] = eth_get_sym_key ((ethernet_header_t *) p[0]); |
| key[1] = eth_get_sym_key ((ethernet_header_t *) p[1]); |
| key[2] = eth_get_sym_key ((ethernet_header_t *) p[2]); |
| key[3] = eth_get_sym_key ((ethernet_header_t *) p[3]); |
| |
| hash[0] = ho_hash (key[0]); |
| hash[1] = ho_hash (key[1]); |
| hash[2] = ho_hash (key[2]); |
| hash[3] = ho_hash (key[3]); |
| |
| hash += 4; |
| n_left_from -= 4; |
| p += 4; |
| } |
| |
| while (n_left_from > 0) |
| { |
| u64 key; |
| |
| key = eth_get_sym_key ((ethernet_header_t *) p[0]); |
| hash[0] = ho_hash (key); |
| |
| hash += 1; |
| n_left_from -= 1; |
| p += 1; |
| } |
| } |
| |
| VNET_REGISTER_HASH_FUNCTION (handoff_eth_sym, static) = { |
| .name = "handoff-eth-sym", |
| .description = "Ethernet/IPv4/IPv6/MPLS headers Symmetric", |
| .priority = 1, |
| .function[VNET_HASH_FN_TYPE_ETHERNET] = handoff_eth_sym_func, |
| }; |
| |
| /* |
| * fd.io coding-style-patch-verification: ON |
| * |
| * Local Variables: |
| * eval: (c-set-style "gnu") |
| * End: |
| */ |