| /* |
| * Copyright (c) 2020 Cisco and/or its affiliates. |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at: |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| #ifndef included_gro_func_h |
| #define included_gro_func_h |
| |
| #include <vnet/ethernet/ethernet.h> |
| #include <vnet/gso/gro.h> |
| #include <vnet/gso/hdr_offset_parser.h> |
| #include <vnet/ip/ip4.h> |
| #include <vnet/ip/ip6.h> |
| #include <vnet/ip/ip6_inlines.h> |
| #include <vnet/udp/udp_packet.h> |
| #include <vnet/tcp/tcp_packet.h> |
| #include <vnet/vnet.h> |
| #include <vnet/interface.h> |
| |
| #define GRO_MIN_PACKET_SIZE 256 |
| #define GRO_PADDED_PACKET_SIZE 64 |
| |
| static_always_inline u8 |
| gro_is_bad_packet (vlib_buffer_t * b, u8 flags, i16 l234_sz) |
| { |
| if (((b->current_length - l234_sz) <= 0) || |
| ((flags &= ~(TCP_FLAG_ACK | TCP_FLAG_PSH)) != 0)) |
| return 1; |
| return 0; |
| } |
| |
| static_always_inline void |
| gro_get_ip4_flow_from_packet (u32 * sw_if_index, |
| ip4_header_t * ip4, tcp_header_t * tcp, |
| gro_flow_key_t * flow_key, int is_l2) |
| { |
| flow_key->sw_if_index[VLIB_RX] = sw_if_index[VLIB_RX]; |
| flow_key->sw_if_index[VLIB_TX] = sw_if_index[VLIB_TX]; |
| ip46_address_set_ip4 (&flow_key->src_address, &ip4->src_address); |
| ip46_address_set_ip4 (&flow_key->dst_address, &ip4->dst_address); |
| flow_key->src_port = tcp->src_port; |
| flow_key->dst_port = tcp->dst_port; |
| } |
| |
| static_always_inline void |
| gro_get_ip6_flow_from_packet (u32 * sw_if_index, |
| ip6_header_t * ip6, tcp_header_t * tcp, |
| gro_flow_key_t * flow_key, int is_l2) |
| { |
| flow_key->sw_if_index[VLIB_RX] = sw_if_index[VLIB_RX]; |
| flow_key->sw_if_index[VLIB_TX] = sw_if_index[VLIB_TX]; |
| ip46_address_set_ip6 (&flow_key->src_address, &ip6->src_address); |
| ip46_address_set_ip6 (&flow_key->dst_address, &ip6->dst_address); |
| flow_key->src_port = tcp->src_port; |
| flow_key->dst_port = tcp->dst_port; |
| } |
| |
| static_always_inline u32 |
| gro_is_ip4_or_ip6_packet (vlib_buffer_t *b0, u8 is_l2) |
| { |
| if (b0->flags & VNET_BUFFER_F_IS_IP4) |
| return VNET_BUFFER_F_IS_IP4; |
| if (b0->flags & VNET_BUFFER_F_IS_IP6) |
| return VNET_BUFFER_F_IS_IP6; |
| if (is_l2) |
| { |
| ethernet_header_t *eh = |
| (ethernet_header_t *) vlib_buffer_get_current (b0); |
| u16 ethertype = clib_net_to_host_u16 (eh->type); |
| |
| if (ethernet_frame_is_tagged (ethertype)) |
| { |
| ethernet_vlan_header_t *vlan = (ethernet_vlan_header_t *) (eh + 1); |
| |
| ethertype = clib_net_to_host_u16 (vlan->type); |
| if (ethertype == ETHERNET_TYPE_VLAN) |
| { |
| vlan++; |
| ethertype = clib_net_to_host_u16 (vlan->type); |
| } |
| } |
| if (ethertype == ETHERNET_TYPE_IP4) |
| return VNET_BUFFER_F_IS_IP4; |
| if (ethertype == ETHERNET_TYPE_IP6) |
| return VNET_BUFFER_F_IS_IP6; |
| } |
| else |
| { |
| if ((((u8 *) vlib_buffer_get_current (b0))[0] & 0xf0) == 0x40) |
| return VNET_BUFFER_F_IS_IP4; |
| if ((((u8 *) vlib_buffer_get_current (b0))[0] & 0xf0) == 0x60) |
| return VNET_BUFFER_F_IS_IP6; |
| } |
| |
| return 0; |
| } |
| |
| typedef enum |
| { |
| GRO_PACKET_ACTION_NONE = 0, |
| GRO_PACKET_ACTION_ENQUEUE = 1, |
| GRO_PACKET_ACTION_FLUSH = 2, |
| } gro_packet_action_t; |
| |
| static_always_inline gro_packet_action_t |
| gro_tcp_sequence_check (tcp_header_t * tcp0, tcp_header_t * tcp1, |
| u32 payload_len0) |
| { |
| u32 next_tcp_seq0 = clib_net_to_host_u32 (tcp0->seq_number); |
| u32 next_tcp_seq1 = clib_net_to_host_u32 (tcp1->seq_number); |
| |
| /* next packet, enqueue */ |
| if (PREDICT_TRUE (next_tcp_seq0 + payload_len0 == next_tcp_seq1)) |
| return GRO_PACKET_ACTION_ENQUEUE; |
| /* flush all packets */ |
| else |
| return GRO_PACKET_ACTION_FLUSH; |
| } |
| |
| static_always_inline void |
| gro_merge_buffers (vlib_main_t * vm, vlib_buffer_t * b0, |
| vlib_buffer_t * b1, u32 bi1, u32 payload_len1, |
| u16 l234_sz1) |
| { |
| vlib_buffer_t *pb = b0; |
| |
| if (PREDICT_FALSE ((b0->flags & VLIB_BUFFER_NEXT_PRESENT) == 0)) |
| b0->total_length_not_including_first_buffer = 0; |
| |
| while (pb->flags & VLIB_BUFFER_NEXT_PRESENT) |
| pb = vlib_get_buffer (vm, pb->next_buffer); |
| |
| vlib_buffer_advance (b1, l234_sz1); |
| pb->flags |= VLIB_BUFFER_NEXT_PRESENT; |
| pb->next_buffer = bi1; |
| b0->total_length_not_including_first_buffer += payload_len1; |
| b0->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; |
| } |
| |
| static_always_inline u32 |
| gro_validate_checksum (vlib_main_t * vm, vlib_buffer_t * b0, |
| generic_header_offset_t * gho0, int is_ip4) |
| { |
| u32 flags = 0; |
| |
| if (b0->flags & VNET_BUFFER_F_OFFLOAD) |
| return VNET_BUFFER_F_L4_CHECKSUM_CORRECT; |
| vlib_buffer_advance (b0, gho0->l3_hdr_offset); |
| if (is_ip4) |
| flags = ip4_tcp_udp_validate_checksum (vm, b0); |
| else |
| flags = ip6_tcp_udp_icmp_validate_checksum (vm, b0); |
| vlib_buffer_advance (b0, -gho0->l3_hdr_offset); |
| return flags; |
| } |
| |
| static_always_inline u32 |
| gro_fix_padded_packet_len (vlib_buffer_t *b0, generic_header_offset_t *gho0, |
| ip4_header_t *ip4_0, ip6_header_t *ip6_0, |
| u32 pkt_len0, u16 l234_sz0) |
| { |
| u32 tcp_payload_len0 = 0; |
| if (gho0->gho_flags & GHO_F_IP4) |
| { |
| tcp_payload_len0 = clib_net_to_host_u16 (ip4_0->length) - |
| ip4_header_bytes (ip4_0) - gho0->l4_hdr_sz; |
| } |
| else |
| { |
| tcp_payload_len0 = |
| clib_net_to_host_u16 (ip6_0->payload_length) - gho0->l4_hdr_sz; |
| } |
| |
| ASSERT (l234_sz0 + tcp_payload_len0 <= pkt_len0); |
| |
| if (PREDICT_FALSE (l234_sz0 + tcp_payload_len0 < pkt_len0)) |
| { |
| /* small packet with padding at the end, remove padding */ |
| b0->current_length = l234_sz0 + tcp_payload_len0; |
| pkt_len0 = b0->current_length; |
| } |
| return pkt_len0; |
| } |
| |
| static_always_inline u32 |
| gro_get_packet_data (vlib_main_t *vm, vlib_buffer_t *b0, |
| generic_header_offset_t *gho0, gro_flow_key_t *flow_key0, |
| u8 is_l2) |
| { |
| ip4_header_t *ip4_0 = 0; |
| ip6_header_t *ip6_0 = 0; |
| tcp_header_t *tcp0 = 0; |
| u32 flags = 0; |
| u32 pkt_len0 = 0; |
| u16 l234_sz0 = 0; |
| u32 sw_if_index0[VLIB_N_RX_TX] = { ~0 }; |
| |
| u32 is_ip0 = gro_is_ip4_or_ip6_packet (b0, is_l2); |
| |
| if (is_ip0 & VNET_BUFFER_F_IS_IP4) |
| vnet_generic_header_offset_parser (b0, gho0, is_l2, 1 /* is_ip4 */ , |
| 0 /* is_ip6 */ ); |
| else if (is_ip0 & VNET_BUFFER_F_IS_IP6) |
| vnet_generic_header_offset_parser (b0, gho0, is_l2, 0 /* is_ip4 */ , |
| 1 /* is_ip6 */ ); |
| else |
| return 0; |
| |
| if (PREDICT_FALSE ((gho0->gho_flags & GHO_F_TCP) == 0)) |
| return 0; |
| |
| ip4_0 = |
| (ip4_header_t *) (vlib_buffer_get_current (b0) + gho0->l3_hdr_offset); |
| ip6_0 = |
| (ip6_header_t *) (vlib_buffer_get_current (b0) + gho0->l3_hdr_offset); |
| tcp0 = |
| (tcp_header_t *) (vlib_buffer_get_current (b0) + gho0->l4_hdr_offset); |
| |
| l234_sz0 = gho0->hdr_sz; |
| if (PREDICT_FALSE (gro_is_bad_packet (b0, tcp0->flags, l234_sz0))) |
| return 0; |
| |
| sw_if_index0[VLIB_RX] = vnet_buffer (b0)->sw_if_index[VLIB_RX]; |
| sw_if_index0[VLIB_TX] = vnet_buffer (b0)->sw_if_index[VLIB_TX]; |
| |
| if (gho0->gho_flags & GHO_F_IP4) |
| { |
| flags = gro_validate_checksum (vm, b0, gho0, 1); |
| gro_get_ip4_flow_from_packet (sw_if_index0, ip4_0, tcp0, flow_key0, |
| is_l2); |
| } |
| else if (gho0->gho_flags & GHO_F_IP6) |
| { |
| flags = gro_validate_checksum (vm, b0, gho0, 0); |
| gro_get_ip6_flow_from_packet (sw_if_index0, ip6_0, tcp0, flow_key0, |
| is_l2); |
| } |
| else |
| return 0; |
| |
| if (PREDICT_FALSE ((flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) == 0)) |
| return 0; |
| |
| pkt_len0 = vlib_buffer_length_in_chain (vm, b0); |
| if (PREDICT_FALSE (pkt_len0 >= TCP_MAX_GSO_SZ)) |
| return 0; |
| |
| if (PREDICT_FALSE (pkt_len0 <= GRO_PADDED_PACKET_SIZE)) |
| { |
| pkt_len0 = |
| gro_fix_padded_packet_len (b0, gho0, ip4_0, ip6_0, pkt_len0, l234_sz0); |
| } |
| return pkt_len0; |
| } |
| |
| static_always_inline u32 |
| gro_coalesce_buffers (vlib_main_t *vm, vlib_buffer_t *b0, vlib_buffer_t *b1, |
| u32 bi1, u8 is_l2) |
| { |
| generic_header_offset_t gho0 = { 0 }; |
| generic_header_offset_t gho1 = { 0 }; |
| gro_flow_key_t flow_key0, flow_key1; |
| ip4_header_t *ip4_0, *ip4_1; |
| ip6_header_t *ip6_0, *ip6_1; |
| tcp_header_t *tcp0, *tcp1; |
| u16 l234_sz0, l234_sz1; |
| u32 pkt_len0, pkt_len1, payload_len0, payload_len1; |
| u32 sw_if_index0[VLIB_N_RX_TX] = { ~0 }; |
| u32 sw_if_index1[VLIB_N_RX_TX] = { ~0 }; |
| |
| u32 is_ip0 = gro_is_ip4_or_ip6_packet (b0, is_l2); |
| u32 is_ip1 = gro_is_ip4_or_ip6_packet (b1, is_l2); |
| |
| if (is_ip0 & VNET_BUFFER_F_IS_IP4) |
| vnet_generic_header_offset_parser (b0, &gho0, is_l2, 1 /* is_ip4 */ , |
| 0 /* is_ip6 */ ); |
| else if (is_ip0 & VNET_BUFFER_F_IS_IP6) |
| vnet_generic_header_offset_parser (b0, &gho0, is_l2, 0 /* is_ip4 */ , |
| 1 /* is_ip6 */ ); |
| else |
| return 0; |
| |
| if (is_ip1 & VNET_BUFFER_F_IS_IP4) |
| vnet_generic_header_offset_parser (b1, &gho1, is_l2, 1 /* is_ip4 */ , |
| 0 /* is_ip6 */ ); |
| else if (is_ip1 & VNET_BUFFER_F_IS_IP6) |
| vnet_generic_header_offset_parser (b1, &gho1, is_l2, 0 /* is_ip4 */ , |
| 1 /* is_ip6 */ ); |
| else |
| return 0; |
| |
| pkt_len0 = vlib_buffer_length_in_chain (vm, b0); |
| pkt_len1 = vlib_buffer_length_in_chain (vm, b1); |
| |
| if (((gho0.gho_flags & GHO_F_TCP) == 0 || pkt_len0 <= GRO_MIN_PACKET_SIZE) || |
| ((gho1.gho_flags & GHO_F_TCP) == 0 || pkt_len1 <= GRO_MIN_PACKET_SIZE)) |
| return 0; |
| |
| ip4_0 = |
| (ip4_header_t *) (vlib_buffer_get_current (b0) + gho0.l3_hdr_offset); |
| ip4_1 = |
| (ip4_header_t *) (vlib_buffer_get_current (b1) + gho1.l3_hdr_offset); |
| ip6_0 = |
| (ip6_header_t *) (vlib_buffer_get_current (b0) + gho0.l3_hdr_offset); |
| ip6_1 = |
| (ip6_header_t *) (vlib_buffer_get_current (b1) + gho1.l3_hdr_offset); |
| |
| tcp0 = (tcp_header_t *) (vlib_buffer_get_current (b0) + gho0.l4_hdr_offset); |
| tcp1 = (tcp_header_t *) (vlib_buffer_get_current (b1) + gho1.l4_hdr_offset); |
| |
| l234_sz0 = gho0.hdr_sz; |
| l234_sz1 = gho1.hdr_sz; |
| |
| if (gro_is_bad_packet (b0, tcp0->flags, l234_sz0) |
| || gro_is_bad_packet (b1, tcp1->flags, l234_sz1)) |
| return 0; |
| |
| sw_if_index0[VLIB_RX] = vnet_buffer (b0)->sw_if_index[VLIB_RX]; |
| sw_if_index0[VLIB_TX] = vnet_buffer (b0)->sw_if_index[VLIB_TX]; |
| |
| sw_if_index1[VLIB_RX] = vnet_buffer (b1)->sw_if_index[VLIB_RX]; |
| sw_if_index1[VLIB_TX] = vnet_buffer (b1)->sw_if_index[VLIB_TX]; |
| |
| if ((gho0.gho_flags & GHO_F_IP4) && (gho1.gho_flags & GHO_F_IP4)) |
| { |
| gro_get_ip4_flow_from_packet (sw_if_index0, ip4_0, tcp0, &flow_key0, |
| is_l2); |
| gro_get_ip4_flow_from_packet (sw_if_index1, ip4_1, tcp1, &flow_key1, |
| is_l2); |
| } |
| else if ((gho0.gho_flags & GHO_F_IP6) && (gho1.gho_flags & GHO_F_IP6)) |
| { |
| gro_get_ip6_flow_from_packet (sw_if_index0, ip6_0, tcp0, &flow_key0, |
| is_l2); |
| gro_get_ip6_flow_from_packet (sw_if_index1, ip6_1, tcp1, &flow_key1, |
| is_l2); |
| } |
| else |
| return 0; |
| |
| if (gro_flow_is_equal (&flow_key0, &flow_key1) == 0) |
| return 0; |
| |
| payload_len0 = pkt_len0 - l234_sz0; |
| payload_len1 = pkt_len1 - l234_sz1; |
| |
| if (pkt_len0 >= TCP_MAX_GSO_SZ || pkt_len1 >= TCP_MAX_GSO_SZ |
| || (pkt_len0 + payload_len1) >= TCP_MAX_GSO_SZ) |
| return 0; |
| |
| if (gro_tcp_sequence_check (tcp0, tcp1, payload_len0) == |
| GRO_PACKET_ACTION_ENQUEUE) |
| { |
| gro_merge_buffers (vm, b0, b1, bi1, payload_len1, l234_sz1); |
| tcp0->flags |= tcp1->flags; |
| return tcp1->ack_number; |
| } |
| |
| return 0; |
| } |
| |
| static_always_inline void |
| gro_fixup_header (vlib_main_t *vm, vlib_buffer_t *b0, u32 ack_number, u8 is_l2) |
| { |
| generic_header_offset_t gho0 = { 0 }; |
| |
| u32 is_ip0 = gro_is_ip4_or_ip6_packet (b0, is_l2); |
| |
| if (is_ip0 & VNET_BUFFER_F_IS_IP4) |
| vnet_generic_header_offset_parser (b0, &gho0, is_l2, 1 /* is_ip4 */ , |
| 0 /* is_ip6 */ ); |
| else if (is_ip0 & VNET_BUFFER_F_IS_IP6) |
| vnet_generic_header_offset_parser (b0, &gho0, is_l2, 0 /* is_ip4 */ , |
| 1 /* is_ip6 */ ); |
| |
| vnet_buffer2 (b0)->gso_size = b0->current_length - gho0.hdr_sz; |
| |
| if (gho0.gho_flags & GHO_F_IP4) |
| { |
| ip4_header_t *ip4 = |
| (ip4_header_t *) (vlib_buffer_get_current (b0) + gho0.l3_hdr_offset); |
| ip4->length = |
| clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) - |
| gho0.l3_hdr_offset); |
| vnet_buffer (b0)->l3_hdr_offset = (u8 *) ip4 - b0->data; |
| b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP4); |
| vnet_buffer_offload_flags_set (b0, (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM | |
| VNET_BUFFER_OFFLOAD_F_IP_CKSUM)); |
| } |
| else if (gho0.gho_flags & GHO_F_IP6) |
| { |
| ip6_header_t *ip6 = |
| (ip6_header_t *) (vlib_buffer_get_current (b0) + gho0.l3_hdr_offset); |
| ip6->payload_length = |
| clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) - |
| gho0.l4_hdr_offset); |
| vnet_buffer (b0)->l3_hdr_offset = (u8 *) ip6 - b0->data; |
| b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP6); |
| vnet_buffer_offload_flags_set (b0, VNET_BUFFER_OFFLOAD_F_TCP_CKSUM); |
| } |
| |
| tcp_header_t *tcp0 = |
| (tcp_header_t *) (vlib_buffer_get_current (b0) + gho0.l4_hdr_offset); |
| vnet_buffer (b0)->l4_hdr_offset = (u8 *) tcp0 - b0->data; |
| tcp0->ack_number = ack_number; |
| b0->flags &= ~VLIB_BUFFER_IS_TRACED; |
| } |
| |
| static_always_inline u32 |
| vnet_gro_flow_table_flush (vlib_main_t * vm, gro_flow_table_t * flow_table, |
| u32 * to) |
| { |
| if (flow_table->flow_table_size > 0) |
| { |
| gro_flow_t *gro_flow; |
| u32 i = 0, j = 0; |
| while (i < GRO_FLOW_TABLE_MAX_SIZE) |
| { |
| gro_flow = &flow_table->gro_flow[i]; |
| if (gro_flow->n_buffers && gro_flow_is_timeout (vm, gro_flow)) |
| { |
| // flush the packet |
| vlib_buffer_t *b0 = |
| vlib_get_buffer (vm, gro_flow->buffer_index); |
| gro_fixup_header (vm, b0, gro_flow->last_ack_number, |
| flow_table->is_l2); |
| to[j] = gro_flow->buffer_index; |
| gro_flow_table_reset_flow (flow_table, gro_flow); |
| flow_table->n_vectors++; |
| j++; |
| } |
| i++; |
| } |
| |
| return j; |
| } |
| return 0; |
| } |
| |
| static_always_inline void |
| vnet_gro_flow_table_schedule_node_on_dispatcher (vlib_main_t *vm, |
| vnet_hw_if_tx_queue_t *txq, |
| gro_flow_table_t *flow_table) |
| { |
| if (gro_flow_table_is_timeout (vm, flow_table)) |
| { |
| u32 to[GRO_FLOW_TABLE_MAX_SIZE] = { 0 }; |
| u32 n_to = vnet_gro_flow_table_flush (vm, flow_table, to); |
| |
| if (n_to > 0) |
| { |
| u32 node_index = flow_table->node_index; |
| vlib_frame_t *f = vlib_get_frame_to_node (vm, node_index); |
| vnet_hw_if_tx_frame_t *ft = vlib_frame_scalar_args (f); |
| u32 *f_to = vlib_frame_vector_args (f); |
| u32 i = 0; |
| |
| ft->shared_queue = txq->shared_queue; |
| ft->queue_id = txq->queue_id; |
| |
| while (i < n_to) |
| { |
| f_to[f->n_vectors] = to[i]; |
| i++; |
| f->n_vectors++; |
| } |
| vlib_put_frame_to_node (vm, node_index, f); |
| } |
| gro_flow_table_set_timeout (vm, flow_table, GRO_FLOW_TABLE_FLUSH); |
| } |
| } |
| |
| static_always_inline u32 |
| vnet_gro_flush_all_packets (vlib_main_t *vm, gro_flow_table_t *flow_table, |
| gro_flow_t *gro_flow, vlib_buffer_t *b_s, u32 *to, |
| u32 bi_s, u32 bi0, u8 is_l2) |
| { |
| flow_table->n_vectors++; |
| flow_table->total_vectors++; |
| gro_fixup_header (vm, b_s, gro_flow->last_ack_number, is_l2); |
| gro_flow->n_buffers = 0; |
| gro_flow_table_reset_flow (flow_table, gro_flow); |
| to[0] = bi_s; |
| to[1] = bi0; |
| return 2; |
| } |
| |
| static_always_inline u32 |
| vnet_gro_flow_table_inline (vlib_main_t * vm, gro_flow_table_t * flow_table, |
| u32 bi0, u32 * to) |
| { |
| vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0); |
| generic_header_offset_t gho0 = { 0 }; |
| gro_flow_t *gro_flow = 0; |
| gro_flow_key_t flow_key0 = { }; |
| tcp_header_t *tcp0 = 0; |
| u32 pkt_len0 = 0; |
| u32 is_flush = 0; |
| u8 is_l2 = flow_table->is_l2; |
| |
| if (!gro_flow_table_is_enable (flow_table)) |
| { |
| to[0] = bi0; |
| return 1; |
| } |
| |
| if (PREDICT_FALSE (b0->flags & VNET_BUFFER_F_GSO)) |
| { |
| to[0] = bi0; |
| return 1; |
| } |
| |
| pkt_len0 = gro_get_packet_data (vm, b0, &gho0, &flow_key0, is_l2); |
| if (pkt_len0 == 0) |
| { |
| to[0] = bi0; |
| return 1; |
| } |
| |
| tcp0 = (tcp_header_t *) (vlib_buffer_get_current (b0) + gho0.l4_hdr_offset); |
| if (PREDICT_TRUE (((tcp0->flags & TCP_FLAG_PSH) == 0) && |
| (pkt_len0 > GRO_MIN_PACKET_SIZE))) |
| gro_flow = gro_flow_table_find_or_add_flow (flow_table, &flow_key0); |
| else |
| { |
| is_flush = 1; |
| gro_flow = gro_flow_table_get_flow (flow_table, &flow_key0); |
| } |
| |
| if (!gro_flow) |
| { |
| to[0] = bi0; |
| return 1; |
| } |
| |
| if (PREDICT_FALSE (gro_flow->n_buffers == 0)) |
| { |
| flow_table->total_vectors++; |
| gro_flow_store_packet (gro_flow, bi0); |
| gro_flow->last_ack_number = tcp0->ack_number; |
| gro_flow_set_timeout (vm, gro_flow, GRO_FLOW_TIMEOUT); |
| return 0; |
| } |
| else |
| { |
| generic_header_offset_t gho_s = { 0 }; |
| tcp_header_t *tcp_s; |
| u16 l234_sz0, l234_sz_s; |
| u32 pkt_len_s, payload_len0, payload_len_s; |
| u32 bi_s = gro_flow->buffer_index; |
| |
| vlib_buffer_t *b_s = vlib_get_buffer (vm, bi_s); |
| u32 is_ip_s = gro_is_ip4_or_ip6_packet (b_s, is_l2); |
| if (is_ip_s & VNET_BUFFER_F_IS_IP4) |
| vnet_generic_header_offset_parser (b_s, &gho_s, is_l2, |
| 1 /* is_ip4 */ , 0 /* is_ip6 */ ); |
| else if (is_ip_s & VNET_BUFFER_F_IS_IP6) |
| vnet_generic_header_offset_parser (b_s, &gho_s, is_l2, |
| 0 /* is_ip4 */ , 1 /* is_ip6 */ ); |
| |
| tcp_s = |
| (tcp_header_t *) (vlib_buffer_get_current (b_s) + |
| gho_s.l4_hdr_offset); |
| pkt_len_s = vlib_buffer_length_in_chain (vm, b_s); |
| l234_sz0 = gho0.hdr_sz; |
| l234_sz_s = gho_s.hdr_sz; |
| payload_len0 = pkt_len0 - l234_sz0; |
| payload_len_s = pkt_len_s - l234_sz_s; |
| gro_packet_action_t action = |
| gro_tcp_sequence_check (tcp_s, tcp0, payload_len_s); |
| |
| if (PREDICT_TRUE (action == GRO_PACKET_ACTION_ENQUEUE)) |
| { |
| if (PREDICT_TRUE (((pkt_len_s + payload_len0) < TCP_MAX_GSO_SZ) && |
| (gro_flow->n_buffers < GRO_FLOW_N_BUFFERS))) |
| { |
| flow_table->total_vectors++; |
| gro_merge_buffers (vm, b_s, b0, bi0, payload_len0, l234_sz0); |
| gro_flow_store_packet (gro_flow, bi0); |
| gro_flow->last_ack_number = tcp0->ack_number; |
| if (PREDICT_FALSE (is_flush)) |
| { |
| flow_table->n_vectors++; |
| tcp_s->flags |= tcp0->flags; |
| gro_fixup_header (vm, b_s, gro_flow->last_ack_number, is_l2); |
| gro_flow->n_buffers = 0; |
| gro_flow_table_reset_flow (flow_table, gro_flow); |
| to[0] = bi_s; |
| return 1; |
| } |
| return 0; |
| } |
| else if (PREDICT_FALSE (is_flush)) |
| // flush the all (current and stored) packets |
| return vnet_gro_flush_all_packets (vm, flow_table, gro_flow, b_s, |
| to, bi_s, bi0, is_l2); |
| else |
| { |
| // flush the stored GSO size packet and buffer the current packet |
| flow_table->n_vectors++; |
| flow_table->total_vectors++; |
| gro_fixup_header (vm, b_s, gro_flow->last_ack_number, is_l2); |
| gro_flow->n_buffers = 0; |
| gro_flow_store_packet (gro_flow, bi0); |
| gro_flow->last_ack_number = tcp0->ack_number; |
| gro_flow_set_timeout (vm, gro_flow, GRO_FLOW_TIMEOUT); |
| to[0] = bi_s; |
| return 1; |
| } |
| } |
| else |
| { |
| // flush the all (current and stored) packets |
| return vnet_gro_flush_all_packets (vm, flow_table, gro_flow, b_s, to, |
| bi_s, bi0, is_l2); |
| } |
| } |
| } |
| |
| /** |
| * coalesce buffers with flow tables |
| */ |
| static_always_inline u32 |
| vnet_gro_inline (vlib_main_t * vm, gro_flow_table_t * flow_table, u32 * from, |
| u16 n_left_from, u32 * to) |
| { |
| u16 count = 0, i = 0; |
| |
| for (i = 0; i < n_left_from; i++) |
| count += vnet_gro_flow_table_inline (vm, flow_table, from[i], &to[count]); |
| |
| return count; |
| } |
| |
| /** |
| * coalesce buffers in opportunistic way without flow tables |
| */ |
| static_always_inline u32 |
| vnet_gro_simple_inline (vlib_main_t * vm, u32 * from, u16 n_left_from, |
| int is_l2) |
| { |
| vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs; |
| vlib_get_buffers (vm, from, b, n_left_from); |
| u32 bi = 1, ack_number = 0; |
| if (PREDICT_TRUE (((b[0]->flags & VNET_BUFFER_F_GSO) == 0))) |
| { |
| while (n_left_from > 1) |
| { |
| if (PREDICT_TRUE (((b[bi]->flags & VNET_BUFFER_F_GSO) == 0))) |
| { |
| u32 ret; |
| if ((ret = |
| gro_coalesce_buffers (vm, b[0], b[bi], from[bi], |
| is_l2)) != 0) |
| { |
| n_left_from -= 1; |
| bi += 1; |
| ack_number = ret; |
| continue; |
| } |
| else |
| break; |
| } |
| else |
| break; |
| } |
| |
| if (bi >= 2) |
| { |
| gro_fixup_header (vm, b[0], ack_number, is_l2); |
| } |
| } |
| return bi; |
| } |
| #endif /* included_gro_func_h */ |
| |
| /* |
| * fd.io coding-style-patch-verification: ON |
| * |
| * Local Variables: |
| * eval: (c-set-style "gnu") |
| * End: |
| */ |