src/vnet/gso/gro_func.h

/*
 * Copyright (c) 2020 Cisco and/or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef included_gro_func_h
#define included_gro_func_h

#include <vnet/ethernet/ethernet.h>
#include <vnet/gso/gro.h>
#include <vnet/gso/hdr_offset_parser.h>
#include <vnet/ip/ip4.h>
#include <vnet/ip/ip6.h>
#include <vnet/ip/ip6_inlines.h>
#include <vnet/udp/udp_packet.h>
#include <vnet/tcp/tcp_packet.h>
#include <vnet/vnet.h>
#include <vnet/interface.h>

#define GRO_MIN_PACKET_SIZE    256
#define GRO_PADDED_PACKET_SIZE 64

static_always_inline u8
gro_is_bad_packet (vlib_buffer_t * b, u8 flags, i16 l234_sz)
{
  if (((b->current_length - l234_sz) <= 0) ||
      ((flags &= ~(TCP_FLAG_ACK | TCP_FLAG_PSH)) != 0))
    return 1;
  return 0;
}

static_always_inline void
gro_get_ip4_flow_from_packet (u32 * sw_if_index,
			      ip4_header_t * ip4, tcp_header_t * tcp,
			      gro_flow_key_t * flow_key, int is_l2)
{
  flow_key->sw_if_index[VLIB_RX] = sw_if_index[VLIB_RX];
  flow_key->sw_if_index[VLIB_TX] = sw_if_index[VLIB_TX];
  ip46_address_set_ip4 (&flow_key->src_address, &ip4->src_address);
  ip46_address_set_ip4 (&flow_key->dst_address, &ip4->dst_address);
  flow_key->src_port = tcp->src_port;
  flow_key->dst_port = tcp->dst_port;
}

static_always_inline void
gro_get_ip6_flow_from_packet (u32 * sw_if_index,
			      ip6_header_t * ip6, tcp_header_t * tcp,
			      gro_flow_key_t * flow_key, int is_l2)
{
  flow_key->sw_if_index[VLIB_RX] = sw_if_index[VLIB_RX];
  flow_key->sw_if_index[VLIB_TX] = sw_if_index[VLIB_TX];
  ip46_address_set_ip6 (&flow_key->src_address, &ip6->src_address);
  ip46_address_set_ip6 (&flow_key->dst_address, &ip6->dst_address);
  flow_key->src_port = tcp->src_port;
  flow_key->dst_port = tcp->dst_port;
}

static_always_inline u32
gro_is_ip4_or_ip6_packet (vlib_buffer_t *b0, u8 is_l2)
{
  if (b0->flags & VNET_BUFFER_F_IS_IP4)
    return VNET_BUFFER_F_IS_IP4;
  if (b0->flags & VNET_BUFFER_F_IS_IP6)
    return VNET_BUFFER_F_IS_IP6;
  if (is_l2)
    {
      ethernet_header_t *eh =
	(ethernet_header_t *) vlib_buffer_get_current (b0);
      u16 ethertype = clib_net_to_host_u16 (eh->type);

      if (ethernet_frame_is_tagged (ethertype))
	{
	  ethernet_vlan_header_t *vlan = (ethernet_vlan_header_t *) (eh + 1);

	  ethertype = clib_net_to_host_u16 (vlan->type);
	  if (ethertype == ETHERNET_TYPE_VLAN)
	    {
	      vlan++;
	      ethertype = clib_net_to_host_u16 (vlan->type);
	    }
	}
      if (ethertype == ETHERNET_TYPE_IP4)
	return VNET_BUFFER_F_IS_IP4;
      if (ethertype == ETHERNET_TYPE_IP6)
	return VNET_BUFFER_F_IS_IP6;
    }
  else
    {
      if ((((u8 *) vlib_buffer_get_current (b0))[0] & 0xf0) == 0x40)
	return VNET_BUFFER_F_IS_IP4;
      if ((((u8 *) vlib_buffer_get_current (b0))[0] & 0xf0) == 0x60)
	return VNET_BUFFER_F_IS_IP6;
    }

  return 0;
}

typedef enum
{
  GRO_PACKET_ACTION_NONE = 0,
  GRO_PACKET_ACTION_ENQUEUE = 1,
  GRO_PACKET_ACTION_FLUSH = 2,
} gro_packet_action_t;

static_always_inline gro_packet_action_t
gro_tcp_sequence_check (tcp_header_t * tcp0, tcp_header_t * tcp1,
			u32 payload_len0)
{
  u32 next_tcp_seq0 = clib_net_to_host_u32 (tcp0->seq_number);
  u32 next_tcp_seq1 = clib_net_to_host_u32 (tcp1->seq_number);

  /* next packet, enqueue */
  if (PREDICT_TRUE (next_tcp_seq0 + payload_len0 == next_tcp_seq1))
    return GRO_PACKET_ACTION_ENQUEUE;
  /* flush all packets */
  else
    return GRO_PACKET_ACTION_FLUSH;
}

static_always_inline void
gro_merge_buffers (vlib_main_t * vm, vlib_buffer_t * b0,
		   vlib_buffer_t * b1, u32 bi1, u32 payload_len1,
		   u16 l234_sz1)
{
  vlib_buffer_t *pb = b0;

  if (PREDICT_FALSE ((b0->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
    b0->total_length_not_including_first_buffer = 0;

  while (pb->flags & VLIB_BUFFER_NEXT_PRESENT)
    pb = vlib_get_buffer (vm, pb->next_buffer);

  vlib_buffer_advance (b1, l234_sz1);
  pb->flags |= VLIB_BUFFER_NEXT_PRESENT;
  pb->next_buffer = bi1;
  b0->total_length_not_including_first_buffer += payload_len1;
  b0->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
}

static_always_inline u32
gro_validate_checksum (vlib_main_t * vm, vlib_buffer_t * b0,
		       generic_header_offset_t * gho0, int is_ip4)
{
  u32 flags = 0;

  if (b0->flags & VNET_BUFFER_F_OFFLOAD)
    return VNET_BUFFER_F_L4_CHECKSUM_CORRECT;
  vlib_buffer_advance (b0, gho0->l3_hdr_offset);
  if (is_ip4)
    flags = ip4_tcp_udp_validate_checksum (vm, b0);
  else
    flags = ip6_tcp_udp_icmp_validate_checksum (vm, b0);
  vlib_buffer_advance (b0, -gho0->l3_hdr_offset);
  return flags;
}

static_always_inline u32
gro_fix_padded_packet_len (vlib_buffer_t *b0, generic_header_offset_t *gho0,
			   ip4_header_t *ip4_0, ip6_header_t *ip6_0,
			   u32 pkt_len0, u16 l234_sz0)
{
  u32 tcp_payload_len0 = 0;
  if (gho0->gho_flags & GHO_F_IP4)
    {
      tcp_payload_len0 = clib_net_to_host_u16 (ip4_0->length) -
			 ip4_header_bytes (ip4_0) - gho0->l4_hdr_sz;
    }
  else
    {
      tcp_payload_len0 =
	clib_net_to_host_u16 (ip6_0->payload_length) - gho0->l4_hdr_sz;
    }

  ASSERT (l234_sz0 + tcp_payload_len0 <= pkt_len0);

  if (PREDICT_FALSE (l234_sz0 + tcp_payload_len0 < pkt_len0))
    {
      /* small packet with padding at the end, remove padding */
      b0->current_length = l234_sz0 + tcp_payload_len0;
      pkt_len0 = b0->current_length;
    }
  return pkt_len0;
}

static_always_inline u32
gro_get_packet_data (vlib_main_t *vm, vlib_buffer_t *b0,
		     generic_header_offset_t *gho0, gro_flow_key_t *flow_key0,
		     u8 is_l2)
{
  ip4_header_t *ip4_0 = 0;
  ip6_header_t *ip6_0 = 0;
  tcp_header_t *tcp0 = 0;
  u32 flags = 0;
  u32 pkt_len0 = 0;
  u16 l234_sz0 = 0;
  u32 sw_if_index0[VLIB_N_RX_TX] = { ~0 };

  u32 is_ip0 = gro_is_ip4_or_ip6_packet (b0, is_l2);

  if (is_ip0 & VNET_BUFFER_F_IS_IP4)
    vnet_generic_header_offset_parser (b0, gho0, is_l2, 1 /* is_ip4 */ ,
				       0 /* is_ip6 */ );
  else if (is_ip0 & VNET_BUFFER_F_IS_IP6)
    vnet_generic_header_offset_parser (b0, gho0, is_l2, 0 /* is_ip4 */ ,
				       1 /* is_ip6 */ );
  else
    return 0;

  if (PREDICT_FALSE ((gho0->gho_flags & GHO_F_TCP) == 0))
    return 0;

  ip4_0 =
    (ip4_header_t *) (vlib_buffer_get_current (b0) + gho0->l3_hdr_offset);
  ip6_0 =
    (ip6_header_t *) (vlib_buffer_get_current (b0) + gho0->l3_hdr_offset);
  tcp0 =
    (tcp_header_t *) (vlib_buffer_get_current (b0) + gho0->l4_hdr_offset);

  l234_sz0 = gho0->hdr_sz;
  if (PREDICT_FALSE (gro_is_bad_packet (b0, tcp0->flags, l234_sz0)))
    return 0;

  sw_if_index0[VLIB_RX] = vnet_buffer (b0)->sw_if_index[VLIB_RX];
  sw_if_index0[VLIB_TX] = vnet_buffer (b0)->sw_if_index[VLIB_TX];

  if (gho0->gho_flags & GHO_F_IP4)
    {
      flags = gro_validate_checksum (vm, b0, gho0, 1);
      gro_get_ip4_flow_from_packet (sw_if_index0, ip4_0, tcp0, flow_key0,
				    is_l2);
    }
  else if (gho0->gho_flags & GHO_F_IP6)
    {
      flags = gro_validate_checksum (vm, b0, gho0, 0);
      gro_get_ip6_flow_from_packet (sw_if_index0, ip6_0, tcp0, flow_key0,
				    is_l2);
    }
  else
    return 0;

  if (PREDICT_FALSE ((flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) == 0))
    return 0;

  pkt_len0 = vlib_buffer_length_in_chain (vm, b0);
  if (PREDICT_FALSE (pkt_len0 >= TCP_MAX_GSO_SZ))
    return 0;

  if (PREDICT_FALSE (pkt_len0 <= GRO_PADDED_PACKET_SIZE))
    {
      pkt_len0 =
	gro_fix_padded_packet_len (b0, gho0, ip4_0, ip6_0, pkt_len0, l234_sz0);
    }
  return pkt_len0;
}

static_always_inline u32
gro_coalesce_buffers (vlib_main_t *vm, vlib_buffer_t *b0, vlib_buffer_t *b1,
		      u32 bi1, u8 is_l2)
{
  generic_header_offset_t gho0 = { 0 };
  generic_header_offset_t gho1 = { 0 };
  gro_flow_key_t flow_key0, flow_key1;
  ip4_header_t *ip4_0, *ip4_1;
  ip6_header_t *ip6_0, *ip6_1;
  tcp_header_t *tcp0, *tcp1;
  u16 l234_sz0, l234_sz1;
  u32 pkt_len0, pkt_len1, payload_len0, payload_len1;
  u32 sw_if_index0[VLIB_N_RX_TX] = { ~0 };
  u32 sw_if_index1[VLIB_N_RX_TX] = { ~0 };

  u32 is_ip0 = gro_is_ip4_or_ip6_packet (b0, is_l2);
  u32 is_ip1 = gro_is_ip4_or_ip6_packet (b1, is_l2);

  if (is_ip0 & VNET_BUFFER_F_IS_IP4)
    vnet_generic_header_offset_parser (b0, &gho0, is_l2, 1 /* is_ip4 */ ,
				       0 /* is_ip6 */ );
  else if (is_ip0 & VNET_BUFFER_F_IS_IP6)
    vnet_generic_header_offset_parser (b0, &gho0, is_l2, 0 /* is_ip4 */ ,
				       1 /* is_ip6 */ );
  else
    return 0;

  if (is_ip1 & VNET_BUFFER_F_IS_IP4)
    vnet_generic_header_offset_parser (b1, &gho1, is_l2, 1 /* is_ip4 */ ,
				       0 /* is_ip6 */ );
  else if (is_ip1 & VNET_BUFFER_F_IS_IP6)
    vnet_generic_header_offset_parser (b1, &gho1, is_l2, 0 /* is_ip4 */ ,
				       1 /* is_ip6 */ );
  else
    return 0;

  pkt_len0 = vlib_buffer_length_in_chain (vm, b0);
  pkt_len1 = vlib_buffer_length_in_chain (vm, b1);

  if (((gho0.gho_flags & GHO_F_TCP) == 0 || pkt_len0 <= GRO_MIN_PACKET_SIZE) ||
      ((gho1.gho_flags & GHO_F_TCP) == 0 || pkt_len1 <= GRO_MIN_PACKET_SIZE))
    return 0;

  ip4_0 =
    (ip4_header_t *) (vlib_buffer_get_current (b0) + gho0.l3_hdr_offset);
  ip4_1 =
    (ip4_header_t *) (vlib_buffer_get_current (b1) + gho1.l3_hdr_offset);
  ip6_0 =
    (ip6_header_t *) (vlib_buffer_get_current (b0) + gho0.l3_hdr_offset);
  ip6_1 =
    (ip6_header_t *) (vlib_buffer_get_current (b1) + gho1.l3_hdr_offset);

  tcp0 = (tcp_header_t *) (vlib_buffer_get_current (b0) + gho0.l4_hdr_offset);
  tcp1 = (tcp_header_t *) (vlib_buffer_get_current (b1) + gho1.l4_hdr_offset);

  l234_sz0 = gho0.hdr_sz;
  l234_sz1 = gho1.hdr_sz;

  if (gro_is_bad_packet (b0, tcp0->flags, l234_sz0)
      || gro_is_bad_packet (b1, tcp1->flags, l234_sz1))
    return 0;

  sw_if_index0[VLIB_RX] = vnet_buffer (b0)->sw_if_index[VLIB_RX];
  sw_if_index0[VLIB_TX] = vnet_buffer (b0)->sw_if_index[VLIB_TX];

  sw_if_index1[VLIB_RX] = vnet_buffer (b1)->sw_if_index[VLIB_RX];
  sw_if_index1[VLIB_TX] = vnet_buffer (b1)->sw_if_index[VLIB_TX];

  if ((gho0.gho_flags & GHO_F_IP4) && (gho1.gho_flags & GHO_F_IP4))
    {
      gro_get_ip4_flow_from_packet (sw_if_index0, ip4_0, tcp0, &flow_key0,
				    is_l2);
      gro_get_ip4_flow_from_packet (sw_if_index1, ip4_1, tcp1, &flow_key1,
				    is_l2);
    }
  else if ((gho0.gho_flags & GHO_F_IP6) && (gho1.gho_flags & GHO_F_IP6))
    {
      gro_get_ip6_flow_from_packet (sw_if_index0, ip6_0, tcp0, &flow_key0,
				    is_l2);
      gro_get_ip6_flow_from_packet (sw_if_index1, ip6_1, tcp1, &flow_key1,
				    is_l2);
    }
  else
    return 0;

  if (gro_flow_is_equal (&flow_key0, &flow_key1) == 0)
    return 0;

  payload_len0 = pkt_len0 - l234_sz0;
  payload_len1 = pkt_len1 - l234_sz1;

  if (pkt_len0 >= TCP_MAX_GSO_SZ || pkt_len1 >= TCP_MAX_GSO_SZ
      || (pkt_len0 + payload_len1) >= TCP_MAX_GSO_SZ)
    return 0;

  if (gro_tcp_sequence_check (tcp0, tcp1, payload_len0) ==
      GRO_PACKET_ACTION_ENQUEUE)
    {
      gro_merge_buffers (vm, b0, b1, bi1, payload_len1, l234_sz1);
      tcp0->flags |= tcp1->flags;
      return tcp1->ack_number;
    }

  return 0;
}

static_always_inline void
gro_fixup_header (vlib_main_t *vm, vlib_buffer_t *b0, u32 ack_number, u8 is_l2)
{
  generic_header_offset_t gho0 = { 0 };

  u32 is_ip0 = gro_is_ip4_or_ip6_packet (b0, is_l2);

  if (is_ip0 & VNET_BUFFER_F_IS_IP4)
    vnet_generic_header_offset_parser (b0, &gho0, is_l2, 1 /* is_ip4 */ ,
				       0 /* is_ip6 */ );
  else if (is_ip0 & VNET_BUFFER_F_IS_IP6)
    vnet_generic_header_offset_parser (b0, &gho0, is_l2, 0 /* is_ip4 */ ,
				       1 /* is_ip6 */ );

  vnet_buffer2 (b0)->gso_size = b0->current_length - gho0.hdr_sz;

  if (gho0.gho_flags & GHO_F_IP4)
    {
      ip4_header_t *ip4 =
	(ip4_header_t *) (vlib_buffer_get_current (b0) + gho0.l3_hdr_offset);
      ip4->length =
	clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) -
			      gho0.l3_hdr_offset);
      b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP4);
      vnet_buffer_offload_flags_set (b0, (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM |
					  VNET_BUFFER_OFFLOAD_F_IP_CKSUM));
    }
  else if (gho0.gho_flags & GHO_F_IP6)
    {
      ip6_header_t *ip6 =
	(ip6_header_t *) (vlib_buffer_get_current (b0) + gho0.l3_hdr_offset);
      ip6->payload_length =
	clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) -
			      gho0.l4_hdr_offset);
      b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP6);
      vnet_buffer_offload_flags_set (b0, VNET_BUFFER_OFFLOAD_F_TCP_CKSUM);
    }

  tcp_header_t *tcp0 =
    (tcp_header_t *) (vlib_buffer_get_current (b0) + gho0.l4_hdr_offset);
  tcp0->ack_number = ack_number;
  b0->flags &= ~VLIB_BUFFER_IS_TRACED;
}

static_always_inline u32
vnet_gro_flow_table_flush (vlib_main_t * vm, gro_flow_table_t * flow_table,
			   u32 * to)
{
  if (flow_table->flow_table_size > 0)
    {
      gro_flow_t *gro_flow;
      u32 i = 0, j = 0;
      while (i < GRO_FLOW_TABLE_MAX_SIZE)
	{
	  gro_flow = &flow_table->gro_flow[i];
	  if (gro_flow->n_buffers && gro_flow_is_timeout (vm, gro_flow))
	    {
	      // flush the packet
	      vlib_buffer_t *b0 =
		vlib_get_buffer (vm, gro_flow->buffer_index);
	      gro_fixup_header (vm, b0, gro_flow->last_ack_number,
				flow_table->is_l2);
	      to[j] = gro_flow->buffer_index;
	      gro_flow_table_reset_flow (flow_table, gro_flow);
	      flow_table->n_vectors++;
	      j++;
	    }
	  i++;
	}

      return j;
    }
  return 0;
}

static_always_inline void
vnet_gro_flow_table_schedule_node_on_dispatcher (vlib_main_t *vm,
						 vnet_hw_if_tx_queue_t *txq,
						 gro_flow_table_t *flow_table)
{
  if (gro_flow_table_is_timeout (vm, flow_table))
    {
      u32 to[GRO_FLOW_TABLE_MAX_SIZE] = { 0 };
      u32 n_to = vnet_gro_flow_table_flush (vm, flow_table, to);

      if (n_to > 0)
	{
	  u32 node_index = flow_table->node_index;
	  vlib_frame_t *f = vlib_get_frame_to_node (vm, node_index);
	  vnet_hw_if_tx_frame_t *ft = vlib_frame_scalar_args (f);
	  u32 *f_to = vlib_frame_vector_args (f);
	  u32 i = 0;

	  ft->shared_queue = txq->shared_queue;
	  ft->queue_id = txq->queue_id;

	  while (i < n_to)
	    {
	      f_to[f->n_vectors] = to[i];
	      i++;
	      f->n_vectors++;
	    }
	  vlib_put_frame_to_node (vm, node_index, f);
	}
      gro_flow_table_set_timeout (vm, flow_table, GRO_FLOW_TABLE_FLUSH);
    }
}

static_always_inline u32
vnet_gro_flush_all_packets (vlib_main_t *vm, gro_flow_table_t *flow_table,
			    gro_flow_t *gro_flow, vlib_buffer_t *b_s, u32 *to,
			    u32 bi_s, u32 bi0, u8 is_l2)
{
  flow_table->n_vectors++;
  flow_table->total_vectors++;
  gro_fixup_header (vm, b_s, gro_flow->last_ack_number, is_l2);
  gro_flow->n_buffers = 0;
  gro_flow_table_reset_flow (flow_table, gro_flow);
  to[0] = bi_s;
  to[1] = bi0;
  return 2;
}

static_always_inline u32
vnet_gro_flow_table_inline (vlib_main_t * vm, gro_flow_table_t * flow_table,
			    u32 bi0, u32 * to)
{
  vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0);
  generic_header_offset_t gho0 = { 0 };
  gro_flow_t *gro_flow = 0;
  gro_flow_key_t flow_key0 = { };
  tcp_header_t *tcp0 = 0;
  u32 pkt_len0 = 0;
  u32 is_flush = 0;
  u8 is_l2 = flow_table->is_l2;

  if (!gro_flow_table_is_enable (flow_table))
    {
      to[0] = bi0;
      return 1;
    }

  if (PREDICT_FALSE (b0->flags & VNET_BUFFER_F_GSO))
    {
      to[0] = bi0;
      return 1;
    }

  pkt_len0 = gro_get_packet_data (vm, b0, &gho0, &flow_key0, is_l2);
  if (pkt_len0 == 0)
    {
      to[0] = bi0;
      return 1;
    }

  tcp0 = (tcp_header_t *) (vlib_buffer_get_current (b0) + gho0.l4_hdr_offset);
  if (PREDICT_TRUE (((tcp0->flags & TCP_FLAG_PSH) == 0) &&
		    (pkt_len0 > GRO_MIN_PACKET_SIZE)))
    gro_flow = gro_flow_table_find_or_add_flow (flow_table, &flow_key0);
  else
    {
      is_flush = 1;
      gro_flow = gro_flow_table_get_flow (flow_table, &flow_key0);
    }

  if (!gro_flow)
    {
      to[0] = bi0;
      return 1;
    }

  if (PREDICT_FALSE (gro_flow->n_buffers == 0))
    {
      flow_table->total_vectors++;
      gro_flow_store_packet (gro_flow, bi0);
      gro_flow->last_ack_number = tcp0->ack_number;
      gro_flow_set_timeout (vm, gro_flow, GRO_FLOW_TIMEOUT);
      return 0;
    }
  else
    {
      generic_header_offset_t gho_s = { 0 };
      tcp_header_t *tcp_s;
      u16 l234_sz0, l234_sz_s;
      u32 pkt_len_s, payload_len0, payload_len_s;
      u32 bi_s = gro_flow->buffer_index;

      vlib_buffer_t *b_s = vlib_get_buffer (vm, bi_s);
      u32 is_ip_s = gro_is_ip4_or_ip6_packet (b_s, is_l2);
      if (is_ip_s & VNET_BUFFER_F_IS_IP4)
	vnet_generic_header_offset_parser (b_s, &gho_s, is_l2,
					   1 /* is_ip4 */ , 0 /* is_ip6 */ );
      else if (is_ip_s & VNET_BUFFER_F_IS_IP6)
	vnet_generic_header_offset_parser (b_s, &gho_s, is_l2,
					   0 /* is_ip4 */ , 1 /* is_ip6 */ );

      tcp_s =
	(tcp_header_t *) (vlib_buffer_get_current (b_s) +
			  gho_s.l4_hdr_offset);
      pkt_len_s = vlib_buffer_length_in_chain (vm, b_s);
      l234_sz0 = gho0.hdr_sz;
      l234_sz_s = gho_s.hdr_sz;
      payload_len0 = pkt_len0 - l234_sz0;
      payload_len_s = pkt_len_s - l234_sz_s;
      gro_packet_action_t action =
	gro_tcp_sequence_check (tcp_s, tcp0, payload_len_s);

      if (PREDICT_TRUE (action == GRO_PACKET_ACTION_ENQUEUE))
	{
	  if (PREDICT_TRUE (((pkt_len_s + payload_len0) < TCP_MAX_GSO_SZ) &&
			    (gro_flow->n_buffers < GRO_FLOW_N_BUFFERS)))
	    {
	      flow_table->total_vectors++;
	      gro_merge_buffers (vm, b_s, b0, bi0, payload_len0, l234_sz0);
	      gro_flow_store_packet (gro_flow, bi0);
	      gro_flow->last_ack_number = tcp0->ack_number;
	      if (PREDICT_FALSE (is_flush))
		{
		  flow_table->n_vectors++;
		  tcp_s->flags |= tcp0->flags;
		  gro_fixup_header (vm, b_s, gro_flow->last_ack_number, is_l2);
		  gro_flow->n_buffers = 0;
		  gro_flow_table_reset_flow (flow_table, gro_flow);
		  to[0] = bi_s;
		  return 1;
		}
	      return 0;
	    }
	  else if (PREDICT_FALSE (is_flush))
	    // flush the all (current and stored) packets
	    return vnet_gro_flush_all_packets (vm, flow_table, gro_flow, b_s,
					       to, bi_s, bi0, is_l2);
	  else
	    {
	      // flush the stored GSO size packet and buffer the current packet
	      flow_table->n_vectors++;
	      flow_table->total_vectors++;
	      gro_fixup_header (vm, b_s, gro_flow->last_ack_number, is_l2);
	      gro_flow->n_buffers = 0;
	      gro_flow_store_packet (gro_flow, bi0);
	      gro_flow->last_ack_number = tcp0->ack_number;
	      gro_flow_set_timeout (vm, gro_flow, GRO_FLOW_TIMEOUT);
	      to[0] = bi_s;
	      return 1;
	    }
	}
      else
	{
	  // flush the all (current and stored) packets
	  return vnet_gro_flush_all_packets (vm, flow_table, gro_flow, b_s, to,
					     bi_s, bi0, is_l2);
	}
    }
}

/**
 * coalesce buffers with flow tables
 */
static_always_inline u32
vnet_gro_inline (vlib_main_t * vm, gro_flow_table_t * flow_table, u32 * from,
		 u16 n_left_from, u32 * to)
{
  u16 count = 0, i = 0;

  for (i = 0; i < n_left_from; i++)
    count += vnet_gro_flow_table_inline (vm, flow_table, from[i], &to[count]);

  return count;
}

/**
 * coalesce buffers in opportunistic way without flow tables
 */
static_always_inline u32
vnet_gro_simple_inline (vlib_main_t * vm, u32 * from, u16 n_left_from,
			int is_l2)
{
  vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
  vlib_get_buffers (vm, from, b, n_left_from);
  u32 bi = 1, ack_number = 0;
  if (PREDICT_TRUE (((b[0]->flags & VNET_BUFFER_F_GSO) == 0)))
    {
      while (n_left_from > 1)
	{
	  if (PREDICT_TRUE (((b[bi]->flags & VNET_BUFFER_F_GSO) == 0)))
	    {
	      u32 ret;
	      if ((ret =
		   gro_coalesce_buffers (vm, b[0], b[bi], from[bi],
					 is_l2)) != 0)
		{
		  n_left_from -= 1;
		  bi += 1;
		  ack_number = ret;
		  continue;
		}
	      else
		break;
	    }
	  else
	    break;
	}

      if (bi >= 2)
	{
	  gro_fixup_header (vm, b[0], ack_number, is_l2);
	}
    }
  return bi;
}
#endif /* included_gro_func_h */

/*
 * fd.io coding-style-patch-verification: ON
 *
 * Local Variables:
 * eval: (c-set-style "gnu")
 * End:
 */