src/vnet/ipsec/ipsec_output.c

/*
 * ipsec_output.c : IPSec output node
 *
 * Copyright (c) 2015 Cisco and/or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <vnet/vnet.h>
#include <vnet/api_errno.h>
#include <vnet/ip/ip.h>

#include <vnet/ipsec/ipsec.h>
#include <vnet/ipsec/ipsec_io.h>

#define foreach_ipsec_output_error                   \
 _(RX_PKTS, "IPSec pkts received")                   \
 _(POLICY_DISCARD, "IPSec policy discard")           \
 _(POLICY_NO_MATCH, "IPSec policy (no match)")       \
 _(POLICY_PROTECT, "IPSec policy protect")           \
 _(POLICY_BYPASS, "IPSec policy bypass")             \
 _(ENCAPS_FAILED, "IPSec encapsulation failed")

typedef enum
{
#define _(sym,str) IPSEC_OUTPUT_ERROR_##sym,
  foreach_ipsec_output_error
#undef _
    IPSEC_DECAP_N_ERROR,
} ipsec_output_error_t;

static char *ipsec_output_error_strings[] = {
#define _(sym,string) string,
  foreach_ipsec_output_error
#undef _
};

typedef struct
{
  u32 spd_id;
  u32 policy_id;
} ipsec_output_trace_t;

/* packet trace format function */
static u8 *
format_ipsec_output_trace (u8 * s, va_list * args)
{
  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  ipsec_output_trace_t *t = va_arg (*args, ipsec_output_trace_t *);

  s = format (s, "spd %u policy %d", t->spd_id, t->policy_id);

  return s;
}

always_inline void
ipsec4_out_spd_add_flow_cache_entry (ipsec_main_t *im, u8 pr, u32 la, u32 ra,
				     u16 lp, u16 rp, u32 pol_id)
{
  u64 hash;
  u8 overwrite = 0, stale_overwrite = 0;
  ipsec4_spd_5tuple_t ip4_5tuple = { .ip4_addr = { (ip4_address_t) la,
						   (ip4_address_t) ra },
				     .port = { lp, rp },
				     .proto = pr };

  ip4_5tuple.kv_16_8.value = (((u64) pol_id) << 32) | ((u64) im->epoch_count);

  hash = ipsec4_hash_16_8 (&ip4_5tuple.kv_16_8);
  hash &= (im->ipsec4_out_spd_hash_num_buckets - 1);

  ipsec_spinlock_lock (&im->ipsec4_out_spd_hash_tbl[hash].bucket_lock);
  /* Check if we are overwriting an existing entry so we know
  whether to increment the flow cache counter. Since flow
  cache counter is reset on any policy add/remove, but
  hash table values are not, we also need to check if the entry
  we are overwriting is stale or not. If it's a stale entry
  overwrite, we still want to increment flow cache counter */
  overwrite = (im->ipsec4_out_spd_hash_tbl[hash].value != 0);
  /* Check for stale entry by comparing with current epoch count */
  if (PREDICT_FALSE (overwrite))
    stale_overwrite =
      (im->epoch_count !=
       ((u32) (im->ipsec4_out_spd_hash_tbl[hash].value & 0xFFFFFFFF)));
  clib_memcpy_fast (&im->ipsec4_out_spd_hash_tbl[hash], &ip4_5tuple.kv_16_8,
		    sizeof (ip4_5tuple.kv_16_8));
  ipsec_spinlock_unlock (&im->ipsec4_out_spd_hash_tbl[hash].bucket_lock);

  /* Increment the counter to track active flow cache entries
    when entering a fresh entry or overwriting a stale one */
  if (!overwrite || stale_overwrite)
    clib_atomic_fetch_add_relax (&im->ipsec4_out_spd_flow_cache_entries, 1);

  return;
}

always_inline ipsec_policy_t *
ipsec4_out_spd_find_flow_cache_entry (ipsec_main_t *im, u8 pr, u32 la, u32 ra,
				      u16 lp, u16 rp)
{
  ipsec_policy_t *p = NULL;
  ipsec4_hash_kv_16_8_t kv_result;
  u64 hash;

  if (PREDICT_FALSE ((pr != IP_PROTOCOL_TCP) && (pr != IP_PROTOCOL_UDP) &&
		     (pr != IP_PROTOCOL_SCTP)))
    {
      lp = 0;
      rp = 0;
    }
  ipsec4_spd_5tuple_t ip4_5tuple = { .ip4_addr = { (ip4_address_t) la,
						   (ip4_address_t) ra },
				     .port = { lp, rp },
				     .proto = pr };

  hash = ipsec4_hash_16_8 (&ip4_5tuple.kv_16_8);
  hash &= (im->ipsec4_out_spd_hash_num_buckets - 1);

  ipsec_spinlock_lock (&im->ipsec4_out_spd_hash_tbl[hash].bucket_lock);
  kv_result = im->ipsec4_out_spd_hash_tbl[hash];
  ipsec_spinlock_unlock (&im->ipsec4_out_spd_hash_tbl[hash].bucket_lock);

  if (ipsec4_hash_key_compare_16_8 ((u64 *) &ip4_5tuple.kv_16_8,
				    (u64 *) &kv_result))
    {
      if (im->epoch_count == ((u32) (kv_result.value & 0xFFFFFFFF)))
	{
	  /* Get the policy based on the index */
	  p =
	    pool_elt_at_index (im->policies, ((u32) (kv_result.value >> 32)));
	}
    }

  return p;
}

always_inline ipsec_policy_t *
ipsec_output_policy_match (ipsec_spd_t *spd, u8 pr, u32 la, u32 ra, u16 lp,
			   u16 rp, u8 flow_cache_enabled)
{
  ipsec_main_t *im = &ipsec_main;
  ipsec_policy_t *p;
  u32 *i;

  if (!spd)
    return 0;

  vec_foreach (i, spd->policies[IPSEC_SPD_POLICY_IP4_OUTBOUND])
  {
    p = pool_elt_at_index (im->policies, *i);
    if (PREDICT_FALSE (p->protocol && (p->protocol != pr)))
      continue;

    if (ra < clib_net_to_host_u32 (p->raddr.start.ip4.as_u32))
      continue;

    if (ra > clib_net_to_host_u32 (p->raddr.stop.ip4.as_u32))
      continue;

    if (la < clib_net_to_host_u32 (p->laddr.start.ip4.as_u32))
      continue;

    if (la > clib_net_to_host_u32 (p->laddr.stop.ip4.as_u32))
      continue;

    if (PREDICT_FALSE ((pr != IP_PROTOCOL_TCP) && (pr != IP_PROTOCOL_UDP) &&
		       (pr != IP_PROTOCOL_SCTP)))
      {
	lp = 0;
	rp = 0;
	goto add_flow_cache;
      }

    if (lp < p->lport.start)
      continue;

    if (lp > p->lport.stop)
      continue;

    if (rp < p->rport.start)
      continue;

    if (rp > p->rport.stop)
      continue;

  add_flow_cache:
    if (flow_cache_enabled)
      {
	/* Add an Entry in Flow cache */
	ipsec4_out_spd_add_flow_cache_entry (
	  im, pr, clib_host_to_net_u32 (la), clib_host_to_net_u32 (ra),
	  clib_host_to_net_u16 (lp), clib_host_to_net_u16 (rp), *i);
      }

    return p;
  }
  return 0;
}

always_inline uword
ip6_addr_match_range (ip6_address_t * a, ip6_address_t * la,
		      ip6_address_t * ua)
{
  if ((memcmp (a->as_u64, la->as_u64, 2 * sizeof (u64)) >= 0) &&
      (memcmp (a->as_u64, ua->as_u64, 2 * sizeof (u64)) <= 0))
    return 1;
  return 0;
}

always_inline ipsec_policy_t *
ipsec6_output_policy_match (ipsec_spd_t * spd,
			    ip6_address_t * la,
			    ip6_address_t * ra, u16 lp, u16 rp, u8 pr)
{
  ipsec_main_t *im = &ipsec_main;
  ipsec_policy_t *p;
  u32 *i;

  if (!spd)
    return 0;

  vec_foreach (i, spd->policies[IPSEC_SPD_POLICY_IP6_OUTBOUND])
  {
    p = pool_elt_at_index (im->policies, *i);
    if (PREDICT_FALSE (p->protocol && (p->protocol != pr)))
      continue;

    if (!ip6_addr_match_range (ra, &p->raddr.start.ip6, &p->raddr.stop.ip6))
      continue;

    if (!ip6_addr_match_range (la, &p->laddr.start.ip6, &p->laddr.stop.ip6))
      continue;

    if (PREDICT_FALSE
	((pr != IP_PROTOCOL_TCP) && (pr != IP_PROTOCOL_UDP)
	 && (pr != IP_PROTOCOL_SCTP)))
      return p;

    if (lp < p->lport.start)
      continue;

    if (lp > p->lport.stop)
      continue;

    if (rp < p->rport.start)
      continue;

    if (rp > p->rport.stop)
      continue;

    return p;
  }

  return 0;
}

static inline uword
ipsec_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
		     vlib_frame_t * from_frame, int is_ipv6)
{
  ipsec_main_t *im = &ipsec_main;

  u32 *from, *to_next = 0, thread_index;
  u32 n_left_from, sw_if_index0, last_sw_if_index = (u32) ~ 0;
  u32 next_node_index = (u32) ~ 0, last_next_node_index = (u32) ~ 0;
  vlib_frame_t *f = 0;
  u32 spd_index0 = ~0;
  ipsec_spd_t *spd0 = 0;
  int bogus;
  u64 nc_protect = 0, nc_bypass = 0, nc_discard = 0, nc_nomatch = 0;
  u8 flow_cache_enabled = im->output_flow_cache_flag;

  from = vlib_frame_vector_args (from_frame);
  n_left_from = from_frame->n_vectors;
  thread_index = vm->thread_index;

  while (n_left_from > 0)
    {
      u32 bi0, pi0, bi1;
      vlib_buffer_t *b0, *b1;
      ipsec_policy_t *p0 = NULL;
      ip4_header_t *ip0;
      ip6_header_t *ip6_0 = 0;
      udp_header_t *udp0;
      u32 iph_offset = 0;
      tcp_header_t *tcp0;
      u64 bytes0;

      bi0 = from[0];
      b0 = vlib_get_buffer (vm, bi0);
      if (n_left_from > 1)
	{
	  bi1 = from[1];
	  b1 = vlib_get_buffer (vm, bi1);
	  CLIB_PREFETCH (b1, CLIB_CACHE_LINE_BYTES * 2, STORE);
	  vlib_prefetch_buffer_data (b1, LOAD);
	}
      sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_TX];
      iph_offset = vnet_buffer (b0)->ip.save_rewrite_length;
      ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0)
			      + iph_offset);

      /* lookup for SPD only if sw_if_index is changed */
      if (PREDICT_FALSE (last_sw_if_index != sw_if_index0))
	{
	  uword *p = hash_get (im->spd_index_by_sw_if_index, sw_if_index0);
	  ALWAYS_ASSERT (p);
	  spd_index0 = p[0];
	  spd0 = pool_elt_at_index (im->spds, spd_index0);
	  last_sw_if_index = sw_if_index0;
	}

      if (is_ipv6)
	{
	  ip6_0 = (ip6_header_t *) ((u8 *) vlib_buffer_get_current (b0)
				    + iph_offset);

	  udp0 = ip6_next_header (ip6_0);
#if 0
	  clib_warning
	    ("packet received from %U port %u to %U port %u spd_id %u",
	     format_ip6_address, &ip6_0->src_address,
	     clib_net_to_host_u16 (udp0->src_port), format_ip6_address,
	     &ip6_0->dst_address, clib_net_to_host_u16 (udp0->dst_port),
	     spd0->id);
#endif

	  p0 = ipsec6_output_policy_match (spd0,
					   &ip6_0->src_address,
					   &ip6_0->dst_address,
					   clib_net_to_host_u16
					   (udp0->src_port),
					   clib_net_to_host_u16
					   (udp0->dst_port), ip6_0->protocol);
	}
      else
	{
	  udp0 = (udp_header_t *) ((u8 *) ip0 + ip4_header_bytes (ip0));

#if 0
	  clib_warning ("packet received from %U to %U port %u",
			format_ip4_address, ip0->src_address.as_u8,
			format_ip4_address, ip0->dst_address.as_u8,
			clib_net_to_host_u16 (udp0->dst_port));
	  clib_warning ("sw_if_index0 %u spd_index0 %u spd_id %u",
			sw_if_index0, spd_index0, spd0->id);
#endif

	  /*
	   * Check whether flow cache is enabled.
	   */
	  if (flow_cache_enabled)
	    {
	      p0 = ipsec4_out_spd_find_flow_cache_entry (
		im, ip0->protocol, ip0->src_address.as_u32,
		ip0->dst_address.as_u32, udp0->src_port, udp0->dst_port);
	    }

	  /* Fall back to linear search if flow cache lookup fails */
	  if (p0 == NULL)
	    {
	      p0 = ipsec_output_policy_match (
		spd0, ip0->protocol,
		clib_net_to_host_u32 (ip0->src_address.as_u32),
		clib_net_to_host_u32 (ip0->dst_address.as_u32),
		clib_net_to_host_u16 (udp0->src_port),
		clib_net_to_host_u16 (udp0->dst_port), flow_cache_enabled);
	    }
	}
      tcp0 = (void *) udp0;

      if (PREDICT_TRUE (p0 != NULL))
	{
	  pi0 = p0 - im->policies;

	  vlib_prefetch_combined_counter (&ipsec_spd_policy_counters,
					  thread_index, pi0);

	  if (is_ipv6)
	    {
	      bytes0 = clib_net_to_host_u16 (ip6_0->payload_length);
	      bytes0 += sizeof (ip6_header_t);
	    }
	  else
	    {
	      bytes0 = clib_net_to_host_u16 (ip0->length);
	    }

	  if (p0->policy == IPSEC_POLICY_ACTION_PROTECT)
	    {
	      ipsec_sa_t *sa = 0;
	      nc_protect++;
	      sa = ipsec_sa_get (p0->sa_index);
	      if (sa->protocol == IPSEC_PROTOCOL_ESP)
		if (is_ipv6)
		  next_node_index = im->esp6_encrypt_node_index;
		else
		  next_node_index = im->esp4_encrypt_node_index;
	      else if (is_ipv6)
		next_node_index = im->ah6_encrypt_node_index;
	      else
		next_node_index = im->ah4_encrypt_node_index;
	      vnet_buffer (b0)->ipsec.sad_index = p0->sa_index;

	      if (PREDICT_FALSE (b0->flags & VNET_BUFFER_F_OFFLOAD))
		{
		  vnet_buffer_oflags_t oflags = vnet_buffer (b0)->oflags;

		  /*
		   * Clearing offload flags before checksum is computed
		   * It guarantees the cache hit!
		   */
		  vnet_buffer_offload_flags_clear (b0, oflags);

		  if (is_ipv6)
		    {
		      if (PREDICT_FALSE (oflags &
					 VNET_BUFFER_OFFLOAD_F_TCP_CKSUM))
			{
			  tcp0->checksum = ip6_tcp_udp_icmp_compute_checksum (
			    vm, b0, ip6_0, &bogus);
			}
		      if (PREDICT_FALSE (oflags &
					 VNET_BUFFER_OFFLOAD_F_UDP_CKSUM))
			{
			  udp0->checksum = ip6_tcp_udp_icmp_compute_checksum (
			    vm, b0, ip6_0, &bogus);
			}
		    }
		  else
		    {
		      if (PREDICT_FALSE (oflags &
					 VNET_BUFFER_OFFLOAD_F_IP_CKSUM))
			{
			  ip0->checksum = ip4_header_checksum (ip0);
			}
		      if (PREDICT_FALSE (oflags &
					 VNET_BUFFER_OFFLOAD_F_TCP_CKSUM))
			{
			  tcp0->checksum =
			    ip4_tcp_udp_compute_checksum (vm, b0, ip0);
			}
		      if (PREDICT_FALSE (oflags &
					 VNET_BUFFER_OFFLOAD_F_UDP_CKSUM))
			{
			  udp0->checksum =
			    ip4_tcp_udp_compute_checksum (vm, b0, ip0);
			}
		    }
		}
	      vlib_buffer_advance (b0, iph_offset);
	    }
	  else if (p0->policy == IPSEC_POLICY_ACTION_BYPASS)
	    {
	      nc_bypass++;
	      next_node_index = get_next_output_feature_node_index (b0, node);
	    }
	  else
	    {
	      nc_discard++;
	      next_node_index = im->error_drop_node_index;
	    }
	  vlib_increment_combined_counter
	    (&ipsec_spd_policy_counters, thread_index, pi0, 1, bytes0);
	}
      else
	{
	  pi0 = ~0;
	  nc_nomatch++;
	  next_node_index = im->error_drop_node_index;
	}

      from += 1;
      n_left_from -= 1;

      if (PREDICT_FALSE ((last_next_node_index != next_node_index) || f == 0))
	{
	  /* if this is not 1st frame */
	  if (f)
	    vlib_put_frame_to_node (vm, last_next_node_index, f);

	  last_next_node_index = next_node_index;

	  f = vlib_get_frame_to_node (vm, next_node_index);

	  /* frame->frame_flags, copy it from node */
	  /* Copy trace flag from next_frame and from runtime. */
	  f->frame_flags |= node->flags & VLIB_NODE_FLAG_TRACE;

	  to_next = vlib_frame_vector_args (f);
	}

      to_next[0] = bi0;
      to_next += 1;
      f->n_vectors++;

      if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE) &&
	  PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
	{
	  ipsec_output_trace_t *tr =
	    vlib_add_trace (vm, node, b0, sizeof (*tr));
	  if (spd0)
	    tr->spd_id = spd0->id;
	  tr->policy_id = pi0;
	}
    }

  vlib_put_frame_to_node (vm, next_node_index, f);
  vlib_node_increment_counter (vm, node->node_index,
			       IPSEC_OUTPUT_ERROR_POLICY_PROTECT, nc_protect);
  vlib_node_increment_counter (vm, node->node_index,
			       IPSEC_OUTPUT_ERROR_POLICY_BYPASS, nc_bypass);
  vlib_node_increment_counter (vm, node->node_index,
			       IPSEC_OUTPUT_ERROR_POLICY_DISCARD, nc_discard);
  vlib_node_increment_counter (vm, node->node_index,
			       IPSEC_OUTPUT_ERROR_POLICY_NO_MATCH,
			       nc_nomatch);
  return from_frame->n_vectors;
}

VLIB_NODE_FN (ipsec4_output_node) (vlib_main_t * vm,
				   vlib_node_runtime_t * node,
				   vlib_frame_t * frame)
{
  return ipsec_output_inline (vm, node, frame, 0);
}

/* *INDENT-OFF* */
VLIB_REGISTER_NODE (ipsec4_output_node) = {
  .name = "ipsec4-output-feature",
  .vector_size = sizeof (u32),
  .format_trace = format_ipsec_output_trace,
  .type = VLIB_NODE_TYPE_INTERNAL,

  .n_errors = ARRAY_LEN(ipsec_output_error_strings),
  .error_strings = ipsec_output_error_strings,

  .n_next_nodes = IPSEC_OUTPUT_N_NEXT,
  .next_nodes = {
#define _(s,n) [IPSEC_OUTPUT_NEXT_##s] = n,
    foreach_ipsec_output_next
#undef _
  },
};
/* *INDENT-ON* */

VLIB_NODE_FN (ipsec6_output_node) (vlib_main_t * vm,
				   vlib_node_runtime_t * node,
				   vlib_frame_t * frame)
{
  return ipsec_output_inline (vm, node, frame, 1);
}

VLIB_REGISTER_NODE (ipsec6_output_node) = {
  .name = "ipsec6-output-feature",
  .vector_size = sizeof (u32),
  .format_trace = format_ipsec_output_trace,
  .type = VLIB_NODE_TYPE_INTERNAL,

  .n_errors = ARRAY_LEN(ipsec_output_error_strings),
  .error_strings = ipsec_output_error_strings,

  .n_next_nodes = IPSEC_OUTPUT_N_NEXT,
  .next_nodes = {
#define _(s,n) [IPSEC_OUTPUT_NEXT_##s] = n,
    foreach_ipsec_output_next
#undef _
  },
};