src/vnet/ipsec/ipsec_output.h - fdio/vpp - Gitiles

 /*
  *------------------------------------------------------------------
  * Copyright (c) 2021 Intel and/or its affiliates.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at:
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  *------------------------------------------------------------------
  */

 #ifndef IPSEC_OUTPUT_H
 #define IPSEC_OUTPUT_H

 #include <vppinfra/types.h>
 #include <vnet/ipsec/ipsec_spd.h>
 #include <vnet/ipsec/ipsec_spd_fp_lookup.h>

 always_inline void
 ipsec4_out_spd_add_flow_cache_entry (ipsec_main_t *im, u8 pr, u32 la, u32 ra,
 				     u16 lp, u16 rp, u32 pol_id)
 {
   u64 hash;
   u8 overwrite = 0, stale_overwrite = 0;
   ipsec4_spd_5tuple_t ip4_5tuple = { .ip4_addr = { (ip4_address_t) la,
 						   (ip4_address_t) ra },
 				     .port = { lp, rp },
 				     .proto = pr };

   ip4_5tuple.kv_16_8.value = (((u64) pol_id) << 32) | ((u64) im->epoch_count);

   hash = ipsec4_hash_16_8 (&ip4_5tuple.kv_16_8);
   hash &= (im->ipsec4_out_spd_hash_num_buckets - 1);

   ipsec_spinlock_lock (&im->ipsec4_out_spd_hash_tbl[hash].bucket_lock);
   /* Check if we are overwriting an existing entry so we know
   whether to increment the flow cache counter. Since flow
   cache counter is reset on any policy add/remove, but
   hash table values are not, we also need to check if the entry
   we are overwriting is stale or not. If it's a stale entry
   overwrite, we still want to increment flow cache counter */
   overwrite = (im->ipsec4_out_spd_hash_tbl[hash].value != 0);
   /* Check for stale entry by comparing with current epoch count */
   if (PREDICT_FALSE (overwrite))
     stale_overwrite =
       (im->epoch_count !=
        ((u32) (im->ipsec4_out_spd_hash_tbl[hash].value & 0xFFFFFFFF)));
   clib_memcpy_fast (&im->ipsec4_out_spd_hash_tbl[hash], &ip4_5tuple.kv_16_8,
 		    sizeof (ip4_5tuple.kv_16_8));
   ipsec_spinlock_unlock (&im->ipsec4_out_spd_hash_tbl[hash].bucket_lock);

   /* Increment the counter to track active flow cache entries
     when entering a fresh entry or overwriting a stale one */
   if (!overwrite || stale_overwrite)
     clib_atomic_fetch_add_relax (&im->ipsec4_out_spd_flow_cache_entries, 1);

   return;
 }

 always_inline void
 ipsec4_out_spd_add_flow_cache_entry_n (ipsec_main_t *im,
 				       ipsec4_spd_5tuple_t *ip4_5tuple,
 				       u32 pol_id)
 {
   u64 hash;
   u8 overwrite = 0, stale_overwrite = 0;

   ip4_5tuple->kv_16_8.value = (((u64) pol_id) << 32) | ((u64) im->epoch_count);

   hash = ipsec4_hash_16_8 (&ip4_5tuple->kv_16_8);
   hash &= (im->ipsec4_out_spd_hash_num_buckets - 1);

   ipsec_spinlock_lock (&im->ipsec4_out_spd_hash_tbl[hash].bucket_lock);
   /* Check if we are overwriting an existing entry so we know
   whether to increment the flow cache counter. Since flow
   cache counter is reset on any policy add/remove, but
   hash table values are not, we also need to check if the entry
   we are overwriting is stale or not. If it's a stale entry
   overwrite, we still want to increment flow cache counter */
   overwrite = (im->ipsec4_out_spd_hash_tbl[hash].value != 0);
   /* Check for stale entry by comparing with current epoch count */
   if (PREDICT_FALSE (overwrite))
     stale_overwrite =
       (im->epoch_count !=
        ((u32) (im->ipsec4_out_spd_hash_tbl[hash].value & 0xFFFFFFFF)));
   clib_memcpy_fast (&im->ipsec4_out_spd_hash_tbl[hash], &ip4_5tuple->kv_16_8,
 		    sizeof (ip4_5tuple->kv_16_8));
   ipsec_spinlock_unlock (&im->ipsec4_out_spd_hash_tbl[hash].bucket_lock);

   /* Increment the counter to track active flow cache entries
     when entering a fresh entry or overwriting a stale one */
   if (!overwrite || stale_overwrite)
     clib_atomic_fetch_add_relax (&im->ipsec4_out_spd_flow_cache_entries, 1);

   return;
 }

 always_inline void
 ipsec_fp_5tuple_from_ip4_range (ipsec_fp_5tuple_t *tuple, u32 la, u32 ra,
 				u16 lp, u16 rp, u8 pr)
 {
   clib_memset (tuple->l3_zero_pad, 0, sizeof (tuple->l3_zero_pad));
   tuple->laddr.as_u32 = clib_host_to_net_u32 (la);
   tuple->raddr.as_u32 = clib_host_to_net_u32 (ra);

   if (PREDICT_FALSE ((pr != IP_PROTOCOL_TCP) && (pr != IP_PROTOCOL_UDP) &&
 		     (pr != IP_PROTOCOL_SCTP)))
     {
       tuple->lport = 0;
       tuple->rport = 0;
     }
   else
     {
       tuple->lport = lp;
       tuple->rport = rp;
     }

   tuple->protocol = pr;
   tuple->is_ipv6 = 0;
 }

 always_inline void
 ipsec_fp_5tuple_from_ip4_range_n (ipsec_fp_5tuple_t *tuples,
 				  ipsec4_spd_5tuple_t *ip4_5tuple, u32 n)
 {
   u32 n_left = n;
   ipsec_fp_5tuple_t *tuple = tuples;

   while (n_left)
     {
       clib_memset (tuple->l3_zero_pad, 0, sizeof (tuple->l3_zero_pad));
       tuple->laddr.as_u32 =
 	clib_host_to_net_u32 (ip4_5tuple->ip4_addr[0].as_u32);
       tuple->raddr.as_u32 =
 	clib_host_to_net_u32 (ip4_5tuple->ip4_addr[1].as_u32);
       if (PREDICT_FALSE ((ip4_5tuple->proto != IP_PROTOCOL_TCP) &&
 			 (ip4_5tuple->proto != IP_PROTOCOL_UDP) &&
 			 (ip4_5tuple->proto != IP_PROTOCOL_SCTP)))
 	{
 	  tuple->lport = 0;
 	  tuple->rport = 0;
 	}
       else
 	{
 	  tuple->lport = ip4_5tuple->port[0];
 	  tuple->rport = ip4_5tuple->port[1];
 	}
       tuple->protocol = ip4_5tuple->proto;
       tuple->is_ipv6 = 0;
       n_left--;
       tuple++;
     }
 }

 always_inline int
 ipsec_output_policy_match_n (ipsec_spd_t *spd,
 			     ipsec4_spd_5tuple_t *ip4_5tuples,
 			     ipsec_policy_t **policies, u32 n,
 			     u8 flow_cache_enabled)
 {
   ipsec_main_t *im = &ipsec_main;
   ipsec_policy_t *p;
   ipsec_policy_t **pp = policies;
   u32 n_left = n;
   ipsec4_spd_5tuple_t *ip4_5tuple = ip4_5tuples;
   u32 policy_ids[n], *policy_id = policy_ids;
   ipsec_fp_5tuple_t tuples[n];
   u32 *i;
   u32 counter = 0;

   if (!spd)
     return 0;

   clib_memset (policies, 0, n * sizeof (ipsec_policy_t *));

   if (im->fp_spd_ipv4_out_is_enabled &&
       PREDICT_TRUE (INDEX_INVALID != spd->fp_spd.ip4_out_lookup_hash_idx))
     {
       ipsec_fp_5tuple_from_ip4_range_n (tuples, ip4_5tuples, n);
       counter += ipsec_fp_out_policy_match_n (&spd->fp_spd, 0, tuples,
 					      policies, policy_ids, n);
     }

   while (n_left)
     {
       if (*pp != 0)
 	goto next;

       vec_foreach (i, spd->policies[IPSEC_SPD_POLICY_IP4_OUTBOUND])
 	{
 	  p = pool_elt_at_index (im->policies, *i);
 	  if (PREDICT_FALSE (p->protocol &&
 			     (p->protocol != ip4_5tuple->proto)))
 	    continue;

 	  if (ip4_5tuple->ip4_addr[0].as_u32 <
 	      clib_net_to_host_u32 (p->raddr.start.ip4.as_u32))
 	    continue;

 	  if (ip4_5tuple->ip4_addr[1].as_u32 >
 	      clib_net_to_host_u32 (p->raddr.stop.ip4.as_u32))
 	    continue;

 	  if (ip4_5tuple->ip4_addr[0].as_u32 <
 	      clib_net_to_host_u32 (p->laddr.start.ip4.as_u32))
 	    continue;

 	  if (ip4_5tuple->ip4_addr[1].as_u32 >
 	      clib_net_to_host_u32 (p->laddr.stop.ip4.as_u32))
 	    continue;

 	  if (PREDICT_FALSE ((ip4_5tuple->proto != IP_PROTOCOL_TCP) &&
 			     (ip4_5tuple->proto != IP_PROTOCOL_UDP) &&
 			     (ip4_5tuple->proto != IP_PROTOCOL_SCTP)))
 	    {
 	      ip4_5tuple->port[0] = 0;
 	      ip4_5tuple->port[1] = 0;
 	      goto add_policy;
 	    }

 	  if (ip4_5tuple->port[0] < p->lport.start)
 	    continue;

 	  if (ip4_5tuple->port[0] > p->lport.stop)
 	    continue;

 	  if (ip4_5tuple->port[1] < p->rport.start)
 	    continue;

 	  if (ip4_5tuple->port[1] > p->rport.stop)
 	    continue;

 	add_policy:
 	  *pp = p;
 	  *policy_id = *i;
 	  counter++;
 	  break;
 	}

     next:
       n_left--;
       pp++;
       ip4_5tuple++;
       policy_id++;
     }

   if (flow_cache_enabled)
     {
       n_left = n;
       policy_id = policy_ids;
       ip4_5tuple = ip4_5tuples;
       pp = policies;

       while (n_left)
 	{
 	  if (*pp != NULL)
 	    {
 	      /* Add an Entry in Flow cache */
 	      ipsec4_out_spd_add_flow_cache_entry_n (im, ip4_5tuple,
 						     *policy_id);
 	    }

 	  n_left--;
 	  policy_id++;
 	  ip4_5tuple++;
 	  pp++;
 	}
     }

   return counter;
 }

 always_inline ipsec_policy_t *
 ipsec4_out_spd_find_flow_cache_entry (ipsec_main_t *im, u8 pr, u32 la, u32 ra,
 				      u16 lp, u16 rp)
 {
   ipsec_policy_t *p = NULL;
   ipsec4_hash_kv_16_8_t kv_result;
   u64 hash;

   if (PREDICT_FALSE ((pr != IP_PROTOCOL_TCP) && (pr != IP_PROTOCOL_UDP) &&
 		     (pr != IP_PROTOCOL_SCTP)))
     {
       lp = 0;
       rp = 0;
     }
   ipsec4_spd_5tuple_t ip4_5tuple = { .ip4_addr = { (ip4_address_t) la,
 						   (ip4_address_t) ra },
 				     .port = { lp, rp },
 				     .proto = pr };

   hash = ipsec4_hash_16_8 (&ip4_5tuple.kv_16_8);
   hash &= (im->ipsec4_out_spd_hash_num_buckets - 1);

   ipsec_spinlock_lock (&im->ipsec4_out_spd_hash_tbl[hash].bucket_lock);
   kv_result = im->ipsec4_out_spd_hash_tbl[hash];
   ipsec_spinlock_unlock (&im->ipsec4_out_spd_hash_tbl[hash].bucket_lock);

   if (ipsec4_hash_key_compare_16_8 ((u64 *) &ip4_5tuple.kv_16_8,
 				    (u64 *) &kv_result))
     {
       if (im->epoch_count == ((u32) (kv_result.value & 0xFFFFFFFF)))
 	{
 	  /* Get the policy based on the index */
 	  p =
 	    pool_elt_at_index (im->policies, ((u32) (kv_result.value >> 32)));
 	}
     }

   return p;
 }

 always_inline ipsec_policy_t *
 ipsec_output_policy_match (ipsec_spd_t *spd, u8 pr, u32 la, u32 ra, u16 lp,
 			   u16 rp, u8 flow_cache_enabled)
 {
   ipsec_main_t *im = &ipsec_main;
   ipsec_policy_t *p;
   ipsec_policy_t *policies[1];
   ipsec_fp_5tuple_t tuples[1];
   u32 fp_policy_ids[1];

   u32 *i;

   if (!spd)
     return 0;

   if (im->fp_spd_ipv4_out_is_enabled &&
       PREDICT_TRUE (INDEX_INVALID != spd->fp_spd.ip4_out_lookup_hash_idx))
     {
       ipsec_fp_5tuple_from_ip4_range (&tuples[0], la, ra, lp, rp, pr);
       ipsec_fp_out_policy_match_n (&spd->fp_spd, 0, tuples, policies,
 				   fp_policy_ids, 1);
       p = policies[0];
       i = fp_policy_ids;
       if (PREDICT_FALSE ((pr != IP_PROTOCOL_TCP) && (pr != IP_PROTOCOL_UDP) &&
 			 (pr != IP_PROTOCOL_SCTP)))
 	{
 	  lp = 0;
 	  rp = 0;
 	}
       goto add_flow_cache;
     }

   vec_foreach (i, spd->policies[IPSEC_SPD_POLICY_IP4_OUTBOUND])
     {
       p = pool_elt_at_index (im->policies, *i);
       if (PREDICT_FALSE ((p->protocol != IPSEC_POLICY_PROTOCOL_ANY) &&
 			 (p->protocol != pr)))
 	continue;

       if (ra < clib_net_to_host_u32 (p->raddr.start.ip4.as_u32))
 	continue;

       if (ra > clib_net_to_host_u32 (p->raddr.stop.ip4.as_u32))
 	continue;

       if (la < clib_net_to_host_u32 (p->laddr.start.ip4.as_u32))
 	continue;

       if (la > clib_net_to_host_u32 (p->laddr.stop.ip4.as_u32))
 	continue;

       if (PREDICT_FALSE ((pr != IP_PROTOCOL_TCP) && (pr != IP_PROTOCOL_UDP) &&
 			 (pr != IP_PROTOCOL_SCTP)))
 	{
 	  lp = 0;
 	  rp = 0;
 	  goto add_flow_cache;
 	}

       if (lp < p->lport.start)
 	continue;

       if (lp > p->lport.stop)
 	continue;

       if (rp < p->rport.start)
 	continue;

       if (rp > p->rport.stop)
 	continue;

     add_flow_cache:
       if (flow_cache_enabled)
 	{
 	  /* Add an Entry in Flow cache */
 	  ipsec4_out_spd_add_flow_cache_entry (
 	    im, pr, clib_host_to_net_u32 (la), clib_host_to_net_u32 (ra),
 	    clib_host_to_net_u16 (lp), clib_host_to_net_u16 (rp), *i);
 	}

       return p;
     }
   return 0;
 }

 always_inline uword
 ip6_addr_match_range (ip6_address_t *a, ip6_address_t *la, ip6_address_t *ua)
 {
   if ((memcmp (a->as_u64, la->as_u64, 2 * sizeof (u64)) >= 0) &&
       (memcmp (a->as_u64, ua->as_u64, 2 * sizeof (u64)) <= 0))
     return 1;
   return 0;
 }

 always_inline void
 ipsec_fp_5tuple_from_ip6_range (ipsec_fp_5tuple_t *tuple, ip6_address_t *la,
 				ip6_address_t *ra, u16 lp, u16 rp, u8 pr)

 {
   clib_memcpy (&tuple->ip6_laddr, la, sizeof (ip6_address_t));
   clib_memcpy (&tuple->ip6_raddr, ra, sizeof (ip6_address_t));

   tuple->lport = lp;
   tuple->rport = rp;
   tuple->protocol = pr;
   tuple->is_ipv6 = 1;
 }

 always_inline ipsec_policy_t *
 ipsec6_output_policy_match (ipsec_spd_t *spd, ip6_address_t *la,
 			    ip6_address_t *ra, u16 lp, u16 rp, u8 pr)
 {
   ipsec_main_t *im = &ipsec_main;
   ipsec_policy_t *p;
   ipsec_policy_t *policies[1];
   ipsec_fp_5tuple_t tuples[1];
   u32 fp_policy_ids[1];

   u32 *i;

   if (!spd)
     return 0;

   if (im->fp_spd_ipv6_out_is_enabled &&
       PREDICT_TRUE (INDEX_INVALID != spd->fp_spd.ip6_out_lookup_hash_idx))
     {

       ipsec_fp_5tuple_from_ip6_range (&tuples[0], la, ra, lp, rp, pr);
       ipsec_fp_out_policy_match_n (&spd->fp_spd, 1, tuples, policies,
 				   fp_policy_ids, 1);
       p = policies[0];
       i = fp_policy_ids;
       return p;
     }

   vec_foreach (i, spd->policies[IPSEC_SPD_POLICY_IP6_OUTBOUND])
     {
       p = pool_elt_at_index (im->policies, *i);
       if (PREDICT_FALSE ((p->protocol != IPSEC_POLICY_PROTOCOL_ANY) &&
 			 (p->protocol != pr)))
 	continue;

       if (!ip6_addr_match_range (ra, &p->raddr.start.ip6, &p->raddr.stop.ip6))
 	continue;

       if (!ip6_addr_match_range (la, &p->laddr.start.ip6, &p->laddr.stop.ip6))
 	continue;

       if (PREDICT_FALSE ((pr != IP_PROTOCOL_TCP) && (pr != IP_PROTOCOL_UDP) &&
 			 (pr != IP_PROTOCOL_SCTP)))
 	return p;

       if (lp < p->lport.start)
 	continue;

       if (lp > p->lport.stop)
 	continue;

       if (rp < p->rport.start)
 	continue;

       if (rp > p->rport.stop)
 	continue;

       return p;
     }

   return 0;
 }

 #endif /* !IPSEC_OUTPUT_H */
	/*
	*------------------------------------------------------------------
	* Copyright (c) 2021 Intel and/or its affiliates.
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at:
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*------------------------------------------------------------------
	*/

	#ifndef IPSEC_OUTPUT_H
	#define IPSEC_OUTPUT_H

	#include <vppinfra/types.h>
	#include <vnet/ipsec/ipsec_spd.h>
	#include <vnet/ipsec/ipsec_spd_fp_lookup.h>

	always_inline void
	ipsec4_out_spd_add_flow_cache_entry (ipsec_main_t *im, u8 pr, u32 la, u32 ra,
	u16 lp, u16 rp, u32 pol_id)
	{
	u64 hash;
	u8 overwrite = 0, stale_overwrite = 0;
	ipsec4_spd_5tuple_t ip4_5tuple = { .ip4_addr = { (ip4_address_t) la,
	(ip4_address_t) ra },
	.port = { lp, rp },
	.proto = pr };

	ip4_5tuple.kv_16_8.value = (((u64) pol_id) << 32) \| ((u64) im->epoch_count);

	hash = ipsec4_hash_16_8 (&ip4_5tuple.kv_16_8);
	hash &= (im->ipsec4_out_spd_hash_num_buckets - 1);

	ipsec_spinlock_lock (&im->ipsec4_out_spd_hash_tbl[hash].bucket_lock);
	/* Check if we are overwriting an existing entry so we know
	whether to increment the flow cache counter. Since flow
	cache counter is reset on any policy add/remove, but
	hash table values are not, we also need to check if the entry
	we are overwriting is stale or not. If it's a stale entry
	overwrite, we still want to increment flow cache counter */
	overwrite = (im->ipsec4_out_spd_hash_tbl[hash].value != 0);
	/* Check for stale entry by comparing with current epoch count */
	if (PREDICT_FALSE (overwrite))
	stale_overwrite =
	(im->epoch_count !=
	((u32) (im->ipsec4_out_spd_hash_tbl[hash].value & 0xFFFFFFFF)));
	clib_memcpy_fast (&im->ipsec4_out_spd_hash_tbl[hash], &ip4_5tuple.kv_16_8,
	sizeof (ip4_5tuple.kv_16_8));
	ipsec_spinlock_unlock (&im->ipsec4_out_spd_hash_tbl[hash].bucket_lock);

	/* Increment the counter to track active flow cache entries
	when entering a fresh entry or overwriting a stale one */
	if (!overwrite \|\| stale_overwrite)
	clib_atomic_fetch_add_relax (&im->ipsec4_out_spd_flow_cache_entries, 1);

	return;
	}

	always_inline void
	ipsec4_out_spd_add_flow_cache_entry_n (ipsec_main_t *im,
	ipsec4_spd_5tuple_t *ip4_5tuple,
	u32 pol_id)
	{
	u64 hash;
	u8 overwrite = 0, stale_overwrite = 0;

	ip4_5tuple->kv_16_8.value = (((u64) pol_id) << 32) \| ((u64) im->epoch_count);

	hash = ipsec4_hash_16_8 (&ip4_5tuple->kv_16_8);
	hash &= (im->ipsec4_out_spd_hash_num_buckets - 1);

	ipsec_spinlock_lock (&im->ipsec4_out_spd_hash_tbl[hash].bucket_lock);
	/* Check if we are overwriting an existing entry so we know
	whether to increment the flow cache counter. Since flow
	cache counter is reset on any policy add/remove, but
	hash table values are not, we also need to check if the entry
	we are overwriting is stale or not. If it's a stale entry
	overwrite, we still want to increment flow cache counter */
	overwrite = (im->ipsec4_out_spd_hash_tbl[hash].value != 0);
	/* Check for stale entry by comparing with current epoch count */
	if (PREDICT_FALSE (overwrite))
	stale_overwrite =
	(im->epoch_count !=
	((u32) (im->ipsec4_out_spd_hash_tbl[hash].value & 0xFFFFFFFF)));
	clib_memcpy_fast (&im->ipsec4_out_spd_hash_tbl[hash], &ip4_5tuple->kv_16_8,
	sizeof (ip4_5tuple->kv_16_8));
	ipsec_spinlock_unlock (&im->ipsec4_out_spd_hash_tbl[hash].bucket_lock);

	/* Increment the counter to track active flow cache entries
	when entering a fresh entry or overwriting a stale one */
	if (!overwrite \|\| stale_overwrite)
	clib_atomic_fetch_add_relax (&im->ipsec4_out_spd_flow_cache_entries, 1);

	return;
	}

	always_inline void
	ipsec_fp_5tuple_from_ip4_range (ipsec_fp_5tuple_t *tuple, u32 la, u32 ra,
	u16 lp, u16 rp, u8 pr)
	{
	clib_memset (tuple->l3_zero_pad, 0, sizeof (tuple->l3_zero_pad));
	tuple->laddr.as_u32 = clib_host_to_net_u32 (la);
	tuple->raddr.as_u32 = clib_host_to_net_u32 (ra);

	if (PREDICT_FALSE ((pr != IP_PROTOCOL_TCP) && (pr != IP_PROTOCOL_UDP) &&
	(pr != IP_PROTOCOL_SCTP)))
	{
	tuple->lport = 0;
	tuple->rport = 0;
	}
	else
	{
	tuple->lport = lp;
	tuple->rport = rp;
	}

	tuple->protocol = pr;
	tuple->is_ipv6 = 0;
	}

	always_inline void
	ipsec_fp_5tuple_from_ip4_range_n (ipsec_fp_5tuple_t *tuples,
	ipsec4_spd_5tuple_t *ip4_5tuple, u32 n)
	{
	u32 n_left = n;
	ipsec_fp_5tuple_t *tuple = tuples;

	while (n_left)
	{
	clib_memset (tuple->l3_zero_pad, 0, sizeof (tuple->l3_zero_pad));
	tuple->laddr.as_u32 =
	clib_host_to_net_u32 (ip4_5tuple->ip4_addr[0].as_u32);
	tuple->raddr.as_u32 =
	clib_host_to_net_u32 (ip4_5tuple->ip4_addr[1].as_u32);
	if (PREDICT_FALSE ((ip4_5tuple->proto != IP_PROTOCOL_TCP) &&
	(ip4_5tuple->proto != IP_PROTOCOL_UDP) &&
	(ip4_5tuple->proto != IP_PROTOCOL_SCTP)))
	{
	tuple->lport = 0;
	tuple->rport = 0;
	}
	else
	{
	tuple->lport = ip4_5tuple->port[0];
	tuple->rport = ip4_5tuple->port[1];
	}
	tuple->protocol = ip4_5tuple->proto;
	tuple->is_ipv6 = 0;
	n_left--;
	tuple++;
	}
	}

	always_inline int
	ipsec_output_policy_match_n (ipsec_spd_t *spd,
	ipsec4_spd_5tuple_t *ip4_5tuples,
	ipsec_policy_t **policies, u32 n,
	u8 flow_cache_enabled)
	{
	ipsec_main_t *im = &ipsec_main;
	ipsec_policy_t *p;
	ipsec_policy_t **pp = policies;
	u32 n_left = n;
	ipsec4_spd_5tuple_t *ip4_5tuple = ip4_5tuples;
	u32 policy_ids[n], *policy_id = policy_ids;
	ipsec_fp_5tuple_t tuples[n];
	u32 *i;
	u32 counter = 0;

	if (!spd)
	return 0;

	clib_memset (policies, 0, n * sizeof (ipsec_policy_t *));

	if (im->fp_spd_ipv4_out_is_enabled &&
	PREDICT_TRUE (INDEX_INVALID != spd->fp_spd.ip4_out_lookup_hash_idx))
	{
	ipsec_fp_5tuple_from_ip4_range_n (tuples, ip4_5tuples, n);
	counter += ipsec_fp_out_policy_match_n (&spd->fp_spd, 0, tuples,
	policies, policy_ids, n);
	}

	while (n_left)
	{
	if (*pp != 0)
	goto next;

	vec_foreach (i, spd->policies[IPSEC_SPD_POLICY_IP4_OUTBOUND])
	{
	p = pool_elt_at_index (im->policies, *i);
	if (PREDICT_FALSE (p->protocol &&
	(p->protocol != ip4_5tuple->proto)))
	continue;

	if (ip4_5tuple->ip4_addr[0].as_u32 <
	clib_net_to_host_u32 (p->raddr.start.ip4.as_u32))
	continue;

	if (ip4_5tuple->ip4_addr[1].as_u32 >
	clib_net_to_host_u32 (p->raddr.stop.ip4.as_u32))
	continue;

	if (ip4_5tuple->ip4_addr[0].as_u32 <
	clib_net_to_host_u32 (p->laddr.start.ip4.as_u32))
	continue;

	if (ip4_5tuple->ip4_addr[1].as_u32 >
	clib_net_to_host_u32 (p->laddr.stop.ip4.as_u32))
	continue;

	if (PREDICT_FALSE ((ip4_5tuple->proto != IP_PROTOCOL_TCP) &&
	(ip4_5tuple->proto != IP_PROTOCOL_UDP) &&
	(ip4_5tuple->proto != IP_PROTOCOL_SCTP)))
	{
	ip4_5tuple->port[0] = 0;
	ip4_5tuple->port[1] = 0;
	goto add_policy;
	}

	if (ip4_5tuple->port[0] < p->lport.start)
	continue;

	if (ip4_5tuple->port[0] > p->lport.stop)
	continue;

	if (ip4_5tuple->port[1] < p->rport.start)
	continue;

	if (ip4_5tuple->port[1] > p->rport.stop)
	continue;

	add_policy:
	*pp = p;
	policy_id = i;
	counter++;
	break;
	}

	next:
	n_left--;
	pp++;
	ip4_5tuple++;
	policy_id++;
	}

	if (flow_cache_enabled)
	{
	n_left = n;
	policy_id = policy_ids;
	ip4_5tuple = ip4_5tuples;
	pp = policies;

	while (n_left)
	{
	if (*pp != NULL)
	{
	/* Add an Entry in Flow cache */
	ipsec4_out_spd_add_flow_cache_entry_n (im, ip4_5tuple,
	*policy_id);
	}

	n_left--;
	policy_id++;
	ip4_5tuple++;
	pp++;
	}
	}

	return counter;
	}

	always_inline ipsec_policy_t *
	ipsec4_out_spd_find_flow_cache_entry (ipsec_main_t *im, u8 pr, u32 la, u32 ra,
	u16 lp, u16 rp)
	{
	ipsec_policy_t *p = NULL;
	ipsec4_hash_kv_16_8_t kv_result;
	u64 hash;

	if (PREDICT_FALSE ((pr != IP_PROTOCOL_TCP) && (pr != IP_PROTOCOL_UDP) &&
	(pr != IP_PROTOCOL_SCTP)))
	{
	lp = 0;
	rp = 0;
	}
	ipsec4_spd_5tuple_t ip4_5tuple = { .ip4_addr = { (ip4_address_t) la,
	(ip4_address_t) ra },
	.port = { lp, rp },
	.proto = pr };

	hash = ipsec4_hash_16_8 (&ip4_5tuple.kv_16_8);
	hash &= (im->ipsec4_out_spd_hash_num_buckets - 1);

	ipsec_spinlock_lock (&im->ipsec4_out_spd_hash_tbl[hash].bucket_lock);
	kv_result = im->ipsec4_out_spd_hash_tbl[hash];
	ipsec_spinlock_unlock (&im->ipsec4_out_spd_hash_tbl[hash].bucket_lock);

	if (ipsec4_hash_key_compare_16_8 ((u64 *) &ip4_5tuple.kv_16_8,
	(u64 *) &kv_result))
	{
	if (im->epoch_count == ((u32) (kv_result.value & 0xFFFFFFFF)))
	{
	/* Get the policy based on the index */
	p =
	pool_elt_at_index (im->policies, ((u32) (kv_result.value >> 32)));
	}
	}

	return p;
	}

	always_inline ipsec_policy_t *
	ipsec_output_policy_match (ipsec_spd_t *spd, u8 pr, u32 la, u32 ra, u16 lp,
	u16 rp, u8 flow_cache_enabled)
	{
	ipsec_main_t *im = &ipsec_main;
	ipsec_policy_t *p;
	ipsec_policy_t *policies[1];
	ipsec_fp_5tuple_t tuples[1];
	u32 fp_policy_ids[1];

	u32 *i;

	if (!spd)
	return 0;

	if (im->fp_spd_ipv4_out_is_enabled &&
	PREDICT_TRUE (INDEX_INVALID != spd->fp_spd.ip4_out_lookup_hash_idx))
	{
	ipsec_fp_5tuple_from_ip4_range (&tuples[0], la, ra, lp, rp, pr);
	ipsec_fp_out_policy_match_n (&spd->fp_spd, 0, tuples, policies,
	fp_policy_ids, 1);
	p = policies[0];
	i = fp_policy_ids;
	if (PREDICT_FALSE ((pr != IP_PROTOCOL_TCP) && (pr != IP_PROTOCOL_UDP) &&
	(pr != IP_PROTOCOL_SCTP)))
	{
	lp = 0;
	rp = 0;
	}
	goto add_flow_cache;
	}

	vec_foreach (i, spd->policies[IPSEC_SPD_POLICY_IP4_OUTBOUND])
	{
	p = pool_elt_at_index (im->policies, *i);
	if (PREDICT_FALSE ((p->protocol != IPSEC_POLICY_PROTOCOL_ANY) &&
	(p->protocol != pr)))
	continue;

	if (ra < clib_net_to_host_u32 (p->raddr.start.ip4.as_u32))
	continue;

	if (ra > clib_net_to_host_u32 (p->raddr.stop.ip4.as_u32))
	continue;

	if (la < clib_net_to_host_u32 (p->laddr.start.ip4.as_u32))
	continue;

	if (la > clib_net_to_host_u32 (p->laddr.stop.ip4.as_u32))
	continue;

	if (PREDICT_FALSE ((pr != IP_PROTOCOL_TCP) && (pr != IP_PROTOCOL_UDP) &&
	(pr != IP_PROTOCOL_SCTP)))
	{
	lp = 0;
	rp = 0;
	goto add_flow_cache;
	}

	if (lp < p->lport.start)
	continue;

	if (lp > p->lport.stop)
	continue;

	if (rp < p->rport.start)
	continue;

	if (rp > p->rport.stop)
	continue;

	add_flow_cache:
	if (flow_cache_enabled)
	{
	/* Add an Entry in Flow cache */
	ipsec4_out_spd_add_flow_cache_entry (
	im, pr, clib_host_to_net_u32 (la), clib_host_to_net_u32 (ra),
	clib_host_to_net_u16 (lp), clib_host_to_net_u16 (rp), *i);
	}

	return p;
	}
	return 0;
	}

	always_inline uword
	ip6_addr_match_range (ip6_address_t a, ip6_address_t la, ip6_address_t *ua)
	{
	if ((memcmp (a->as_u64, la->as_u64, 2 * sizeof (u64)) >= 0) &&
	(memcmp (a->as_u64, ua->as_u64, 2 * sizeof (u64)) <= 0))
	return 1;
	return 0;
	}

	always_inline void
	ipsec_fp_5tuple_from_ip6_range (ipsec_fp_5tuple_t tuple, ip6_address_t la,
	ip6_address_t *ra, u16 lp, u16 rp, u8 pr)

	{
	clib_memcpy (&tuple->ip6_laddr, la, sizeof (ip6_address_t));
	clib_memcpy (&tuple->ip6_raddr, ra, sizeof (ip6_address_t));

	tuple->lport = lp;
	tuple->rport = rp;
	tuple->protocol = pr;
	tuple->is_ipv6 = 1;
	}

	always_inline ipsec_policy_t *
	ipsec6_output_policy_match (ipsec_spd_t spd, ip6_address_t la,
	ip6_address_t *ra, u16 lp, u16 rp, u8 pr)
	{
	ipsec_main_t *im = &ipsec_main;
	ipsec_policy_t *p;
	ipsec_policy_t *policies[1];
	ipsec_fp_5tuple_t tuples[1];
	u32 fp_policy_ids[1];

	u32 *i;

	if (!spd)
	return 0;

	if (im->fp_spd_ipv6_out_is_enabled &&
	PREDICT_TRUE (INDEX_INVALID != spd->fp_spd.ip6_out_lookup_hash_idx))
	{

	ipsec_fp_5tuple_from_ip6_range (&tuples[0], la, ra, lp, rp, pr);
	ipsec_fp_out_policy_match_n (&spd->fp_spd, 1, tuples, policies,
	fp_policy_ids, 1);
	p = policies[0];
	i = fp_policy_ids;
	return p;
	}

	vec_foreach (i, spd->policies[IPSEC_SPD_POLICY_IP6_OUTBOUND])
	{
	p = pool_elt_at_index (im->policies, *i);
	if (PREDICT_FALSE ((p->protocol != IPSEC_POLICY_PROTOCOL_ANY) &&
	(p->protocol != pr)))
	continue;

	if (!ip6_addr_match_range (ra, &p->raddr.start.ip6, &p->raddr.stop.ip6))
	continue;

	if (!ip6_addr_match_range (la, &p->laddr.start.ip6, &p->laddr.stop.ip6))
	continue;

	if (PREDICT_FALSE ((pr != IP_PROTOCOL_TCP) && (pr != IP_PROTOCOL_UDP) &&
	(pr != IP_PROTOCOL_SCTP)))
	return p;

	if (lp < p->lport.start)
	continue;

	if (lp > p->lport.stop)
	continue;

	if (rp < p->rport.start)
	continue;

	if (rp > p->rport.stop)
	continue;

	return p;
	}

	return 0;
	}

	#endif /* !IPSEC_OUTPUT_H */