src/vppinfra/vector/ip_csum.h - fdio/vpp - Gitiles

 /* SPDX-License-Identifier: Apache-2.0
  * Copyright(c) 2021 Cisco Systems, Inc.
  */

 #ifndef included_vector_ip_csum_h
 #define included_vector_ip_csum_h
 #include <vppinfra/clib.h>
 typedef struct
 {
   u64 sum;
   u8 odd;
 } clib_ip_csum_t;

 #if defined(CLIB_HAVE_VEC128)
 static_always_inline u64x2
 clib_ip_csum_cvt_and_add_4 (u32x4 v)
 {
   return ((u64x2) u32x4_interleave_lo ((u32x4) v, u32x4_zero ()) +
 	  (u64x2) u32x4_interleave_hi ((u32x4) v, u32x4_zero ()));
 }
 static_always_inline u64
 clib_ip_csum_hadd_2 (u64x2 v)
 {
   return v[0] + v[1];
 }
 #endif

 #if defined(CLIB_HAVE_VEC256)
 static_always_inline u64x4
 clib_ip_csum_cvt_and_add_8 (u32x8 v)
 {
   return ((u64x4) u32x8_interleave_lo ((u32x8) v, u32x8_zero ()) +
 	  (u64x4) u32x8_interleave_hi ((u32x8) v, u32x8_zero ()));
 }
 static_always_inline u64
 clib_ip_csum_hadd_4 (u64x4 v)
 {
   return clib_ip_csum_hadd_2 (u64x4_extract_lo (v) + u64x4_extract_hi (v));
 }
 #endif

 #if defined(CLIB_HAVE_VEC512)
 static_always_inline u64x8
 clib_ip_csum_cvt_and_add_16 (u32x16 v)
 {
   return ((u64x8) u32x16_interleave_lo ((u32x16) v, u32x16_zero ()) +
 	  (u64x8) u32x16_interleave_hi ((u32x16) v, u32x16_zero ()));
 }
 static_always_inline u64
 clib_ip_csum_hadd_8 (u64x8 v)
 {
   return clib_ip_csum_hadd_4 (u64x8_extract_lo (v) + u64x8_extract_hi (v));
 }
 #endif

 static_always_inline void
 clib_ip_csum_inline (clib_ip_csum_t *c, u8 *dst, u8 *src, u16 count,
 		     int is_copy)
 {
   if (c->odd)
     {
       c->odd = 0;
       c->sum += (u16) src[0] << 8;
       count--;
       src++;
       if (is_copy)
 	dst++[0] = src[0];
     }

 #if defined(CLIB_HAVE_VEC512)
   u64x8 sum8 = {};

   while (count >= 512)
     {
       u32x16u *s = (u32x16u *) src;
       sum8 += clib_ip_csum_cvt_and_add_16 (s[0]);
       sum8 += clib_ip_csum_cvt_and_add_16 (s[1]);
       sum8 += clib_ip_csum_cvt_and_add_16 (s[2]);
       sum8 += clib_ip_csum_cvt_and_add_16 (s[3]);
       sum8 += clib_ip_csum_cvt_and_add_16 (s[8]);
       sum8 += clib_ip_csum_cvt_and_add_16 (s[5]);
       sum8 += clib_ip_csum_cvt_and_add_16 (s[6]);
       sum8 += clib_ip_csum_cvt_and_add_16 (s[7]);
       count -= 512;
       src += 512;
       if (is_copy)
 	{
 	  u32x16u *d = (u32x16u *) dst;
 	  d[0] = s[0];
 	  d[1] = s[1];
 	  d[2] = s[2];
 	  d[3] = s[3];
 	  d[4] = s[4];
 	  d[5] = s[5];
 	  d[6] = s[6];
 	  d[7] = s[7];
 	  dst += 512;
 	}
     }

   while (count >= 64)
     {
       u32x16u *s = (u32x16u *) src;
       sum8 += clib_ip_csum_cvt_and_add_16 (s[0]);
       count -= 64;
       src += 64;
       if (is_copy)
 	{
 	  u32x16u *d = (u32x16u *) dst;
 	  d[0] = s[0];
 	  dst += 512;
 	}
     }

 #ifdef CLIB_HAVE_VEC256_MASK_LOAD_STORE
   if (count)
     {
       u64 mask = pow2_mask (count);
       u32x16 v = (u32x16) u8x64_mask_load_zero (src, mask);
       sum8 += clib_ip_csum_cvt_and_add_16 (v);
       c->odd = count & 1;
       if (is_copy)
 	u32x16_mask_store (v, dst, mask);
     }
   c->sum += clib_ip_csum_hadd_8 (sum8);
   return;
 #endif

   c->sum += clib_ip_csum_hadd_8 (sum8);
 #elif defined(CLIB_HAVE_VEC256)
   u64x4 sum4 = {};

   while (count >= 256)
     {
       u32x8u *s = (u32x8u *) src;
       sum4 += clib_ip_csum_cvt_and_add_8 (s[0]);
       sum4 += clib_ip_csum_cvt_and_add_8 (s[1]);
       sum4 += clib_ip_csum_cvt_and_add_8 (s[2]);
       sum4 += clib_ip_csum_cvt_and_add_8 (s[3]);
       sum4 += clib_ip_csum_cvt_and_add_8 (s[4]);
       sum4 += clib_ip_csum_cvt_and_add_8 (s[5]);
       sum4 += clib_ip_csum_cvt_and_add_8 (s[6]);
       sum4 += clib_ip_csum_cvt_and_add_8 (s[7]);
       count -= 256;
       src += 256;
       if (is_copy)
 	{
 	  u32x8u *d = (u32x8u *) dst;
 	  d[0] = s[0];
 	  d[1] = s[1];
 	  d[2] = s[2];
 	  d[3] = s[3];
 	  d[4] = s[4];
 	  d[5] = s[5];
 	  d[6] = s[6];
 	  d[7] = s[7];
 	  dst += 256;
 	}
     }

   while (count >= 32)
     {
       u32x8u *s = (u32x8u *) src;
       sum4 += clib_ip_csum_cvt_and_add_8 (s[0]);
       count -= 32;
       src += 32;
       if (is_copy)
 	{
 	  u32x8u *d = (u32x8u *) dst;
 	  d[0] = s[0];
 	  dst += 32;
 	}
     }

 #ifdef CLIB_HAVE_VEC256_MASK_LOAD_STORE
   if (count)
     {
       u32 mask = pow2_mask (count);
       u32x8 v = (u32x8) u8x32_mask_load_zero (src, mask);
       sum4 += clib_ip_csum_cvt_and_add_8 (v);
       c->odd = count & 1;
       if (is_copy)
 	u32x8_mask_store (v, dst, mask);
     }
   c->sum += clib_ip_csum_hadd_4 (sum4);
   return;
 #endif

   c->sum += clib_ip_csum_hadd_4 (sum4);
 #elif defined(CLIB_HAVE_VEC128)
   u64x2 sum2 = {};

   while (count >= 128)
     {
       u32x4u *s = (u32x4u *) src;
       sum2 += clib_ip_csum_cvt_and_add_4 (s[0]);
       sum2 += clib_ip_csum_cvt_and_add_4 (s[1]);
       sum2 += clib_ip_csum_cvt_and_add_4 (s[2]);
       sum2 += clib_ip_csum_cvt_and_add_4 (s[3]);
       sum2 += clib_ip_csum_cvt_and_add_4 (s[4]);
       sum2 += clib_ip_csum_cvt_and_add_4 (s[5]);
       sum2 += clib_ip_csum_cvt_and_add_4 (s[6]);
       sum2 += clib_ip_csum_cvt_and_add_4 (s[7]);
       count -= 128;
       src += 128;
       if (is_copy)
 	{
 	  u32x4u *d = (u32x4u *) dst;
 	  d[0] = s[0];
 	  d[1] = s[1];
 	  d[2] = s[2];
 	  d[3] = s[3];
 	  d[4] = s[4];
 	  d[5] = s[5];
 	  d[6] = s[6];
 	  d[7] = s[7];
 	  dst += 128;
 	}
     }

   while (count >= 16)
     {
       u32x4u *s = (u32x4u *) src;
       sum2 += clib_ip_csum_cvt_and_add_4 (s[0]);
       count -= 16;
       src += 16;
       if (is_copy)
 	{
 	  u32x4u *d = (u32x4u *) dst;
 	  d[0] = s[0];
 	  dst += 16;
 	}
     }
   c->sum += clib_ip_csum_hadd_2 (sum2);
 #else
   while (count >= 4)
     {
       u32 v = *((u32 *) src);
       c->sum += v;
       count -= 4;
       src += 4;
       if (is_copy)
 	{
 	  *(u32 *) dst = v;
 	  dst += 4;
 	}
     }
 #endif
   while (count >= 2)
     {
       u16 v = *((u16 *) src);
       c->sum += v;
       count -= 2;
       src += 2;
       if (is_copy)
 	{
 	  *(u16 *) dst = v;
 	  dst += 2;
 	}
     }

   if (count)
     {
       c->odd = 1;
       c->sum += (u16) src[0];
       if (is_copy)
 	dst[0] = src[0];
     }
 }

 static_always_inline u16
 clib_ip_csum_fold (clib_ip_csum_t *c)
 {
   u64 sum = c->sum;
 #if defined(__x86_64__) && defined(__BMI2__)
   u64 tmp = sum;
   asm volatile(
     /* using ADC is much faster than mov, shift, add sequence
      * compiler produces */
     "shr	$32, %[sum]			\n\t"
     "add	%k[tmp], %k[sum]		\n\t"
     "mov	$16, %k[tmp]			\n\t"
     "shrx	%k[tmp], %k[sum], %k[tmp]	\n\t"
     "adc	%w[tmp], %w[sum]		\n\t"
     "adc	$0, %w[sum]			\n\t"
     : [ sum ] "+&r"(sum), [ tmp ] "+&r"(tmp));
 #else
   sum = ((u32) sum) + (sum >> 32);
   sum = ((u16) sum) + (sum >> 16);
   sum = ((u16) sum) + (sum >> 16);
 #endif
   return (~((u16) sum));
 }

 static_always_inline void
 clib_ip_csum_chunk (clib_ip_csum_t *c, u8 *src, u16 count)
 {
   return clib_ip_csum_inline (c, 0, src, count, 0);
 }

 static_always_inline void
 clib_ip_csum_and_copy_chunk (clib_ip_csum_t *c, u8 *src, u8 *dst, u16 count)
 {
   return clib_ip_csum_inline (c, dst, src, count, 1);
 }

 static_always_inline u16
 clib_ip_csum (u8 *src, u16 count)
 {
   clib_ip_csum_t c = {};
   if (COMPILE_TIME_CONST (count) && count == 12)
     {
       for (int i = 0; i < 3; i++)
 	c.sum += ((u32 *) src)[i];
     }
   else if (COMPILE_TIME_CONST (count) && count == 20)
     {
       for (int i = 0; i < 5; i++)
 	c.sum += ((u32 *) src)[i];
     }
   else if (COMPILE_TIME_CONST (count) && count == 40)
     {
       for (int i = 0; i < 10; i++)
 	c.sum += ((u32 *) src)[i];
     }
   else
     clib_ip_csum_inline (&c, 0, src, count, 0);
   return clib_ip_csum_fold (&c);
 }

 static_always_inline u16
 clib_ip_csum_and_copy (u8 *dst, u8 *src, u16 count)
 {
   clib_ip_csum_t c = {};
   clib_ip_csum_inline (&c, dst, src, count, 1);
   return clib_ip_csum_fold (&c);
 }

 #endif
	/* SPDX-License-Identifier: Apache-2.0
	* Copyright(c) 2021 Cisco Systems, Inc.
	*/

	#ifndef included_vector_ip_csum_h
	#define included_vector_ip_csum_h
	#include <vppinfra/clib.h>
	typedef struct
	{
	u64 sum;
	u8 odd;
	} clib_ip_csum_t;

	#if defined(CLIB_HAVE_VEC128)
	static_always_inline u64x2
	clib_ip_csum_cvt_and_add_4 (u32x4 v)
	{
	return ((u64x2) u32x4_interleave_lo ((u32x4) v, u32x4_zero ()) +
	(u64x2) u32x4_interleave_hi ((u32x4) v, u32x4_zero ()));
	}
	static_always_inline u64
	clib_ip_csum_hadd_2 (u64x2 v)
	{
	return v[0] + v[1];
	}
	#endif

	#if defined(CLIB_HAVE_VEC256)
	static_always_inline u64x4
	clib_ip_csum_cvt_and_add_8 (u32x8 v)
	{
	return ((u64x4) u32x8_interleave_lo ((u32x8) v, u32x8_zero ()) +
	(u64x4) u32x8_interleave_hi ((u32x8) v, u32x8_zero ()));
	}
	static_always_inline u64
	clib_ip_csum_hadd_4 (u64x4 v)
	{
	return clib_ip_csum_hadd_2 (u64x4_extract_lo (v) + u64x4_extract_hi (v));
	}
	#endif

	#if defined(CLIB_HAVE_VEC512)
	static_always_inline u64x8
	clib_ip_csum_cvt_and_add_16 (u32x16 v)
	{
	return ((u64x8) u32x16_interleave_lo ((u32x16) v, u32x16_zero ()) +
	(u64x8) u32x16_interleave_hi ((u32x16) v, u32x16_zero ()));
	}
	static_always_inline u64
	clib_ip_csum_hadd_8 (u64x8 v)
	{
	return clib_ip_csum_hadd_4 (u64x8_extract_lo (v) + u64x8_extract_hi (v));
	}
	#endif

	static_always_inline void
	clib_ip_csum_inline (clib_ip_csum_t c, u8 dst, u8 *src, u16 count,
	int is_copy)
	{
	if (c->odd)
	{
	c->odd = 0;
	c->sum += (u16) src[0] << 8;
	count--;
	src++;
	if (is_copy)
	dst++[0] = src[0];
	}

	#if defined(CLIB_HAVE_VEC512)
	u64x8 sum8 = {};

	while (count >= 512)
	{
	u32x16u s = (u32x16u ) src;
	sum8 += clib_ip_csum_cvt_and_add_16 (s[0]);
	sum8 += clib_ip_csum_cvt_and_add_16 (s[1]);
	sum8 += clib_ip_csum_cvt_and_add_16 (s[2]);
	sum8 += clib_ip_csum_cvt_and_add_16 (s[3]);
	sum8 += clib_ip_csum_cvt_and_add_16 (s[8]);
	sum8 += clib_ip_csum_cvt_and_add_16 (s[5]);
	sum8 += clib_ip_csum_cvt_and_add_16 (s[6]);
	sum8 += clib_ip_csum_cvt_and_add_16 (s[7]);
	count -= 512;
	src += 512;
	if (is_copy)
	{
	u32x16u d = (u32x16u ) dst;
	d[0] = s[0];
	d[1] = s[1];
	d[2] = s[2];
	d[3] = s[3];
	d[4] = s[4];
	d[5] = s[5];
	d[6] = s[6];
	d[7] = s[7];
	dst += 512;
	}
	}

	while (count >= 64)
	{
	u32x16u s = (u32x16u ) src;
	sum8 += clib_ip_csum_cvt_and_add_16 (s[0]);
	count -= 64;
	src += 64;
	if (is_copy)
	{
	u32x16u d = (u32x16u ) dst;
	d[0] = s[0];
	dst += 512;
	}
	}

	#ifdef CLIB_HAVE_VEC256_MASK_LOAD_STORE
	if (count)
	{
	u64 mask = pow2_mask (count);
	u32x16 v = (u32x16) u8x64_mask_load_zero (src, mask);
	sum8 += clib_ip_csum_cvt_and_add_16 (v);
	c->odd = count & 1;
	if (is_copy)
	u32x16_mask_store (v, dst, mask);
	}
	c->sum += clib_ip_csum_hadd_8 (sum8);
	return;
	#endif

	c->sum += clib_ip_csum_hadd_8 (sum8);
	#elif defined(CLIB_HAVE_VEC256)
	u64x4 sum4 = {};

	while (count >= 256)
	{
	u32x8u s = (u32x8u ) src;
	sum4 += clib_ip_csum_cvt_and_add_8 (s[0]);
	sum4 += clib_ip_csum_cvt_and_add_8 (s[1]);
	sum4 += clib_ip_csum_cvt_and_add_8 (s[2]);
	sum4 += clib_ip_csum_cvt_and_add_8 (s[3]);
	sum4 += clib_ip_csum_cvt_and_add_8 (s[4]);
	sum4 += clib_ip_csum_cvt_and_add_8 (s[5]);
	sum4 += clib_ip_csum_cvt_and_add_8 (s[6]);
	sum4 += clib_ip_csum_cvt_and_add_8 (s[7]);
	count -= 256;
	src += 256;
	if (is_copy)
	{
	u32x8u d = (u32x8u ) dst;
	d[0] = s[0];
	d[1] = s[1];
	d[2] = s[2];
	d[3] = s[3];
	d[4] = s[4];
	d[5] = s[5];
	d[6] = s[6];
	d[7] = s[7];
	dst += 256;
	}
	}

	while (count >= 32)
	{
	u32x8u s = (u32x8u ) src;
	sum4 += clib_ip_csum_cvt_and_add_8 (s[0]);
	count -= 32;
	src += 32;
	if (is_copy)
	{
	u32x8u d = (u32x8u ) dst;
	d[0] = s[0];
	dst += 32;
	}
	}

	#ifdef CLIB_HAVE_VEC256_MASK_LOAD_STORE
	if (count)
	{
	u32 mask = pow2_mask (count);
	u32x8 v = (u32x8) u8x32_mask_load_zero (src, mask);
	sum4 += clib_ip_csum_cvt_and_add_8 (v);
	c->odd = count & 1;
	if (is_copy)
	u32x8_mask_store (v, dst, mask);
	}
	c->sum += clib_ip_csum_hadd_4 (sum4);
	return;
	#endif

	c->sum += clib_ip_csum_hadd_4 (sum4);
	#elif defined(CLIB_HAVE_VEC128)
	u64x2 sum2 = {};

	while (count >= 128)
	{
	u32x4u s = (u32x4u ) src;
	sum2 += clib_ip_csum_cvt_and_add_4 (s[0]);
	sum2 += clib_ip_csum_cvt_and_add_4 (s[1]);
	sum2 += clib_ip_csum_cvt_and_add_4 (s[2]);
	sum2 += clib_ip_csum_cvt_and_add_4 (s[3]);
	sum2 += clib_ip_csum_cvt_and_add_4 (s[4]);
	sum2 += clib_ip_csum_cvt_and_add_4 (s[5]);
	sum2 += clib_ip_csum_cvt_and_add_4 (s[6]);
	sum2 += clib_ip_csum_cvt_and_add_4 (s[7]);
	count -= 128;
	src += 128;
	if (is_copy)
	{
	u32x4u d = (u32x4u ) dst;
	d[0] = s[0];
	d[1] = s[1];
	d[2] = s[2];
	d[3] = s[3];
	d[4] = s[4];
	d[5] = s[5];
	d[6] = s[6];
	d[7] = s[7];
	dst += 128;
	}
	}

	while (count >= 16)
	{
	u32x4u s = (u32x4u ) src;
	sum2 += clib_ip_csum_cvt_and_add_4 (s[0]);
	count -= 16;
	src += 16;
	if (is_copy)
	{
	u32x4u d = (u32x4u ) dst;
	d[0] = s[0];
	dst += 16;
	}
	}
	c->sum += clib_ip_csum_hadd_2 (sum2);
	#else
	while (count >= 4)
	{
	u32 v = ((u32 ) src);
	c->sum += v;
	count -= 4;
	src += 4;
	if (is_copy)
	{
	(u32 ) dst = v;
	dst += 4;
	}
	}
	#endif
	while (count >= 2)
	{
	u16 v = ((u16 ) src);
	c->sum += v;
	count -= 2;
	src += 2;
	if (is_copy)
	{
	(u16 ) dst = v;
	dst += 2;
	}
	}

	if (count)
	{
	c->odd = 1;
	c->sum += (u16) src[0];
	if (is_copy)
	dst[0] = src[0];
	}
	}

	static_always_inline u16
	clib_ip_csum_fold (clib_ip_csum_t *c)
	{
	u64 sum = c->sum;
	#if defined(__x86_64__) && defined(__BMI2__)
	u64 tmp = sum;
	asm volatile(
	/* using ADC is much faster than mov, shift, add sequence
	* compiler produces */
	"shr $32, %[sum] \n\t"
	"add %k[tmp], %k[sum] \n\t"
	"mov $16, %k[tmp] \n\t"
	"shrx %k[tmp], %k[sum], %k[tmp] \n\t"
	"adc %w[tmp], %w[sum] \n\t"
	"adc $0, %w[sum] \n\t"
	: [ sum ] "+&r"(sum), [ tmp ] "+&r"(tmp));
	#else
	sum = ((u32) sum) + (sum >> 32);
	sum = ((u16) sum) + (sum >> 16);
	sum = ((u16) sum) + (sum >> 16);
	#endif
	return (~((u16) sum));
	}

	static_always_inline void
	clib_ip_csum_chunk (clib_ip_csum_t c, u8 src, u16 count)
	{
	return clib_ip_csum_inline (c, 0, src, count, 0);
	}

	static_always_inline void
	clib_ip_csum_and_copy_chunk (clib_ip_csum_t c, u8 src, u8 *dst, u16 count)
	{
	return clib_ip_csum_inline (c, dst, src, count, 1);
	}

	static_always_inline u16
	clib_ip_csum (u8 *src, u16 count)
	{
	clib_ip_csum_t c = {};
	if (COMPILE_TIME_CONST (count) && count == 12)
	{
	for (int i = 0; i < 3; i++)
	c.sum += ((u32 *) src)[i];
	}
	else if (COMPILE_TIME_CONST (count) && count == 20)
	{
	for (int i = 0; i < 5; i++)
	c.sum += ((u32 *) src)[i];
	}
	else if (COMPILE_TIME_CONST (count) && count == 40)
	{
	for (int i = 0; i < 10; i++)
	c.sum += ((u32 *) src)[i];
	}
	else
	clib_ip_csum_inline (&c, 0, src, count, 0);
	return clib_ip_csum_fold (&c);
	}

	static_always_inline u16
	clib_ip_csum_and_copy (u8 dst, u8 src, u16 count)
	{
	clib_ip_csum_t c = {};
	clib_ip_csum_inline (&c, dst, src, count, 1);
	return clib_ip_csum_fold (&c);
	}

	#endif