Damjan Marion | c576622 | 2018-04-16 00:18:34 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2018 Cisco and/or its affiliates. |
| 3 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | * you may not use this file except in compliance with the License. |
| 5 | * You may obtain a copy of the License at: |
| 6 | * |
| 7 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | * |
| 9 | * Unless required by applicable law or agreed to in writing, software |
| 10 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | * See the License for the specific language governing permissions and |
| 13 | * limitations under the License. |
| 14 | */ |
| 15 | |
| 16 | #ifndef included_vector_avx2_h |
| 17 | #define included_vector_avx2_h |
| 18 | |
| 19 | #include <vppinfra/clib.h> |
| 20 | #include <x86intrin.h> |
| 21 | |
Damjan Marion | a52e166 | 2018-05-19 00:04:23 +0200 | [diff] [blame] | 22 | /* *INDENT-OFF* */ |
Damjan Marion | c576622 | 2018-04-16 00:18:34 +0200 | [diff] [blame] | 23 | #define foreach_avx2_vec256i \ |
Damjan Marion | 4fce7f7 | 2018-07-16 14:18:23 +0200 | [diff] [blame^] | 24 | _(i,8,32,epi8) _(i,16,16,epi16) _(i,32,8,epi32) _(i,64,4,epi64) |
Damjan Marion | c576622 | 2018-04-16 00:18:34 +0200 | [diff] [blame] | 25 | #define foreach_avx2_vec256u \ |
Damjan Marion | 4fce7f7 | 2018-07-16 14:18:23 +0200 | [diff] [blame^] | 26 | _(u,8,32,epi8) _(u,16,16,epi16) _(u,32,8,epi32) _(u,64,4,epi64) |
Damjan Marion | c576622 | 2018-04-16 00:18:34 +0200 | [diff] [blame] | 27 | #define foreach_avx2_vec256f \ |
| 28 | _(f,32,8,ps) _(f,64,4,pd) |
| 29 | |
Damjan Marion | 4fce7f7 | 2018-07-16 14:18:23 +0200 | [diff] [blame^] | 30 | #define _mm256_set1_epi64 _mm256_set1_epi64x |
| 31 | |
Damjan Marion | a52e166 | 2018-05-19 00:04:23 +0200 | [diff] [blame] | 32 | /* splat, load_unaligned, store_unaligned, is_all_zero, is_equal, |
| 33 | is_all_equal */ |
Damjan Marion | c576622 | 2018-04-16 00:18:34 +0200 | [diff] [blame] | 34 | #define _(t, s, c, i) \ |
| 35 | static_always_inline t##s##x##c \ |
| 36 | t##s##x##c##_splat (t##s x) \ |
| 37 | { return (t##s##x##c) _mm256_set1_##i (x); } \ |
| 38 | \ |
| 39 | static_always_inline t##s##x##c \ |
| 40 | t##s##x##c##_load_unaligned (void *p) \ |
| 41 | { return (t##s##x##c) _mm256_loadu_si256 (p); } \ |
| 42 | \ |
| 43 | static_always_inline void \ |
| 44 | t##s##x##c##_store_unaligned (t##s##x##c v, void *p) \ |
| 45 | { _mm256_storeu_si256 ((__m256i *) p, (__m256i) v); } \ |
| 46 | \ |
| 47 | static_always_inline int \ |
| 48 | t##s##x##c##_is_all_zero (t##s##x##c x) \ |
| 49 | { return _mm256_testz_si256 ((__m256i) x, (__m256i) x); } \ |
| 50 | \ |
| 51 | static_always_inline int \ |
Damjan Marion | 1486477 | 2018-05-22 14:07:47 +0200 | [diff] [blame] | 52 | t##s##x##c##_is_equal (t##s##x##c a, t##s##x##c b) \ |
| 53 | { return t##s##x##c##_is_all_zero (a ^ b); } \ |
Damjan Marion | c576622 | 2018-04-16 00:18:34 +0200 | [diff] [blame] | 54 | \ |
Damjan Marion | a52e166 | 2018-05-19 00:04:23 +0200 | [diff] [blame] | 55 | static_always_inline int \ |
| 56 | t##s##x##c##_is_all_equal (t##s##x##c v, t##s x) \ |
Damjan Marion | 4fce7f7 | 2018-07-16 14:18:23 +0200 | [diff] [blame^] | 57 | { return t##s##x##c##_is_equal (v, t##s##x##c##_splat (x)); } \ |
| 58 | \ |
| 59 | static_always_inline t##s##x##c \ |
| 60 | t##s##x##c##_interleave_lo (t##s##x##c a, t##s##x##c b) \ |
| 61 | { return (t##s##x##c) _mm256_unpacklo_##i ((__m256i) a, (__m256i) b); } \ |
| 62 | \ |
| 63 | static_always_inline t##s##x##c \ |
| 64 | t##s##x##c##_interleave_hi (t##s##x##c a, t##s##x##c b) \ |
| 65 | { return (t##s##x##c) _mm256_unpackhi_##i ((__m256i) a, (__m256i) b); } \ |
| 66 | |
Damjan Marion | c576622 | 2018-04-16 00:18:34 +0200 | [diff] [blame] | 67 | |
| 68 | foreach_avx2_vec256i foreach_avx2_vec256u |
| 69 | #undef _ |
Damjan Marion | a52e166 | 2018-05-19 00:04:23 +0200 | [diff] [blame] | 70 | /* *INDENT-ON* */ |
| 71 | |
| 72 | always_inline u32x8 |
Damjan Marion | c576622 | 2018-04-16 00:18:34 +0200 | [diff] [blame] | 73 | u32x8_permute (u32x8 v, u32x8 idx) |
| 74 | { |
| 75 | return (u32x8) _mm256_permutevar8x32_epi32 ((__m256i) v, (__m256i) idx); |
| 76 | } |
| 77 | |
Damjan Marion | 1cf9a16 | 2018-05-23 20:21:51 +0200 | [diff] [blame] | 78 | /* _extract_lo, _extract_hi */ |
| 79 | /* *INDENT-OFF* */ |
| 80 | #define _(t1,t2) \ |
| 81 | always_inline t1 \ |
| 82 | t2##_extract_lo (t2 v) \ |
| 83 | { return (t1) _mm256_extracti128_si256 ((__m256i) v, 0); } \ |
| 84 | \ |
| 85 | always_inline t1 \ |
| 86 | t2##_extract_hi (t2 v) \ |
| 87 | { return (t1) _mm256_extracti128_si256 ((__m256i) v, 1); } \ |
| 88 | \ |
| 89 | always_inline t2 \ |
| 90 | t2##_insert_lo (t2 v1, t1 v2) \ |
| 91 | { return (t2) _mm256_inserti128_si256 ((__m256i) v1, (__m128i) v2, 0); }\ |
| 92 | \ |
| 93 | always_inline t2 \ |
| 94 | t2##_insert_hi (t2 v1, t1 v2) \ |
| 95 | { return (t2) _mm256_inserti128_si256 ((__m256i) v1, (__m128i) v2, 1); }\ |
Damjan Marion | c576622 | 2018-04-16 00:18:34 +0200 | [diff] [blame] | 96 | |
Damjan Marion | 1cf9a16 | 2018-05-23 20:21:51 +0200 | [diff] [blame] | 97 | _(u8x16, u8x32) |
| 98 | _(u16x8, u16x16) |
| 99 | _(u32x4, u32x8) |
| 100 | _(u64x2, u64x4) |
| 101 | #undef _ |
| 102 | /* *INDENT-ON* */ |
Damjan Marion | c576622 | 2018-04-16 00:18:34 +0200 | [diff] [blame] | 103 | |
Damjan Marion | ee7f0bd | 2018-05-05 12:30:28 +0200 | [diff] [blame] | 104 | |
Damjan Marion | 1cf9a16 | 2018-05-23 20:21:51 +0200 | [diff] [blame] | 105 | |
Damjan Marion | ee7f0bd | 2018-05-05 12:30:28 +0200 | [diff] [blame] | 106 | |
Damjan Marion | 8c3f8a2 | 2018-05-17 21:12:13 +0200 | [diff] [blame] | 107 | static_always_inline u32 |
| 108 | u8x32_msb_mask (u8x32 v) |
| 109 | { |
| 110 | return _mm256_movemask_epi8 ((__m256i) v); |
| 111 | } |
| 112 | |
Damjan Marion | afe56de | 2018-05-17 12:44:00 +0200 | [diff] [blame] | 113 | /* _extend_to_ */ |
| 114 | /* *INDENT-OFF* */ |
| 115 | #define _(f,t,i) \ |
| 116 | static_always_inline t \ |
| 117 | f##_extend_to_##t (f x) \ |
| 118 | { return (t) _mm256_cvt##i ((__m128i) x); } |
| 119 | |
| 120 | _(u16x8, u32x8, epu16_epi32) |
| 121 | _(u16x8, u64x4, epu16_epi64) |
| 122 | _(u32x4, u64x4, epu32_epi64) |
| 123 | _(u8x16, u16x16, epu8_epi64) |
| 124 | _(u8x16, u32x8, epu8_epi32) |
| 125 | _(u8x16, u64x4, epu8_epi64) |
| 126 | _(i16x8, i32x8, epi16_epi32) |
| 127 | _(i16x8, i64x4, epi16_epi64) |
| 128 | _(i32x4, i64x4, epi32_epi64) |
| 129 | _(i8x16, i16x16, epi8_epi64) |
| 130 | _(i8x16, i32x8, epi8_epi32) |
| 131 | _(i8x16, i64x4, epi8_epi64) |
| 132 | #undef _ |
| 133 | /* *INDENT-ON* */ |
| 134 | |
Damjan Marion | bf129f4 | 2018-06-27 13:03:26 +0200 | [diff] [blame] | 135 | static_always_inline u16x16 |
| 136 | u16x16_byte_swap (u16x16 v) |
| 137 | { |
| 138 | u8x32 swap = { |
| 139 | 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, |
| 140 | 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14 |
| 141 | }; |
| 142 | return (u16x16) _mm256_shuffle_epi8 ((__m256i) v, (__m256i) swap); |
| 143 | } |
| 144 | |
| 145 | static_always_inline u32x8 |
| 146 | u32x8_hadd (u32x8 v1, u32x8 v2) |
| 147 | { |
| 148 | return (u32x8) _mm256_hadd_epi32 ((__m256i) v1, (__m256i) v2); |
| 149 | } |
| 150 | |
Damjan Marion | 08bca80 | 2018-06-18 22:21:40 +0200 | [diff] [blame] | 151 | static_always_inline u16x16 |
| 152 | u16x16_mask_last (u16x16 v, u8 n_last) |
| 153 | { |
| 154 | const u16x16 masks[17] = { |
| 155 | {0}, |
| 156 | {-1}, |
| 157 | {-1, -1}, |
| 158 | {-1, -1, -1}, |
| 159 | {-1, -1, -1, -1}, |
| 160 | {-1, -1, -1, -1, -1}, |
| 161 | {-1, -1, -1, -1, -1, -1}, |
| 162 | {-1, -1, -1, -1, -1, -1, -1}, |
| 163 | {-1, -1, -1, -1, -1, -1, -1, -1}, |
| 164 | {-1, -1, -1, -1, -1, -1, -1, -1, -1}, |
| 165 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, |
| 166 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, |
| 167 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, |
| 168 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, |
| 169 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, |
| 170 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, |
| 171 | {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, |
| 172 | }; |
| 173 | |
| 174 | ASSERT (n_last < 17); |
| 175 | |
| 176 | return v & masks[16 - n_last]; |
| 177 | } |
| 178 | |
Damjan Marion | c576622 | 2018-04-16 00:18:34 +0200 | [diff] [blame] | 179 | #endif /* included_vector_avx2_h */ |
| 180 | |
| 181 | /* |
| 182 | * fd.io coding-style-patch-verification: ON |
| 183 | * |
| 184 | * Local Variables: |
| 185 | * eval: (c-set-style "gnu") |
| 186 | * End: |
| 187 | */ |