Christophe Fontaine | 33e8195 | 2016-12-19 14:41:52 +0100 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2015 Cisco and/or its affiliates. |
| 3 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | * you may not use this file except in compliance with the License. |
| 5 | * You may obtain a copy of the License at: |
| 6 | * |
| 7 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | * |
| 9 | * Unless required by applicable law or agreed to in writing, software |
| 10 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | * See the License for the specific language governing permissions and |
| 13 | * limitations under the License. |
| 14 | */ |
| 15 | |
| 16 | #ifndef included_vector_neon_h |
| 17 | #define included_vector_neon_h |
| 18 | #include <arm_neon.h> |
| 19 | |
Christophe Fontaine | 33e8195 | 2016-12-19 14:41:52 +0100 | [diff] [blame] | 20 | /* Arithmetic */ |
Christophe Fontaine | 33e8195 | 2016-12-19 14:41:52 +0100 | [diff] [blame] | 21 | #define u16x8_sub_saturate(a,b) vsubq_u16(a,b) |
| 22 | #define i16x8_sub_saturate(a,b) vsubq_s16(a,b) |
Lijian Zhang | f5942d5 | 2018-10-31 13:35:20 +0800 | [diff] [blame] | 23 | /* Dummy. Aid making uniform macros */ |
| 24 | #define vreinterpretq_u8_u8(a) a |
Lijian.Zhang | e6a47cf | 2019-03-12 18:32:39 +0800 | [diff] [blame] | 25 | /* Implement the missing intrinsics to make uniform macros */ |
| 26 | #define vminvq_u64(x) \ |
| 27 | ({ \ |
| 28 | u64 x0 = vgetq_lane_u64(x, 0); \ |
| 29 | u64 x1 = vgetq_lane_u64(x, 1); \ |
| 30 | x0 < x1 ? x0 : x1; \ |
| 31 | }) |
Christophe Fontaine | 33e8195 | 2016-12-19 14:41:52 +0100 | [diff] [blame] | 32 | |
Gabriel Ganne | b81831d | 2017-12-05 17:33:37 +0100 | [diff] [blame] | 33 | /* Converts all ones/zeros compare mask to bitmap. */ |
| 34 | always_inline u32 |
Lijian Zhang | a22ba88 | 2018-10-25 18:50:33 +0800 | [diff] [blame] | 35 | u8x16_compare_byte_mask (u8x16 v) |
Gabriel Ganne | b81831d | 2017-12-05 17:33:37 +0100 | [diff] [blame] | 36 | { |
Lijian Zhang | a22ba88 | 2018-10-25 18:50:33 +0800 | [diff] [blame] | 37 | uint8x16_t mask = { 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, |
| 38 | 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80 |
| 39 | }; |
| 40 | /* v --> [0xFF, 0x00, 0xFF, 0xFF, 0xFF, 0x00, 0xFF, 0x00, ... ] */ |
| 41 | uint8x16_t x = vandq_u8 (v, mask); |
| 42 | /* after v & mask, |
| 43 | * x --> [0x01, 0x00, 0x04, 0x08, 0x10, 0x00, 0x40, 0x00, ... ] */ |
| 44 | uint64x2_t x64 = vpaddlq_u32 (vpaddlq_u16 (vpaddlq_u8 (x))); |
| 45 | /* after merge, x64 --> [0x5D, 0x.. ] */ |
| 46 | return (u32) (vgetq_lane_u64 (x64, 0) + (vgetq_lane_u64 (x64, 1) << 8)); |
Gabriel Ganne | b81831d | 2017-12-05 17:33:37 +0100 | [diff] [blame] | 47 | } |
Christophe Fontaine | 33e8195 | 2016-12-19 14:41:52 +0100 | [diff] [blame] | 48 | |
Sirshak Das | 536953d | 2018-06-26 13:08:46 -0500 | [diff] [blame] | 49 | /* *INDENT-OFF* */ |
| 50 | #define foreach_neon_vec128i \ |
| 51 | _(i,8,16,s8) _(i,16,8,s16) _(i,32,4,s32) _(i,64,2,s64) |
| 52 | #define foreach_neon_vec128u \ |
| 53 | _(u,8,16,u8) _(u,16,8,u16) _(u,32,4,u32) _(u,64,2,u64) |
| 54 | #define foreach_neon_vec128f \ |
| 55 | _(f,32,4,f32) _(f,64,2,f64) |
Adrian Oanca | 22ac59b | 2018-02-23 16:27:41 +0100 | [diff] [blame] | 56 | |
Sirshak Das | 536953d | 2018-06-26 13:08:46 -0500 | [diff] [blame] | 57 | #define _(t, s, c, i) \ |
| 58 | static_always_inline t##s##x##c \ |
| 59 | t##s##x##c##_splat (t##s x) \ |
| 60 | { return (t##s##x##c) vdupq_n_##i (x); } \ |
| 61 | \ |
| 62 | static_always_inline t##s##x##c \ |
| 63 | t##s##x##c##_load_unaligned (void *p) \ |
| 64 | { return (t##s##x##c) vld1q_##i (p); } \ |
| 65 | \ |
| 66 | static_always_inline void \ |
| 67 | t##s##x##c##_store_unaligned (t##s##x##c v, void *p) \ |
| 68 | { vst1q_##i (p, v); } \ |
| 69 | \ |
| 70 | static_always_inline int \ |
| 71 | t##s##x##c##_is_all_zero (t##s##x##c x) \ |
Lijian.Zhang | e6a47cf | 2019-03-12 18:32:39 +0800 | [diff] [blame] | 72 | { return !!(vminvq_u##s (vceqq_##i (vdupq_n_##i(0), x))); } \ |
Sirshak Das | 536953d | 2018-06-26 13:08:46 -0500 | [diff] [blame] | 73 | \ |
| 74 | static_always_inline int \ |
| 75 | t##s##x##c##_is_equal (t##s##x##c a, t##s##x##c b) \ |
Lijian.Zhang | e6a47cf | 2019-03-12 18:32:39 +0800 | [diff] [blame] | 76 | { return !!(vminvq_u##s (vceqq_##i (a, b))); } \ |
Sirshak Das | 536953d | 2018-06-26 13:08:46 -0500 | [diff] [blame] | 77 | \ |
| 78 | static_always_inline int \ |
| 79 | t##s##x##c##_is_all_equal (t##s##x##c v, t##s x) \ |
| 80 | { return t##s##x##c##_is_equal (v, t##s##x##c##_splat (x)); }; \ |
Lijian Zhang | f5942d5 | 2018-10-31 13:35:20 +0800 | [diff] [blame] | 81 | \ |
| 82 | static_always_inline u32 \ |
| 83 | t##s##x##c##_zero_byte_mask (t##s##x##c x) \ |
| 84 | { uint8x16_t v = vreinterpretq_u8_u##s (vceqq_##i (vdupq_n_##i(0), x)); \ |
| 85 | return u8x16_compare_byte_mask (v); } \ |
Lijian.Zhang | 37c8378 | 2019-04-04 15:26:26 +0800 | [diff] [blame] | 86 | \ |
| 87 | static_always_inline u##s##x##c \ |
| 88 | t##s##x##c##_is_greater (t##s##x##c a, t##s##x##c b) \ |
| 89 | { return (u##s##x##c) vcgtq_##i (a, b); } \ |
| 90 | \ |
| 91 | static_always_inline t##s##x##c \ |
| 92 | t##s##x##c##_blend (t##s##x##c dst, t##s##x##c src, u##s##x##c mask) \ |
| 93 | { return (t##s##x##c) vbslq_##i (mask, src, dst); } |
Adrian Oanca | 22ac59b | 2018-02-23 16:27:41 +0100 | [diff] [blame] | 94 | |
Sirshak Das | 536953d | 2018-06-26 13:08:46 -0500 | [diff] [blame] | 95 | foreach_neon_vec128i foreach_neon_vec128u |
| 96 | |
| 97 | #undef _ |
| 98 | /* *INDENT-ON* */ |
| 99 | |
Sirshak Das | afc725a | 2018-07-27 01:13:33 -0500 | [diff] [blame] | 100 | static_always_inline u16x8 |
| 101 | u16x8_byte_swap (u16x8 v) |
| 102 | { |
Sirshak Das | 6da42de | 2018-08-22 12:02:04 +0800 | [diff] [blame] | 103 | return (u16x8) vrev16q_u8 ((u8x16) v); |
Sirshak Das | afc725a | 2018-07-27 01:13:33 -0500 | [diff] [blame] | 104 | } |
| 105 | |
Damjan Marion | dd648aa | 2020-03-12 11:56:00 +0100 | [diff] [blame] | 106 | static_always_inline u32x4 |
| 107 | u32x4_byte_swap (u32x4 v) |
| 108 | { |
| 109 | return vrev64q_u32 (v); |
| 110 | } |
| 111 | |
Sirshak Das | afc725a | 2018-07-27 01:13:33 -0500 | [diff] [blame] | 112 | static_always_inline u8x16 |
| 113 | u8x16_shuffle (u8x16 v, u8x16 m) |
| 114 | { |
| 115 | return (u8x16) vqtbl1q_u8 (v, m); |
| 116 | } |
| 117 | |
Sirshak Das | 61f325d | 2018-08-03 11:24:51 -0500 | [diff] [blame] | 118 | static_always_inline u32x4 |
| 119 | u32x4_hadd (u32x4 v1, u32x4 v2) |
| 120 | { |
| 121 | return (u32x4) vpaddq_u32 (v1, v2); |
| 122 | } |
| 123 | |
Sirshak Das | 8e5d5db | 2018-08-22 14:04:33 +0800 | [diff] [blame] | 124 | static_always_inline u64x2 |
| 125 | u32x4_extend_to_u64x2 (u32x4 v) |
| 126 | { |
| 127 | return vmovl_u32 (vget_low_u32 (v)); |
| 128 | } |
| 129 | |
Sirshak Das | 759226e | 2018-08-22 08:46:52 +0800 | [diff] [blame] | 130 | static_always_inline u64x2 |
| 131 | u32x4_extend_to_u64x2_high (u32x4 v) |
| 132 | { |
Sirshak Das | 4f61117 | 2018-10-09 11:28:44 -0500 | [diff] [blame] | 133 | return vmovl_high_u32 (v); |
Sirshak Das | 759226e | 2018-08-22 08:46:52 +0800 | [diff] [blame] | 134 | } |
| 135 | |
Lijian Zhang | a22ba88 | 2018-10-25 18:50:33 +0800 | [diff] [blame] | 136 | /* Creates a mask made up of the MSB of each byte of the source vector */ |
| 137 | static_always_inline u16 |
| 138 | u8x16_msb_mask (u8x16 v) |
| 139 | { |
| 140 | int8x16_t shift = |
| 141 | { -7, -6, -5, -4, -3, -2, -1, 0, -7, -6, -5, -4, -3, -2, -1, 0 }; |
| 142 | /* v --> [0x80, 0x7F, 0xF0, 0xAF, 0xF0, 0x00, 0xF2, 0x00, ... ] */ |
| 143 | uint8x16_t x = vshlq_u8 (vandq_u8 (v, vdupq_n_u8 (0x80)), shift); |
| 144 | /* after (v & 0x80) >> shift, |
| 145 | * x --> [0x01, 0x00, 0x04, 0x08, 0x10, 0x00, 0x40, 0x00, ... ] */ |
| 146 | uint64x2_t x64 = vpaddlq_u32 (vpaddlq_u16 (vpaddlq_u8 (x))); |
| 147 | /* after merge, x64 --> [0x5D, 0x.. ] */ |
| 148 | return (u16) (vgetq_lane_u64 (x64, 0) + (vgetq_lane_u64 (x64, 1) << 8)); |
| 149 | } |
| 150 | |
Lijian.Zhang | 107aa83 | 2019-05-29 17:08:47 +0800 | [diff] [blame] | 151 | static_always_inline u64x2 |
| 152 | u64x2_gather (void *p0, void *p1) |
| 153 | { |
| 154 | u64x2 r = vdupq_n_u64 (*(u64 *) p0); |
| 155 | r = vsetq_lane_u64 (*(u64 *) p1, r, 1); |
| 156 | return r; |
| 157 | } |
| 158 | |
| 159 | static_always_inline u32x4 |
| 160 | u32x4_gather (void *p0, void *p1, void *p2, void *p3) |
| 161 | { |
| 162 | u32x4 r = vdupq_n_u32 (*(u32 *) p0); |
| 163 | r = vsetq_lane_u32 (*(u32 *) p1, r, 1); |
| 164 | r = vsetq_lane_u32 (*(u32 *) p2, r, 2); |
| 165 | r = vsetq_lane_u32 (*(u32 *) p3, r, 3); |
| 166 | return r; |
| 167 | } |
| 168 | |
Lijian.Zhang | 0f98451 | 2019-05-29 17:13:42 +0800 | [diff] [blame] | 169 | static_always_inline void |
| 170 | u64x2_scatter (u64x2 r, void *p0, void *p1) |
| 171 | { |
| 172 | *(u64 *) p0 = vgetq_lane_u64 (r, 0); |
| 173 | *(u64 *) p1 = vgetq_lane_u64 (r, 1); |
| 174 | } |
| 175 | |
| 176 | static_always_inline void |
| 177 | u32x4_scatter (u32x4 r, void *p0, void *p1, void *p2, void *p3) |
| 178 | { |
| 179 | *(u32 *) p0 = vgetq_lane_u32 (r, 0); |
| 180 | *(u32 *) p1 = vgetq_lane_u32 (r, 1); |
| 181 | *(u32 *) p2 = vgetq_lane_u32 (r, 2); |
| 182 | *(u32 *) p3 = vgetq_lane_u32 (r, 3); |
| 183 | } |
| 184 | |
Damjan Marion | 776644e | 2020-01-31 10:24:07 +0100 | [diff] [blame] | 185 | static_always_inline u32 |
| 186 | u32x4_min_scalar (u32x4 v) |
| 187 | { |
| 188 | return vminvq_u32 (v); |
| 189 | } |
| 190 | |
Damjan Marion | 4339c36 | 2020-02-18 15:12:07 +0100 | [diff] [blame] | 191 | #define u8x16_word_shift_left(x,n) vextq_u8(u8x16_splat (0), x, 16 - n) |
| 192 | #define u8x16_word_shift_right(x,n) vextq_u8(x, u8x16_splat (0), n) |
Damjan Marion | 9372374 | 2020-02-12 20:27:46 +0100 | [diff] [blame] | 193 | |
Damjan Marion | 622b5ce | 2020-02-12 10:59:14 +0100 | [diff] [blame] | 194 | static_always_inline u8x16 |
| 195 | u8x16_reflect (u8x16 v) |
| 196 | { |
| 197 | u8x16 mask = { |
| 198 | 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 |
| 199 | }; |
| 200 | return (u8x16) vqtbl1q_u8 (v, mask); |
| 201 | } |
| 202 | |
Damjan Marion | f75defa | 2020-02-13 18:14:06 +0100 | [diff] [blame] | 203 | static_always_inline u8x16 |
| 204 | u8x16_xor3 (u8x16 a, u8x16 b, u8x16 c) |
| 205 | { |
| 206 | #if __GNUC__ == 8 && __ARM_FEATURE_SHA3 == 1 |
| 207 | u8x16 r; |
| 208 | __asm__ ("eor3 %0.16b,%1.16b,%2.16b,%3.16b": "=w" (r): "0" (a), "w" (b), "w" (c):); |
| 209 | return r; |
| 210 | #endif |
| 211 | return a ^ b ^ c; |
| 212 | } |
| 213 | |
Lijian Zhang | a22ba88 | 2018-10-25 18:50:33 +0800 | [diff] [blame] | 214 | #define CLIB_HAVE_VEC128_MSB_MASK |
| 215 | |
Sirshak Das | 536953d | 2018-06-26 13:08:46 -0500 | [diff] [blame] | 216 | #define CLIB_HAVE_VEC128_UNALIGNED_LOAD_STORE |
| 217 | #define CLIB_VEC128_SPLAT_DEFINED |
Christophe Fontaine | 33e8195 | 2016-12-19 14:41:52 +0100 | [diff] [blame] | 218 | #endif /* included_vector_neon_h */ |
| 219 | |
| 220 | /* |
| 221 | * fd.io coding-style-patch-verification: ON |
| 222 | * |
| 223 | * Local Variables: |
| 224 | * eval: (c-set-style "gnu") |
| 225 | * End: |
| 226 | */ |