blob: d523e88da89c475f8189eea1381b7eb2642575c3 [file] [log] [blame]
Damjan Marion522e4862016-03-04 12:44:14 +01001/*
2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16#ifndef included_clib_cpu_h
17#define included_clib_cpu_h
18
Damjan Marion0a78fa12019-01-19 23:45:36 +010019#include <sys/syscall.h>
Damjan Marion1c80e832016-05-11 23:07:18 +020020#include <vppinfra/format.h>
21
22/*
23 * multiarchitecture support. Adding new entry will produce
24 * new graph node function variant optimized for specific cpu
25 * microarchitecture.
26 * Order is important for runtime selection, as 1st match wins...
27 */
28
29#if __x86_64__ && CLIB_DEBUG == 0
30#define foreach_march_variant(macro, x) \
31 macro(avx2, x, "arch=core-avx2")
32#else
33#define foreach_march_variant(macro, x)
34#endif
35
36
Damjan Marion5ad75f52018-11-29 14:40:30 +010037#if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0
Damjan Marion24223172018-05-28 16:22:14 +020038#define CLIB_CPU_OPTIMIZED __attribute__ ((optimize ("O3")))
Damjan Marion1c80e832016-05-11 23:07:18 +020039#else
40#define CLIB_CPU_OPTIMIZED
41#endif
42
43
44#define CLIB_MULTIARCH_ARCH_CHECK(arch, fn, tgt) \
45 if (clib_cpu_supports_ ## arch()) \
46 return & fn ## _ ##arch;
47
Damjan Marion652d2e12019-02-02 00:15:27 +010048/* FIXME to be removed */
49#define CLIB_MULTIARCH_SELECT_FN(fn,...)
Damjan Marion1c80e832016-05-11 23:07:18 +020050
Damjan Marion812b32d2018-05-28 21:26:47 +020051#ifdef CLIB_MARCH_VARIANT
Damjan Marion04f3db32017-11-10 21:55:45 +010052#define __CLIB_MULTIARCH_FN(a,b) a##_##b
53#define _CLIB_MULTIARCH_FN(a,b) __CLIB_MULTIARCH_FN(a,b)
Damjan Marion812b32d2018-05-28 21:26:47 +020054#define CLIB_MULTIARCH_FN(fn) _CLIB_MULTIARCH_FN(fn,CLIB_MARCH_VARIANT)
Damjan Marion04f3db32017-11-10 21:55:45 +010055#else
56#define CLIB_MULTIARCH_FN(fn) fn
57#endif
Damjan Marion1c80e832016-05-11 23:07:18 +020058
Damjan Marion812b32d2018-05-28 21:26:47 +020059#define CLIB_MARCH_SFX CLIB_MULTIARCH_FN
60
Damjan Marion910d3692019-01-21 11:48:34 +010061typedef struct _clib_march_fn_registration
62{
63 void *function;
64 int priority;
65 struct _clib_march_fn_registration *next;
66 char *name;
67} clib_march_fn_registration;
68
69static_always_inline void *
70clib_march_select_fn_ptr (clib_march_fn_registration * r)
71{
72 void *rv = 0;
73 int last_prio = -1;
74
75 while (r)
76 {
77 if (last_prio < r->priority)
78 {
79 last_prio = r->priority;
80 rv = r->function;
81 }
82 r = r->next;
83 }
84 return rv;
85}
86
87#define CLIB_MARCH_FN_POINTER(fn) \
88 clib_march_select_fn_ptr (fn##_march_fn_registrations);
89
90#define _CLIB_MARCH_FN_REGISTRATION(fn) \
91static clib_march_fn_registration \
92CLIB_MARCH_SFX(fn##_march_fn_registration) = \
93{ \
94 .name = CLIB_MARCH_VARIANT_STR \
95}; \
96\
97static void __clib_constructor \
98fn##_march_register () \
99{ \
100 clib_march_fn_registration *r; \
101 r = & CLIB_MARCH_SFX (fn##_march_fn_registration); \
102 r->priority = CLIB_MARCH_FN_PRIORITY(); \
103 r->next = fn##_march_fn_registrations; \
104 r->function = CLIB_MARCH_SFX (fn); \
105 fn##_march_fn_registrations = r; \
106}
107
108#ifdef CLIB_MARCH_VARIANT
109#define CLIB_MARCH_FN_REGISTRATION(fn) \
110extern clib_march_fn_registration *fn##_march_fn_registrations; \
111_CLIB_MARCH_FN_REGISTRATION(fn)
112#else
113#define CLIB_MARCH_FN_REGISTRATION(fn) \
114clib_march_fn_registration *fn##_march_fn_registrations = 0; \
115_CLIB_MARCH_FN_REGISTRATION(fn)
116#endif
Damjan Marion1c80e832016-05-11 23:07:18 +0200117#define foreach_x86_64_flags \
118_ (sse3, 1, ecx, 0) \
119_ (ssse3, 1, ecx, 9) \
120_ (sse41, 1, ecx, 19) \
121_ (sse42, 1, ecx, 20) \
122_ (avx, 1, ecx, 28) \
123_ (avx2, 7, ebx, 5) \
124_ (avx512f, 7, ebx, 16) \
Gabriel Ganne73cb0062017-12-05 14:26:33 +0100125_ (x86_aes, 1, ecx, 25) \
Damjan Marionc0e939b2016-11-12 11:50:01 +0100126_ (sha, 7, ebx, 29) \
127_ (invariant_tsc, 0x80000007, edx, 8)
Damjan Marion1c80e832016-05-11 23:07:18 +0200128
Gabriel Ganne73cb0062017-12-05 14:26:33 +0100129
130#define foreach_aarch64_flags \
131_ (fp, 0) \
132_ (asimd, 1) \
133_ (evtstrm, 2) \
134_ (aarch64_aes, 3) \
135_ (pmull, 4) \
136_ (sha1, 5) \
137_ (sha2, 6) \
138_ (crc32, 7) \
139_ (atomics, 8) \
140_ (fphp, 9) \
141_ (asimdhp, 10) \
142_ (cpuid, 11) \
143_ (asimdrdm, 12) \
144_ (jscvt, 13) \
145_ (fcma, 14) \
146_ (lrcpc, 15) \
147_ (dcpop, 16) \
148_ (sha3, 17) \
149_ (sm3, 18) \
150_ (sm4, 19) \
151_ (asimddp, 20) \
152_ (sha512, 21) \
153_ (sve, 22)
154
Damjan Marion0a78fa12019-01-19 23:45:36 +0100155static inline u32
Damjan Marionee721412019-01-27 17:54:11 +0100156clib_get_current_cpu_id ()
Damjan Marion0a78fa12019-01-19 23:45:36 +0100157{
158 unsigned cpu, node;
159 syscall (__NR_getcpu, &cpu, &node, 0);
160 return cpu;
161}
162
163static inline u32
164clib_get_current_numa_node ()
165{
166 unsigned cpu, node;
167 syscall (__NR_getcpu, &cpu, &node, 0);
168 return node;
169}
170
Christophe Fontaine33e81952016-12-19 14:41:52 +0100171#if defined(__x86_64__)
172#include "cpuid.h"
173
Damjan Marion1c80e832016-05-11 23:07:18 +0200174static inline int
Dave Barachc3799992016-08-15 11:12:27 -0400175clib_get_cpuid (const u32 lev, u32 * eax, u32 * ebx, u32 * ecx, u32 * edx)
Damjan Marion1c80e832016-05-11 23:07:18 +0200176{
177 if ((u32) __get_cpuid_max (0x80000000 & lev, 0) < lev)
178 return 0;
179 if (lev == 7)
Dave Barachc3799992016-08-15 11:12:27 -0400180 __cpuid_count (lev, 0, *eax, *ebx, *ecx, *edx);
Damjan Marion1c80e832016-05-11 23:07:18 +0200181 else
Dave Barachc3799992016-08-15 11:12:27 -0400182 __cpuid (lev, *eax, *ebx, *ecx, *edx);
Damjan Marion1c80e832016-05-11 23:07:18 +0200183 return 1;
184}
185
186
187#define _(flag, func, reg, bit) \
188static inline int \
189clib_cpu_supports_ ## flag() \
190{ \
191 u32 __attribute__((unused)) eax, ebx = 0, ecx = 0, edx = 0; \
192 clib_get_cpuid (func, &eax, &ebx, &ecx, &edx); \
193 \
194 return ((reg & (1 << bit)) != 0); \
195}
Dave Barachc3799992016-08-15 11:12:27 -0400196foreach_x86_64_flags
Damjan Marion1c80e832016-05-11 23:07:18 +0200197#undef _
Gabriel Ganne73cb0062017-12-05 14:26:33 +0100198#else /* __x86_64__ */
Christophe Fontaine33e81952016-12-19 14:41:52 +0100199
200#define _(flag, func, reg, bit) \
201static inline int clib_cpu_supports_ ## flag() { return 0; }
202foreach_x86_64_flags
203#undef _
Gabriel Ganne73cb0062017-12-05 14:26:33 +0100204#endif /* __x86_64__ */
205#if defined(__aarch64__)
206#include <sys/auxv.h>
207#define _(flag, bit) \
208static inline int \
209clib_cpu_supports_ ## flag() \
210{ \
211 unsigned long hwcap = getauxval(AT_HWCAP); \
212 return (hwcap & (1 << bit)); \
213}
214 foreach_aarch64_flags
215#undef _
216#else /* ! __x86_64__ && !__aarch64__ */
217#define _(flag, bit) \
218static inline int clib_cpu_supports_ ## flag() { return 0; }
219 foreach_aarch64_flags
220#undef _
221#endif /* __x86_64__, __aarch64__ */
222/*
223 * aes is the only feature with the same name in both flag lists
224 * handle this by prefixing it with the arch name, and handling it
225 * with the custom function below
226 */
227 static inline int
228clib_cpu_supports_aes ()
229{
230#if defined (__aarch64__)
231 return clib_cpu_supports_x86_aes ();
232#elif defined (__aarch64__)
233 return clib_cpu_supports_aarch64_aes ();
234#else
235 return 0;
Christophe Fontaine33e81952016-12-19 14:41:52 +0100236#endif
Gabriel Ganne73cb0062017-12-05 14:26:33 +0100237}
238
Damjan Marion812b32d2018-05-28 21:26:47 +0200239static inline int
240clib_cpu_march_priority_avx512 ()
241{
242 if (clib_cpu_supports_avx512f ())
243 return 20;
244 return -1;
245}
246
247static inline int
248clib_cpu_march_priority_avx2 ()
249{
250 if (clib_cpu_supports_avx2 ())
Damjan Marion64593152019-03-12 19:59:22 +0100251 return 50;
Damjan Marion812b32d2018-05-28 21:26:47 +0200252 return -1;
253}
254
Lijian Zhang2e237212018-09-10 17:13:56 +0800255static inline u32
256clib_cpu_implementer ()
257{
258 char buf[128];
259 static u32 implementer = -1;
260
261 if (-1 != implementer)
262 return implementer;
263
264 FILE *fp = fopen ("/proc/cpuinfo", "r");
265 if (!fp)
266 return implementer;
267
268 while (!feof (fp))
269 {
270 if (!fgets (buf, sizeof (buf), fp))
271 break;
272 buf[127] = '\0';
273 if (strstr (buf, "CPU implementer"))
274 implementer = (u32) strtol (memchr (buf, ':', 128) + 2, NULL, 0);
275 if (-1 != implementer)
276 break;
277 }
278 fclose (fp);
279
280 return implementer;
281}
282
283static inline u32
284clib_cpu_part ()
285{
286 char buf[128];
287 static u32 part = -1;
288
289 if (-1 != part)
290 return part;
291
292 FILE *fp = fopen ("/proc/cpuinfo", "r");
293 if (!fp)
294 return part;
295
296 while (!feof (fp))
297 {
298 if (!fgets (buf, sizeof (buf), fp))
299 break;
300 buf[127] = '\0';
301 if (strstr (buf, "CPU part"))
302 part = (u32) strtol (memchr (buf, ':', 128) + 2, NULL, 0);
303 if (-1 != part)
304 break;
305 }
306 fclose (fp);
307
308 return part;
309}
310
311#define AARCH64_CPU_IMPLEMENTER_THUNERDERX2 0x43
312#define AARCH64_CPU_PART_THUNERDERX2 0x0af
313#define AARCH64_CPU_IMPLEMENTER_QDF24XX 0x51
314#define AARCH64_CPU_PART_QDF24XX 0xc00
315#define AARCH64_CPU_IMPLEMENTER_CORTEXA72 0x41
316#define AARCH64_CPU_PART_CORTEXA72 0xd08
317
318static inline int
319clib_cpu_march_priority_thunderx2t99 ()
320{
321 if ((AARCH64_CPU_IMPLEMENTER_THUNERDERX2 == clib_cpu_implementer ()) &&
322 (AARCH64_CPU_PART_THUNERDERX2 == clib_cpu_part ()))
323 return 20;
324 return -1;
325}
326
327static inline int
328clib_cpu_march_priority_qdf24xx ()
329{
330 if ((AARCH64_CPU_IMPLEMENTER_QDF24XX == clib_cpu_implementer ()) &&
331 (AARCH64_CPU_PART_QDF24XX == clib_cpu_part ()))
332 return 20;
333 return -1;
334}
335
336static inline int
337clib_cpu_march_priority_cortexa72 ()
338{
339 if ((AARCH64_CPU_IMPLEMENTER_CORTEXA72 == clib_cpu_implementer ()) &&
340 (AARCH64_CPU_PART_CORTEXA72 == clib_cpu_part ()))
341 return 10;
342 return -1;
343}
344
Damjan Marion812b32d2018-05-28 21:26:47 +0200345#ifdef CLIB_MARCH_VARIANT
346#define CLIB_MARCH_FN_PRIORITY() CLIB_MARCH_SFX(clib_cpu_march_priority)()
347#else
348#define CLIB_MARCH_FN_PRIORITY() 0
349#endif
Gabriel Ganne73cb0062017-12-05 14:26:33 +0100350#endif /* included_clib_cpu_h */
351
Florin Coras983cc7d2018-09-18 23:11:55 -0700352#define CLIB_MARCH_FN_CONSTRUCTOR(fn) \
353static void __clib_constructor \
354CLIB_MARCH_SFX(fn ## _march_constructor) (void) \
355{ \
356 if (CLIB_MARCH_FN_PRIORITY() > fn ## _selected_priority) \
357 { \
358 fn ## _selected = & CLIB_MARCH_SFX (fn ## _ma); \
359 fn ## _selected_priority = CLIB_MARCH_FN_PRIORITY(); \
360 } \
361} \
362
363#ifndef CLIB_MARCH_VARIANT
364#define CLIB_MARCH_FN(fn, rtype, _args...) \
365 static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args); \
366 rtype (*fn ## _selected) (_args) = & CLIB_MARCH_SFX (fn ## _ma); \
367 int fn ## _selected_priority = 0; \
368 static inline rtype CLIB_CPU_OPTIMIZED \
369 CLIB_MARCH_SFX (fn ## _ma)(_args)
370#else
371#define CLIB_MARCH_FN(fn, rtype, _args...) \
372 static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args); \
373 extern int (*fn ## _selected) (_args); \
374 extern int fn ## _selected_priority; \
375 CLIB_MARCH_FN_CONSTRUCTOR (fn) \
376 static rtype CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (fn ## _ma)(_args)
377#endif
378
379#define CLIB_MARCH_FN_SELECT(fn) (* fn ## _selected)
380
Gabriel Ganne73cb0062017-12-05 14:26:33 +0100381format_function_t format_cpu_uarch;
Damjan Marion522e4862016-03-04 12:44:14 +0100382format_function_t format_cpu_model_name;
Damjan Marion1c80e832016-05-11 23:07:18 +0200383format_function_t format_cpu_flags;
Damjan Marion522e4862016-03-04 12:44:14 +0100384
Dave Barachc3799992016-08-15 11:12:27 -0400385/*
386 * fd.io coding-style-patch-verification: ON
387 *
388 * Local Variables:
389 * eval: (c-set-style "gnu")
390 * End:
391 */