1/* 2 * kmp_os.h -- KPTS runtime header file. 3 */ 4 5//===----------------------------------------------------------------------===// 6// 7// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 8// See https://llvm.org/LICENSE.txt for license information. 9// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 10// 11//===----------------------------------------------------------------------===// 12 13#ifndef KMP_OS_H 14#define KMP_OS_H 15 16#include "kmp_config.h" 17#include <atomic> 18#include <stdarg.h> 19#include <stdlib.h> 20#include <string.h> 21 22#define KMP_FTN_PLAIN 1 23#define KMP_FTN_APPEND 2 24#define KMP_FTN_UPPER 3 25/* 26#define KMP_FTN_PREPEND 4 27#define KMP_FTN_UAPPEND 5 28*/ 29 30#define KMP_PTR_SKIP (sizeof(void *)) 31 32/* -------------------------- Compiler variations ------------------------ */ 33 34#define KMP_OFF 0 35#define KMP_ON 1 36 37#define KMP_MEM_CONS_VOLATILE 0 38#define KMP_MEM_CONS_FENCE 1 39 40#ifndef KMP_MEM_CONS_MODEL 41#define KMP_MEM_CONS_MODEL KMP_MEM_CONS_VOLATILE 42#endif 43 44#ifndef __has_cpp_attribute 45#define __has_cpp_attribute(x) 0 46#endif 47 48#ifndef __has_attribute 49#define __has_attribute(x) 0 50#endif 51 52/* ------------------------- Compiler recognition ---------------------- */ 53#define KMP_COMPILER_ICC 0 54#define KMP_COMPILER_GCC 0 55#define KMP_COMPILER_CLANG 0 56#define KMP_COMPILER_MSVC 0 57#define KMP_COMPILER_ICX 0 58 59#if __INTEL_CLANG_COMPILER 60#undef KMP_COMPILER_ICX 61#define KMP_COMPILER_ICX 1 62#elif defined(__INTEL_COMPILER) 63#undef KMP_COMPILER_ICC 64#define KMP_COMPILER_ICC 1 65#elif defined(__clang__) 66#undef KMP_COMPILER_CLANG 67#define KMP_COMPILER_CLANG 1 68#elif defined(__GNUC__) 69#undef KMP_COMPILER_GCC 70#define KMP_COMPILER_GCC 1 71#elif defined(_MSC_VER) 72#undef KMP_COMPILER_MSVC 73#define KMP_COMPILER_MSVC 1 74#else 75#error Unknown compiler 76#endif 77 78#if (KMP_OS_LINUX || KMP_OS_WINDOWS || KMP_OS_FREEBSD || KMP_OS_AIX) && \ 79 !KMP_OS_WASI 80#define KMP_AFFINITY_SUPPORTED 1 81#if KMP_OS_WINDOWS && KMP_ARCH_X86_64 82#define KMP_GROUP_AFFINITY 1 83#else 84#define KMP_GROUP_AFFINITY 0 85#endif 86#else 87#define KMP_AFFINITY_SUPPORTED 0 88#define KMP_GROUP_AFFINITY 0 89#endif 90 91#if (KMP_OS_LINUX || (KMP_OS_FREEBSD && __FreeBSD_version >= 1301000)) 92#define KMP_HAVE_SCHED_GETCPU 1 93#else 94#define KMP_HAVE_SCHED_GETCPU 0 95#endif 96 97/* Check for quad-precision extension. */ 98#define KMP_HAVE_QUAD 0 99#if KMP_ARCH_X86 || KMP_ARCH_X86_64 100#if KMP_COMPILER_ICC || KMP_COMPILER_ICX 101/* _Quad is already defined for icc */ 102#undef KMP_HAVE_QUAD 103#define KMP_HAVE_QUAD 1 104#elif KMP_COMPILER_CLANG 105/* Clang doesn't support a software-implemented 106 128-bit extended precision type yet */ 107typedef long double _Quad; 108#elif KMP_COMPILER_GCC 109/* GCC on NetBSD lacks __multc3/__divtc3 builtins needed for quad until 110 NetBSD 10.0 which ships with GCC 10.5 */ 111#if (!KMP_OS_NETBSD || __GNUC__ >= 10) 112typedef __float128 _Quad; 113#undef KMP_HAVE_QUAD 114#define KMP_HAVE_QUAD 1 115#endif 116#elif KMP_COMPILER_MSVC 117typedef long double _Quad; 118#endif 119#else 120#if __LDBL_MAX_EXP__ >= 16384 && KMP_COMPILER_GCC 121typedef long double _Quad; 122#undef KMP_HAVE_QUAD 123#define KMP_HAVE_QUAD 1 124#endif 125#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 126 127#define KMP_USE_X87CONTROL 0 128#if KMP_OS_WINDOWS 129#define KMP_END_OF_LINE "\r\n" 130typedef char kmp_int8; 131typedef unsigned char kmp_uint8; 132typedef short kmp_int16; 133typedef unsigned short kmp_uint16; 134typedef int kmp_int32; 135typedef unsigned int kmp_uint32; 136#define KMP_INT32_SPEC "d" 137#define KMP_UINT32_SPEC "u" 138#ifndef KMP_STRUCT64 139typedef __int64 kmp_int64; 140typedef unsigned __int64 kmp_uint64; 141#define KMP_INT64_SPEC "I64d" 142#define KMP_UINT64_SPEC "I64u" 143#else 144struct kmp_struct64 { 145 kmp_int32 a, b; 146}; 147typedef struct kmp_struct64 kmp_int64; 148typedef struct kmp_struct64 kmp_uint64; 149/* Not sure what to use for KMP_[U]INT64_SPEC here */ 150#endif 151#if KMP_ARCH_X86 && KMP_MSVC_COMPAT 152#undef KMP_USE_X87CONTROL 153#define KMP_USE_X87CONTROL 1 154#endif 155#if KMP_ARCH_X86_64 || KMP_ARCH_AARCH64 156#define KMP_INTPTR 1 157typedef __int64 kmp_intptr_t; 158typedef unsigned __int64 kmp_uintptr_t; 159#define KMP_INTPTR_SPEC "I64d" 160#define KMP_UINTPTR_SPEC "I64u" 161#endif 162#endif /* KMP_OS_WINDOWS */ 163 164#if KMP_OS_UNIX 165#define KMP_END_OF_LINE "\n" 166typedef char kmp_int8; 167typedef unsigned char kmp_uint8; 168typedef short kmp_int16; 169typedef unsigned short kmp_uint16; 170typedef int kmp_int32; 171typedef unsigned int kmp_uint32; 172typedef long long kmp_int64; 173typedef unsigned long long kmp_uint64; 174#define KMP_INT32_SPEC "d" 175#define KMP_UINT32_SPEC "u" 176#define KMP_INT64_SPEC "lld" 177#define KMP_UINT64_SPEC "llu" 178#endif /* KMP_OS_UNIX */ 179 180#if KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_MIPS || KMP_ARCH_WASM || \ 181 KMP_ARCH_PPC 182#define KMP_SIZE_T_SPEC KMP_UINT32_SPEC 183#elif KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 || \ 184 KMP_ARCH_MIPS64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 || \ 185 KMP_ARCH_VE || KMP_ARCH_S390X 186#define KMP_SIZE_T_SPEC KMP_UINT64_SPEC 187#else 188#error "Can't determine size_t printf format specifier." 189#endif 190 191#if KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_WASM || KMP_ARCH_PPC 192#define KMP_SIZE_T_MAX (0xFFFFFFFF) 193#else 194#define KMP_SIZE_T_MAX (0xFFFFFFFFFFFFFFFF) 195#endif 196 197typedef size_t kmp_size_t; 198typedef float kmp_real32; 199typedef double kmp_real64; 200 201#ifndef KMP_INTPTR 202#define KMP_INTPTR 1 203typedef long kmp_intptr_t; 204typedef unsigned long kmp_uintptr_t; 205#define KMP_INTPTR_SPEC "ld" 206#define KMP_UINTPTR_SPEC "lu" 207#endif 208 209#ifdef BUILD_I8 210typedef kmp_int64 kmp_int; 211typedef kmp_uint64 kmp_uint; 212#else 213typedef kmp_int32 kmp_int; 214typedef kmp_uint32 kmp_uint; 215#endif /* BUILD_I8 */ 216#define KMP_INT_MAX ((kmp_int32)0x7FFFFFFF) 217#define KMP_INT_MIN ((kmp_int32)0x80000000) 218 219// stdarg handling 220#if (KMP_ARCH_ARM || KMP_ARCH_X86_64 || KMP_ARCH_AARCH64 || KMP_ARCH_WASM) && \ 221 (KMP_OS_FREEBSD || KMP_OS_LINUX || KMP_OS_WASI) 222typedef va_list *kmp_va_list; 223#define kmp_va_deref(ap) (*(ap)) 224#define kmp_va_addr_of(ap) (&(ap)) 225#else 226typedef va_list kmp_va_list; 227#define kmp_va_deref(ap) (ap) 228#define kmp_va_addr_of(ap) (ap) 229#endif 230 231#ifdef __cplusplus 232// macros to cast out qualifiers and to re-interpret types 233#define CCAST(type, var) const_cast<type>(var) 234#define RCAST(type, var) reinterpret_cast<type>(var) 235//------------------------------------------------------------------------- 236// template for debug prints specification ( d, u, lld, llu ), and to obtain 237// signed/unsigned flavors of a type 238template <typename T> struct traits_t {}; 239// int 240template <> struct traits_t<signed int> { 241 typedef signed int signed_t; 242 typedef unsigned int unsigned_t; 243 typedef double floating_t; 244 static char const *spec; 245 static const signed_t max_value = 0x7fffffff; 246 static const signed_t min_value = 0x80000000; 247 static const int type_size = sizeof(signed_t); 248}; 249// unsigned int 250template <> struct traits_t<unsigned int> { 251 typedef signed int signed_t; 252 typedef unsigned int unsigned_t; 253 typedef double floating_t; 254 static char const *spec; 255 static const unsigned_t max_value = 0xffffffff; 256 static const unsigned_t min_value = 0x00000000; 257 static const int type_size = sizeof(unsigned_t); 258}; 259// long 260template <> struct traits_t<signed long> { 261 typedef signed long signed_t; 262 typedef unsigned long unsigned_t; 263 typedef long double floating_t; 264 static char const *spec; 265 static const int type_size = sizeof(signed_t); 266}; 267// long long 268template <> struct traits_t<signed long long> { 269 typedef signed long long signed_t; 270 typedef unsigned long long unsigned_t; 271 typedef long double floating_t; 272 static char const *spec; 273 static const signed_t max_value = 0x7fffffffffffffffLL; 274 static const signed_t min_value = 0x8000000000000000LL; 275 static const int type_size = sizeof(signed_t); 276}; 277// unsigned long long 278template <> struct traits_t<unsigned long long> { 279 typedef signed long long signed_t; 280 typedef unsigned long long unsigned_t; 281 typedef long double floating_t; 282 static char const *spec; 283 static const unsigned_t max_value = 0xffffffffffffffffLL; 284 static const unsigned_t min_value = 0x0000000000000000LL; 285 static const int type_size = sizeof(unsigned_t); 286}; 287//------------------------------------------------------------------------- 288#else 289#define CCAST(type, var) (type)(var) 290#define RCAST(type, var) (type)(var) 291#endif // __cplusplus 292 293#define KMP_EXPORT extern /* export declaration in guide libraries */ 294 295#if __GNUC__ >= 4 && !defined(__MINGW32__) 296#define __forceinline __inline 297#endif 298 299/* Check if the OS/arch can support user-level mwait */ 300// All mwait code tests for UMWAIT first, so it should only fall back to ring3 301// MWAIT for KNL. 302#define KMP_HAVE_MWAIT \ 303 ((KMP_ARCH_X86 || KMP_ARCH_X86_64) && (KMP_OS_LINUX || KMP_OS_WINDOWS) && \ 304 !KMP_MIC2) 305#define KMP_HAVE_UMWAIT \ 306 ((KMP_ARCH_X86 || KMP_ARCH_X86_64) && (KMP_OS_LINUX || KMP_OS_WINDOWS) && \ 307 !KMP_MIC) 308 309#if KMP_OS_WINDOWS 310// Don't include everything related to NT status code, we'll do that explicitly 311#define WIN32_NO_STATUS 312#include <windows.h> 313 314static inline int KMP_GET_PAGE_SIZE(void) { 315 SYSTEM_INFO si; 316 GetSystemInfo(&si); 317 return si.dwPageSize; 318} 319#else 320#define KMP_GET_PAGE_SIZE() getpagesize() 321#endif 322 323#define PAGE_ALIGNED(_addr) \ 324 (!((size_t)_addr & (size_t)(KMP_GET_PAGE_SIZE() - 1))) 325#define ALIGN_TO_PAGE(x) \ 326 (void *)(((size_t)(x)) & ~((size_t)(KMP_GET_PAGE_SIZE() - 1))) 327 328/* ---------- Support for cache alignment, padding, etc. ----------------*/ 329 330#ifdef __cplusplus 331extern "C" { 332#endif // __cplusplus 333 334#define INTERNODE_CACHE_LINE 4096 /* for multi-node systems */ 335 336/* Define the default size of the cache line */ 337#ifndef CACHE_LINE 338#define CACHE_LINE 128 /* cache line size in bytes */ 339#else 340#if (CACHE_LINE < 64) && !defined(KMP_OS_DARWIN) 341// 2006-02-13: This produces too many warnings on OS X*. Disable for now 342#warning CACHE_LINE is too small. 343#endif 344#endif /* CACHE_LINE */ 345 346#define KMP_CACHE_PREFETCH(ADDR) /* nothing */ 347 348// Define attribute that indicates that the fall through from the previous 349// case label is intentional and should not be diagnosed by a compiler 350// Code from libcxx/include/__config 351// Use a function like macro to imply that it must be followed by a semicolon 352#if __cplusplus > 201402L && __has_cpp_attribute(fallthrough) 353#define KMP_FALLTHROUGH() [[fallthrough]] 354// icc cannot properly tell this attribute is absent so force off 355#elif KMP_COMPILER_ICC 356#define KMP_FALLTHROUGH() ((void)0) 357#elif __has_cpp_attribute(clang::fallthrough) 358#define KMP_FALLTHROUGH() [[clang::fallthrough]] 359#elif __has_attribute(fallthrough) || __GNUC__ >= 7 360#define KMP_FALLTHROUGH() __attribute__((__fallthrough__)) 361#else 362#define KMP_FALLTHROUGH() ((void)0) 363#endif 364 365#if KMP_HAVE_ATTRIBUTE_WAITPKG 366#define KMP_ATTRIBUTE_TARGET_WAITPKG __attribute__((target("waitpkg"))) 367#else 368#define KMP_ATTRIBUTE_TARGET_WAITPKG /* Nothing */ 369#endif 370 371#if KMP_HAVE_ATTRIBUTE_RTM 372#define KMP_ATTRIBUTE_TARGET_RTM __attribute__((target("rtm"))) 373#else 374#define KMP_ATTRIBUTE_TARGET_RTM /* Nothing */ 375#endif 376 377// Define attribute that indicates a function does not return 378#if __cplusplus >= 201103L 379#define KMP_NORETURN [[noreturn]] 380#elif KMP_OS_WINDOWS 381#define KMP_NORETURN __declspec(noreturn) 382#else 383#define KMP_NORETURN __attribute__((noreturn)) 384#endif 385 386#if KMP_OS_WINDOWS && KMP_MSVC_COMPAT 387#define KMP_ALIGN(bytes) __declspec(align(bytes)) 388#define KMP_THREAD_LOCAL __declspec(thread) 389#define KMP_ALIAS /* Nothing */ 390#else 391#define KMP_ALIGN(bytes) __attribute__((aligned(bytes))) 392#define KMP_THREAD_LOCAL __thread 393#define KMP_ALIAS(alias_of) __attribute__((alias(alias_of))) 394#endif 395 396#if KMP_HAVE_WEAK_ATTRIBUTE && !KMP_DYNAMIC_LIB 397#define KMP_WEAK_ATTRIBUTE_EXTERNAL __attribute__((weak)) 398#else 399#define KMP_WEAK_ATTRIBUTE_EXTERNAL /* Nothing */ 400#endif 401 402#if KMP_HAVE_WEAK_ATTRIBUTE 403#define KMP_WEAK_ATTRIBUTE_INTERNAL __attribute__((weak)) 404#else 405#define KMP_WEAK_ATTRIBUTE_INTERNAL /* Nothing */ 406#endif 407 408// Define KMP_VERSION_SYMBOL and KMP_EXPAND_NAME 409#ifndef KMP_STR 410#define KMP_STR(x) _KMP_STR(x) 411#define _KMP_STR(x) #x 412#endif 413 414#ifdef KMP_USE_VERSION_SYMBOLS 415// If using versioned symbols, KMP_EXPAND_NAME prepends 416// __kmp_api_ to the real API name 417#define KMP_EXPAND_NAME(api_name) _KMP_EXPAND_NAME(api_name) 418#define _KMP_EXPAND_NAME(api_name) __kmp_api_##api_name 419#define KMP_VERSION_SYMBOL(api_name, ver_num, ver_str) \ 420 _KMP_VERSION_SYMBOL(api_name, ver_num, ver_str, "VERSION") 421#define _KMP_VERSION_SYMBOL(api_name, ver_num, ver_str, default_ver) \ 422 __typeof__(__kmp_api_##api_name) __kmp_api_##api_name##_##ver_num##_alias \ 423 __attribute__((alias(KMP_STR(__kmp_api_##api_name)))); \ 424 __asm__( \ 425 ".symver " KMP_STR(__kmp_api_##api_name##_##ver_num##_alias) "," KMP_STR( \ 426 api_name) "@" ver_str "\n\t"); \ 427 __asm__(".symver " KMP_STR(__kmp_api_##api_name) "," KMP_STR( \ 428 api_name) "@@" default_ver "\n\t") 429 430#define KMP_VERSION_OMPC_SYMBOL(apic_name, api_name, ver_num, ver_str) \ 431 _KMP_VERSION_OMPC_SYMBOL(apic_name, api_name, ver_num, ver_str, "VERSION") 432#define _KMP_VERSION_OMPC_SYMBOL(apic_name, api_name, ver_num, ver_str, \ 433 default_ver) \ 434 __typeof__(__kmp_api_##apic_name) __kmp_api_##apic_name##_##ver_num##_alias \ 435 __attribute__((alias(KMP_STR(__kmp_api_##apic_name)))); \ 436 __asm__(".symver " KMP_STR(__kmp_api_##apic_name) "," KMP_STR( \ 437 apic_name) "@@" default_ver "\n\t"); \ 438 __asm__( \ 439 ".symver " KMP_STR(__kmp_api_##apic_name##_##ver_num##_alias) "," KMP_STR( \ 440 api_name) "@" ver_str "\n\t") 441 442#else // KMP_USE_VERSION_SYMBOLS 443#define KMP_EXPAND_NAME(api_name) api_name 444#define KMP_VERSION_SYMBOL(api_name, ver_num, ver_str) /* Nothing */ 445#define KMP_VERSION_OMPC_SYMBOL(apic_name, api_name, ver_num, \ 446 ver_str) /* Nothing */ 447#endif // KMP_USE_VERSION_SYMBOLS 448 449/* Temporary note: if performance testing of this passes, we can remove 450 all references to KMP_DO_ALIGN and replace with KMP_ALIGN. */ 451#define KMP_DO_ALIGN(bytes) KMP_ALIGN(bytes) 452#define KMP_ALIGN_CACHE KMP_ALIGN(CACHE_LINE) 453#define KMP_ALIGN_CACHE_INTERNODE KMP_ALIGN(INTERNODE_CACHE_LINE) 454 455/* General purpose fence types for memory operations */ 456enum kmp_mem_fence_type { 457 kmp_no_fence, /* No memory fence */ 458 kmp_acquire_fence, /* Acquire (read) memory fence */ 459 kmp_release_fence, /* Release (write) memory fence */ 460 kmp_full_fence /* Full (read+write) memory fence */ 461}; 462 463// Synchronization primitives 464 465#if KMP_ASM_INTRINS && KMP_OS_WINDOWS && !((KMP_ARCH_AARCH64 || KMP_ARCH_ARM) && (KMP_COMPILER_CLANG || KMP_COMPILER_GCC)) 466 467#if KMP_MSVC_COMPAT && !KMP_COMPILER_CLANG 468#pragma intrinsic(InterlockedExchangeAdd) 469#pragma intrinsic(InterlockedCompareExchange) 470#pragma intrinsic(InterlockedExchange) 471#if !(KMP_COMPILER_ICX && KMP_32_BIT_ARCH) 472#pragma intrinsic(InterlockedExchange64) 473#endif 474#endif 475 476// Using InterlockedIncrement / InterlockedDecrement causes a library loading 477// ordering problem, so we use InterlockedExchangeAdd instead. 478#define KMP_TEST_THEN_INC32(p) InterlockedExchangeAdd((volatile long *)(p), 1) 479#define KMP_TEST_THEN_INC_ACQ32(p) \ 480 InterlockedExchangeAdd((volatile long *)(p), 1) 481#define KMP_TEST_THEN_ADD4_32(p) InterlockedExchangeAdd((volatile long *)(p), 4) 482#define KMP_TEST_THEN_ADD4_ACQ32(p) \ 483 InterlockedExchangeAdd((volatile long *)(p), 4) 484#define KMP_TEST_THEN_DEC32(p) InterlockedExchangeAdd((volatile long *)(p), -1) 485#define KMP_TEST_THEN_DEC_ACQ32(p) \ 486 InterlockedExchangeAdd((volatile long *)(p), -1) 487#define KMP_TEST_THEN_ADD32(p, v) \ 488 InterlockedExchangeAdd((volatile long *)(p), (v)) 489 490#define KMP_COMPARE_AND_STORE_RET32(p, cv, sv) \ 491 InterlockedCompareExchange((volatile long *)(p), (long)(sv), (long)(cv)) 492 493#define KMP_XCHG_FIXED32(p, v) \ 494 InterlockedExchange((volatile long *)(p), (long)(v)) 495#define KMP_XCHG_FIXED64(p, v) \ 496 InterlockedExchange64((volatile kmp_int64 *)(p), (kmp_int64)(v)) 497 498inline kmp_real32 KMP_XCHG_REAL32(volatile kmp_real32 *p, kmp_real32 v) { 499 kmp_int32 tmp = InterlockedExchange((volatile long *)p, *(long *)&v); 500 return *(kmp_real32 *)&tmp; 501} 502 503#define KMP_TEST_THEN_OR8(p, v) __kmp_test_then_or8((p), (v)) 504#define KMP_TEST_THEN_AND8(p, v) __kmp_test_then_and8((p), (v)) 505#define KMP_TEST_THEN_OR32(p, v) __kmp_test_then_or32((p), (v)) 506#define KMP_TEST_THEN_AND32(p, v) __kmp_test_then_and32((p), (v)) 507#define KMP_TEST_THEN_OR64(p, v) __kmp_test_then_or64((p), (v)) 508#define KMP_TEST_THEN_AND64(p, v) __kmp_test_then_and64((p), (v)) 509 510extern kmp_int8 __kmp_test_then_or8(volatile kmp_int8 *p, kmp_int8 v); 511extern kmp_int8 __kmp_test_then_and8(volatile kmp_int8 *p, kmp_int8 v); 512extern kmp_int32 __kmp_test_then_add32(volatile kmp_int32 *p, kmp_int32 v); 513extern kmp_uint32 __kmp_test_then_or32(volatile kmp_uint32 *p, kmp_uint32 v); 514extern kmp_uint32 __kmp_test_then_and32(volatile kmp_uint32 *p, kmp_uint32 v); 515extern kmp_int64 __kmp_test_then_add64(volatile kmp_int64 *p, kmp_int64 v); 516extern kmp_uint64 __kmp_test_then_or64(volatile kmp_uint64 *p, kmp_uint64 v); 517extern kmp_uint64 __kmp_test_then_and64(volatile kmp_uint64 *p, kmp_uint64 v); 518 519#if KMP_ARCH_AARCH64 && KMP_COMPILER_MSVC && !KMP_COMPILER_CLANG 520#define KMP_TEST_THEN_INC64(p) _InterlockedExchangeAdd64((p), 1LL) 521#define KMP_TEST_THEN_INC_ACQ64(p) _InterlockedExchangeAdd64_acq((p), 1LL) 522#define KMP_TEST_THEN_ADD4_64(p) _InterlockedExchangeAdd64((p), 4LL) 523// #define KMP_TEST_THEN_ADD4_ACQ64(p) _InterlockedExchangeAdd64_acq((p), 4LL) 524// #define KMP_TEST_THEN_DEC64(p) _InterlockedExchangeAdd64((p), -1LL) 525// #define KMP_TEST_THEN_DEC_ACQ64(p) _InterlockedExchangeAdd64_acq((p), -1LL) 526// #define KMP_TEST_THEN_ADD8(p, v) _InterlockedExchangeAdd8((p), (v)) 527#define KMP_TEST_THEN_ADD64(p, v) _InterlockedExchangeAdd64((p), (v)) 528 529#define KMP_COMPARE_AND_STORE_ACQ8(p, cv, sv) \ 530 __kmp_compare_and_store_acq8((p), (cv), (sv)) 531#define KMP_COMPARE_AND_STORE_REL8(p, cv, sv) \ 532 __kmp_compare_and_store_rel8((p), (cv), (sv)) 533#define KMP_COMPARE_AND_STORE_ACQ16(p, cv, sv) \ 534 __kmp_compare_and_store_acq16((p), (cv), (sv)) 535/* 536#define KMP_COMPARE_AND_STORE_REL16(p, cv, sv) \ 537 __kmp_compare_and_store_rel16((p), (cv), (sv)) 538*/ 539#define KMP_COMPARE_AND_STORE_ACQ32(p, cv, sv) \ 540 __kmp_compare_and_store_acq32((volatile kmp_int32 *)(p), (kmp_int32)(cv), \ 541 (kmp_int32)(sv)) 542#define KMP_COMPARE_AND_STORE_REL32(p, cv, sv) \ 543 __kmp_compare_and_store_rel32((volatile kmp_int32 *)(p), (kmp_int32)(cv), \ 544 (kmp_int32)(sv)) 545#define KMP_COMPARE_AND_STORE_ACQ64(p, cv, sv) \ 546 __kmp_compare_and_store_acq64((volatile kmp_int64 *)(p), (kmp_int64)(cv), \ 547 (kmp_int64)(sv)) 548#define KMP_COMPARE_AND_STORE_REL64(p, cv, sv) \ 549 __kmp_compare_and_store_rel64((volatile kmp_int64 *)(p), (kmp_int64)(cv), \ 550 (kmp_int64)(sv)) 551#define KMP_COMPARE_AND_STORE_PTR(p, cv, sv) \ 552 __kmp_compare_and_store_ptr((void *volatile *)(p), (void *)(cv), (void *)(sv)) 553 554// KMP_COMPARE_AND_STORE expects this order: pointer, compare, exchange 555// _InterlockedCompareExchange expects this order: pointer, exchange, compare 556// KMP_COMPARE_AND_STORE also returns a bool indicating a successful write. A 557// write is successful if the return value of _InterlockedCompareExchange is the 558// same as the compare value. 559inline kmp_int8 __kmp_compare_and_store_acq8(volatile kmp_int8 *p, kmp_int8 cv, 560 kmp_int8 sv) { 561 return _InterlockedCompareExchange8_acq(p, sv, cv) == cv; 562} 563 564inline kmp_int8 __kmp_compare_and_store_rel8(volatile kmp_int8 *p, kmp_int8 cv, 565 kmp_int8 sv) { 566 return _InterlockedCompareExchange8_rel(p, sv, cv) == cv; 567} 568 569inline kmp_int16 __kmp_compare_and_store_acq16(volatile kmp_int16 *p, 570 kmp_int16 cv, kmp_int16 sv) { 571 return _InterlockedCompareExchange16_acq(p, sv, cv) == cv; 572} 573 574inline kmp_int16 __kmp_compare_and_store_rel16(volatile kmp_int16 *p, 575 kmp_int16 cv, kmp_int16 sv) { 576 return _InterlockedCompareExchange16_rel(p, sv, cv) == cv; 577} 578 579inline kmp_int32 __kmp_compare_and_store_acq32(volatile kmp_int32 *p, 580 kmp_int32 cv, kmp_int32 sv) { 581 return _InterlockedCompareExchange_acq((volatile long *)p, sv, cv) == cv; 582} 583 584inline kmp_int32 __kmp_compare_and_store_rel32(volatile kmp_int32 *p, 585 kmp_int32 cv, kmp_int32 sv) { 586 return _InterlockedCompareExchange_rel((volatile long *)p, sv, cv) == cv; 587} 588 589inline kmp_int32 __kmp_compare_and_store_acq64(volatile kmp_int64 *p, 590 kmp_int64 cv, kmp_int64 sv) { 591 return _InterlockedCompareExchange64_acq(p, sv, cv) == cv; 592} 593 594inline kmp_int32 __kmp_compare_and_store_rel64(volatile kmp_int64 *p, 595 kmp_int64 cv, kmp_int64 sv) { 596 return _InterlockedCompareExchange64_rel(p, sv, cv) == cv; 597} 598 599inline kmp_int32 __kmp_compare_and_store_ptr(void *volatile *p, void *cv, 600 void *sv) { 601 return _InterlockedCompareExchangePointer(p, sv, cv) == cv; 602} 603 604// The _RET versions return the value instead of a bool 605 606#define KMP_COMPARE_AND_STORE_RET8(p, cv, sv) \ 607 _InterlockedCompareExchange8((p), (sv), (cv)) 608#define KMP_COMPARE_AND_STORE_RET16(p, cv, sv) \ 609 _InterlockedCompareExchange16((p), (sv), (cv)) 610 611#define KMP_COMPARE_AND_STORE_RET64(p, cv, sv) \ 612 _InterlockedCompareExchange64((volatile kmp_int64 *)(p), (kmp_int64)(sv), \ 613 (kmp_int64)(cv)) 614 615 616#define KMP_XCHG_FIXED8(p, v) \ 617 _InterlockedExchange8((volatile kmp_int8 *)(p), (kmp_int8)(v)); 618#define KMP_XCHG_FIXED16(p, v) _InterlockedExchange16((p), (v)); 619#define KMP_XCHG_REAL64(p, v) __kmp_xchg_real64((p), (v)); 620 621inline kmp_real64 __kmp_xchg_real64(volatile kmp_real64 *p, kmp_real64 v) { 622 kmp_int64 tmp = _InterlockedExchange64((volatile kmp_int64 *)p, *(kmp_int64 623 *)&v); return *(kmp_real64 *)&tmp; 624} 625 626#else // !KMP_ARCH_AARCH64 627 628// Routines that we still need to implement in assembly. 629extern kmp_int8 __kmp_test_then_add8(volatile kmp_int8 *p, kmp_int8 v); 630 631extern kmp_int8 __kmp_compare_and_store8(volatile kmp_int8 *p, kmp_int8 cv, 632 kmp_int8 sv); 633extern kmp_int16 __kmp_compare_and_store16(volatile kmp_int16 *p, kmp_int16 cv, 634 kmp_int16 sv); 635extern kmp_int32 __kmp_compare_and_store32(volatile kmp_int32 *p, kmp_int32 cv, 636 kmp_int32 sv); 637extern kmp_int32 __kmp_compare_and_store64(volatile kmp_int64 *p, kmp_int64 cv, 638 kmp_int64 sv); 639extern kmp_int8 __kmp_compare_and_store_ret8(volatile kmp_int8 *p, kmp_int8 cv, 640 kmp_int8 sv); 641extern kmp_int16 __kmp_compare_and_store_ret16(volatile kmp_int16 *p, 642 kmp_int16 cv, kmp_int16 sv); 643extern kmp_int32 __kmp_compare_and_store_ret32(volatile kmp_int32 *p, 644 kmp_int32 cv, kmp_int32 sv); 645extern kmp_int64 __kmp_compare_and_store_ret64(volatile kmp_int64 *p, 646 kmp_int64 cv, kmp_int64 sv); 647 648extern kmp_int8 __kmp_xchg_fixed8(volatile kmp_int8 *p, kmp_int8 v); 649extern kmp_int16 __kmp_xchg_fixed16(volatile kmp_int16 *p, kmp_int16 v); 650extern kmp_int32 __kmp_xchg_fixed32(volatile kmp_int32 *p, kmp_int32 v); 651extern kmp_int64 __kmp_xchg_fixed64(volatile kmp_int64 *p, kmp_int64 v); 652extern kmp_real32 __kmp_xchg_real32(volatile kmp_real32 *p, kmp_real32 v); 653extern kmp_real64 __kmp_xchg_real64(volatile kmp_real64 *p, kmp_real64 v); 654 655//#define KMP_TEST_THEN_INC32(p) __kmp_test_then_add32((p), 1) 656//#define KMP_TEST_THEN_INC_ACQ32(p) __kmp_test_then_add32((p), 1) 657#define KMP_TEST_THEN_INC64(p) __kmp_test_then_add64((p), 1LL) 658#define KMP_TEST_THEN_INC_ACQ64(p) __kmp_test_then_add64((p), 1LL) 659//#define KMP_TEST_THEN_ADD4_32(p) __kmp_test_then_add32((p), 4) 660//#define KMP_TEST_THEN_ADD4_ACQ32(p) __kmp_test_then_add32((p), 4) 661#define KMP_TEST_THEN_ADD4_64(p) __kmp_test_then_add64((p), 4LL) 662#define KMP_TEST_THEN_ADD4_ACQ64(p) __kmp_test_then_add64((p), 4LL) 663//#define KMP_TEST_THEN_DEC32(p) __kmp_test_then_add32((p), -1) 664//#define KMP_TEST_THEN_DEC_ACQ32(p) __kmp_test_then_add32((p), -1) 665#define KMP_TEST_THEN_DEC64(p) __kmp_test_then_add64((p), -1LL) 666#define KMP_TEST_THEN_DEC_ACQ64(p) __kmp_test_then_add64((p), -1LL) 667//#define KMP_TEST_THEN_ADD32(p, v) __kmp_test_then_add32((p), (v)) 668#define KMP_TEST_THEN_ADD8(p, v) __kmp_test_then_add8((p), (v)) 669#define KMP_TEST_THEN_ADD64(p, v) __kmp_test_then_add64((p), (v)) 670 671 672#define KMP_COMPARE_AND_STORE_ACQ8(p, cv, sv) \ 673 __kmp_compare_and_store8((p), (cv), (sv)) 674#define KMP_COMPARE_AND_STORE_REL8(p, cv, sv) \ 675 __kmp_compare_and_store8((p), (cv), (sv)) 676#define KMP_COMPARE_AND_STORE_ACQ16(p, cv, sv) \ 677 __kmp_compare_and_store16((p), (cv), (sv)) 678#define KMP_COMPARE_AND_STORE_REL16(p, cv, sv) \ 679 __kmp_compare_and_store16((p), (cv), (sv)) 680#define KMP_COMPARE_AND_STORE_ACQ32(p, cv, sv) \ 681 __kmp_compare_and_store32((volatile kmp_int32 *)(p), (kmp_int32)(cv), \ 682 (kmp_int32)(sv)) 683#define KMP_COMPARE_AND_STORE_REL32(p, cv, sv) \ 684 __kmp_compare_and_store32((volatile kmp_int32 *)(p), (kmp_int32)(cv), \ 685 (kmp_int32)(sv)) 686#define KMP_COMPARE_AND_STORE_ACQ64(p, cv, sv) \ 687 __kmp_compare_and_store64((volatile kmp_int64 *)(p), (kmp_int64)(cv), \ 688 (kmp_int64)(sv)) 689#define KMP_COMPARE_AND_STORE_REL64(p, cv, sv) \ 690 __kmp_compare_and_store64((volatile kmp_int64 *)(p), (kmp_int64)(cv), \ 691 (kmp_int64)(sv)) 692 693#if KMP_ARCH_X86 694#define KMP_COMPARE_AND_STORE_PTR(p, cv, sv) \ 695 __kmp_compare_and_store32((volatile kmp_int32 *)(p), (kmp_int32)(cv), \ 696 (kmp_int32)(sv)) 697#else /* 64 bit pointers */ 698#define KMP_COMPARE_AND_STORE_PTR(p, cv, sv) \ 699 __kmp_compare_and_store64((volatile kmp_int64 *)(p), (kmp_int64)(cv), \ 700 (kmp_int64)(sv)) 701#endif /* KMP_ARCH_X86 */ 702 703#define KMP_COMPARE_AND_STORE_RET8(p, cv, sv) \ 704 __kmp_compare_and_store_ret8((p), (cv), (sv)) 705#define KMP_COMPARE_AND_STORE_RET16(p, cv, sv) \ 706 __kmp_compare_and_store_ret16((p), (cv), (sv)) 707#define KMP_COMPARE_AND_STORE_RET64(p, cv, sv) \ 708 __kmp_compare_and_store_ret64((volatile kmp_int64 *)(p), (kmp_int64)(cv), \ 709 (kmp_int64)(sv)) 710 711#define KMP_XCHG_FIXED8(p, v) \ 712 __kmp_xchg_fixed8((volatile kmp_int8 *)(p), (kmp_int8)(v)); 713#define KMP_XCHG_FIXED16(p, v) __kmp_xchg_fixed16((p), (v)); 714//#define KMP_XCHG_FIXED32(p, v) __kmp_xchg_fixed32((p), (v)); 715//#define KMP_XCHG_FIXED64(p, v) __kmp_xchg_fixed64((p), (v)); 716//#define KMP_XCHG_REAL32(p, v) __kmp_xchg_real32((p), (v)); 717#define KMP_XCHG_REAL64(p, v) __kmp_xchg_real64((p), (v)); 718#endif 719 720#elif (KMP_ASM_INTRINS && KMP_OS_UNIX) || !(KMP_ARCH_X86 || KMP_ARCH_X86_64) 721 722/* cast p to correct type so that proper intrinsic will be used */ 723#define KMP_TEST_THEN_INC32(p) \ 724 __sync_fetch_and_add((volatile kmp_int32 *)(p), 1) 725#define KMP_TEST_THEN_INC_ACQ32(p) \ 726 __sync_fetch_and_add((volatile kmp_int32 *)(p), 1) 727#if KMP_ARCH_MIPS 728#define KMP_TEST_THEN_INC64(p) \ 729 __atomic_fetch_add((volatile kmp_int64 *)(p), 1LL, __ATOMIC_SEQ_CST) 730#define KMP_TEST_THEN_INC_ACQ64(p) \ 731 __atomic_fetch_add((volatile kmp_int64 *)(p), 1LL, __ATOMIC_SEQ_CST) 732#else 733#define KMP_TEST_THEN_INC64(p) \ 734 __sync_fetch_and_add((volatile kmp_int64 *)(p), 1LL) 735#define KMP_TEST_THEN_INC_ACQ64(p) \ 736 __sync_fetch_and_add((volatile kmp_int64 *)(p), 1LL) 737#endif 738#define KMP_TEST_THEN_ADD4_32(p) \ 739 __sync_fetch_and_add((volatile kmp_int32 *)(p), 4) 740#define KMP_TEST_THEN_ADD4_ACQ32(p) \ 741 __sync_fetch_and_add((volatile kmp_int32 *)(p), 4) 742#if KMP_ARCH_MIPS 743#define KMP_TEST_THEN_ADD4_64(p) \ 744 __atomic_fetch_add((volatile kmp_int64 *)(p), 4LL, __ATOMIC_SEQ_CST) 745#define KMP_TEST_THEN_ADD4_ACQ64(p) \ 746 __atomic_fetch_add((volatile kmp_int64 *)(p), 4LL, __ATOMIC_SEQ_CST) 747#define KMP_TEST_THEN_DEC64(p) \ 748 __atomic_fetch_sub((volatile kmp_int64 *)(p), 1LL, __ATOMIC_SEQ_CST) 749#define KMP_TEST_THEN_DEC_ACQ64(p) \ 750 __atomic_fetch_sub((volatile kmp_int64 *)(p), 1LL, __ATOMIC_SEQ_CST) 751#else 752#define KMP_TEST_THEN_ADD4_64(p) \ 753 __sync_fetch_and_add((volatile kmp_int64 *)(p), 4LL) 754#define KMP_TEST_THEN_ADD4_ACQ64(p) \ 755 __sync_fetch_and_add((volatile kmp_int64 *)(p), 4LL) 756#define KMP_TEST_THEN_DEC64(p) \ 757 __sync_fetch_and_sub((volatile kmp_int64 *)(p), 1LL) 758#define KMP_TEST_THEN_DEC_ACQ64(p) \ 759 __sync_fetch_and_sub((volatile kmp_int64 *)(p), 1LL) 760#endif 761#define KMP_TEST_THEN_DEC32(p) \ 762 __sync_fetch_and_sub((volatile kmp_int32 *)(p), 1) 763#define KMP_TEST_THEN_DEC_ACQ32(p) \ 764 __sync_fetch_and_sub((volatile kmp_int32 *)(p), 1) 765#define KMP_TEST_THEN_ADD8(p, v) \ 766 __sync_fetch_and_add((volatile kmp_int8 *)(p), (kmp_int8)(v)) 767#define KMP_TEST_THEN_ADD32(p, v) \ 768 __sync_fetch_and_add((volatile kmp_int32 *)(p), (kmp_int32)(v)) 769#if KMP_ARCH_MIPS 770#define KMP_TEST_THEN_ADD64(p, v) \ 771 __atomic_fetch_add((volatile kmp_uint64 *)(p), (kmp_uint64)(v), \ 772 __ATOMIC_SEQ_CST) 773#else 774#define KMP_TEST_THEN_ADD64(p, v) \ 775 __sync_fetch_and_add((volatile kmp_int64 *)(p), (kmp_int64)(v)) 776#endif 777 778#define KMP_TEST_THEN_OR8(p, v) \ 779 __sync_fetch_and_or((volatile kmp_int8 *)(p), (kmp_int8)(v)) 780#define KMP_TEST_THEN_AND8(p, v) \ 781 __sync_fetch_and_and((volatile kmp_int8 *)(p), (kmp_int8)(v)) 782#define KMP_TEST_THEN_OR32(p, v) \ 783 __sync_fetch_and_or((volatile kmp_uint32 *)(p), (kmp_uint32)(v)) 784#define KMP_TEST_THEN_AND32(p, v) \ 785 __sync_fetch_and_and((volatile kmp_uint32 *)(p), (kmp_uint32)(v)) 786#if KMP_ARCH_MIPS 787#define KMP_TEST_THEN_OR64(p, v) \ 788 __atomic_fetch_or((volatile kmp_uint64 *)(p), (kmp_uint64)(v), \ 789 __ATOMIC_SEQ_CST) 790#define KMP_TEST_THEN_AND64(p, v) \ 791 __atomic_fetch_and((volatile kmp_uint64 *)(p), (kmp_uint64)(v), \ 792 __ATOMIC_SEQ_CST) 793#else 794#define KMP_TEST_THEN_OR64(p, v) \ 795 __sync_fetch_and_or((volatile kmp_uint64 *)(p), (kmp_uint64)(v)) 796#define KMP_TEST_THEN_AND64(p, v) \ 797 __sync_fetch_and_and((volatile kmp_uint64 *)(p), (kmp_uint64)(v)) 798#endif 799 800#define KMP_COMPARE_AND_STORE_ACQ8(p, cv, sv) \ 801 __sync_bool_compare_and_swap((volatile kmp_uint8 *)(p), (kmp_uint8)(cv), \ 802 (kmp_uint8)(sv)) 803#define KMP_COMPARE_AND_STORE_REL8(p, cv, sv) \ 804 __sync_bool_compare_and_swap((volatile kmp_uint8 *)(p), (kmp_uint8)(cv), \ 805 (kmp_uint8)(sv)) 806#define KMP_COMPARE_AND_STORE_ACQ16(p, cv, sv) \ 807 __sync_bool_compare_and_swap((volatile kmp_uint16 *)(p), (kmp_uint16)(cv), \ 808 (kmp_uint16)(sv)) 809#define KMP_COMPARE_AND_STORE_REL16(p, cv, sv) \ 810 __sync_bool_compare_and_swap((volatile kmp_uint16 *)(p), (kmp_uint16)(cv), \ 811 (kmp_uint16)(sv)) 812#define KMP_COMPARE_AND_STORE_ACQ32(p, cv, sv) \ 813 __sync_bool_compare_and_swap((volatile kmp_uint32 *)(p), (kmp_uint32)(cv), \ 814 (kmp_uint32)(sv)) 815#define KMP_COMPARE_AND_STORE_REL32(p, cv, sv) \ 816 __sync_bool_compare_and_swap((volatile kmp_uint32 *)(p), (kmp_uint32)(cv), \ 817 (kmp_uint32)(sv)) 818#define KMP_COMPARE_AND_STORE_PTR(p, cv, sv) \ 819 __sync_bool_compare_and_swap((void *volatile *)(p), (void *)(cv), \ 820 (void *)(sv)) 821 822#define KMP_COMPARE_AND_STORE_RET8(p, cv, sv) \ 823 __sync_val_compare_and_swap((volatile kmp_uint8 *)(p), (kmp_uint8)(cv), \ 824 (kmp_uint8)(sv)) 825#define KMP_COMPARE_AND_STORE_RET16(p, cv, sv) \ 826 __sync_val_compare_and_swap((volatile kmp_uint16 *)(p), (kmp_uint16)(cv), \ 827 (kmp_uint16)(sv)) 828#define KMP_COMPARE_AND_STORE_RET32(p, cv, sv) \ 829 __sync_val_compare_and_swap((volatile kmp_uint32 *)(p), (kmp_uint32)(cv), \ 830 (kmp_uint32)(sv)) 831#if KMP_ARCH_MIPS 832static inline bool mips_sync_bool_compare_and_swap(volatile kmp_uint64 *p, 833 kmp_uint64 cv, 834 kmp_uint64 sv) { 835 return __atomic_compare_exchange(p, &cv, &sv, false, __ATOMIC_SEQ_CST, 836 __ATOMIC_SEQ_CST); 837} 838static inline bool mips_sync_val_compare_and_swap(volatile kmp_uint64 *p, 839 kmp_uint64 cv, 840 kmp_uint64 sv) { 841 __atomic_compare_exchange(p, &cv, &sv, false, __ATOMIC_SEQ_CST, 842 __ATOMIC_SEQ_CST); 843 return cv; 844} 845#define KMP_COMPARE_AND_STORE_ACQ64(p, cv, sv) \ 846 mips_sync_bool_compare_and_swap((volatile kmp_uint64 *)(p), \ 847 (kmp_uint64)(cv), (kmp_uint64)(sv)) 848#define KMP_COMPARE_AND_STORE_REL64(p, cv, sv) \ 849 mips_sync_bool_compare_and_swap((volatile kmp_uint64 *)(p), \ 850 (kmp_uint64)(cv), (kmp_uint64)(sv)) 851#define KMP_COMPARE_AND_STORE_RET64(p, cv, sv) \ 852 mips_sync_val_compare_and_swap((volatile kmp_uint64 *)(p), (kmp_uint64)(cv), \ 853 (kmp_uint64)(sv)) 854#else 855#define KMP_COMPARE_AND_STORE_ACQ64(p, cv, sv) \ 856 __sync_bool_compare_and_swap((volatile kmp_uint64 *)(p), (kmp_uint64)(cv), \ 857 (kmp_uint64)(sv)) 858#define KMP_COMPARE_AND_STORE_REL64(p, cv, sv) \ 859 __sync_bool_compare_and_swap((volatile kmp_uint64 *)(p), (kmp_uint64)(cv), \ 860 (kmp_uint64)(sv)) 861#define KMP_COMPARE_AND_STORE_RET64(p, cv, sv) \ 862 __sync_val_compare_and_swap((volatile kmp_uint64 *)(p), (kmp_uint64)(cv), \ 863 (kmp_uint64)(sv)) 864#endif 865 866#if KMP_OS_DARWIN && defined(__INTEL_COMPILER) && __INTEL_COMPILER >= 1800 867#define KMP_XCHG_FIXED8(p, v) \ 868 __atomic_exchange_1((volatile kmp_uint8 *)(p), (kmp_uint8)(v), \ 869 __ATOMIC_SEQ_CST) 870#else 871#define KMP_XCHG_FIXED8(p, v) \ 872 __sync_lock_test_and_set((volatile kmp_uint8 *)(p), (kmp_uint8)(v)) 873#endif 874#define KMP_XCHG_FIXED16(p, v) \ 875 __sync_lock_test_and_set((volatile kmp_uint16 *)(p), (kmp_uint16)(v)) 876#define KMP_XCHG_FIXED32(p, v) \ 877 __sync_lock_test_and_set((volatile kmp_uint32 *)(p), (kmp_uint32)(v)) 878#define KMP_XCHG_FIXED64(p, v) \ 879 __sync_lock_test_and_set((volatile kmp_uint64 *)(p), (kmp_uint64)(v)) 880 881inline kmp_real32 KMP_XCHG_REAL32(volatile kmp_real32 *p, kmp_real32 v) { 882 volatile kmp_uint32 *up; 883 kmp_uint32 uv; 884 memcpy(&up, &p, sizeof(up)); 885 memcpy(&uv, &v, sizeof(uv)); 886 kmp_int32 tmp = __sync_lock_test_and_set(up, uv); 887 kmp_real32 ftmp; 888 memcpy(&ftmp, &tmp, sizeof(tmp)); 889 return ftmp; 890} 891 892inline kmp_real64 KMP_XCHG_REAL64(volatile kmp_real64 *p, kmp_real64 v) { 893 volatile kmp_uint64 *up; 894 kmp_uint64 uv; 895 memcpy(&up, &p, sizeof(up)); 896 memcpy(&uv, &v, sizeof(uv)); 897 kmp_int64 tmp = __sync_lock_test_and_set(up, uv); 898 kmp_real64 dtmp; 899 memcpy(&dtmp, &tmp, sizeof(tmp)); 900 return dtmp; 901} 902 903#else 904 905extern kmp_int8 __kmp_test_then_add8(volatile kmp_int8 *p, kmp_int8 v); 906extern kmp_int8 __kmp_test_then_or8(volatile kmp_int8 *p, kmp_int8 v); 907extern kmp_int8 __kmp_test_then_and8(volatile kmp_int8 *p, kmp_int8 v); 908extern kmp_int32 __kmp_test_then_add32(volatile kmp_int32 *p, kmp_int32 v); 909extern kmp_uint32 __kmp_test_then_or32(volatile kmp_uint32 *p, kmp_uint32 v); 910extern kmp_uint32 __kmp_test_then_and32(volatile kmp_uint32 *p, kmp_uint32 v); 911extern kmp_int64 __kmp_test_then_add64(volatile kmp_int64 *p, kmp_int64 v); 912extern kmp_uint64 __kmp_test_then_or64(volatile kmp_uint64 *p, kmp_uint64 v); 913extern kmp_uint64 __kmp_test_then_and64(volatile kmp_uint64 *p, kmp_uint64 v); 914 915extern kmp_int8 __kmp_compare_and_store8(volatile kmp_int8 *p, kmp_int8 cv, 916 kmp_int8 sv); 917extern kmp_int16 __kmp_compare_and_store16(volatile kmp_int16 *p, kmp_int16 cv, 918 kmp_int16 sv); 919extern kmp_int32 __kmp_compare_and_store32(volatile kmp_int32 *p, kmp_int32 cv, 920 kmp_int32 sv); 921extern kmp_int32 __kmp_compare_and_store64(volatile kmp_int64 *p, kmp_int64 cv, 922 kmp_int64 sv); 923extern kmp_int8 __kmp_compare_and_store_ret8(volatile kmp_int8 *p, kmp_int8 cv, 924 kmp_int8 sv); 925extern kmp_int16 __kmp_compare_and_store_ret16(volatile kmp_int16 *p, 926 kmp_int16 cv, kmp_int16 sv); 927extern kmp_int32 __kmp_compare_and_store_ret32(volatile kmp_int32 *p, 928 kmp_int32 cv, kmp_int32 sv); 929extern kmp_int64 __kmp_compare_and_store_ret64(volatile kmp_int64 *p, 930 kmp_int64 cv, kmp_int64 sv); 931 932extern kmp_int8 __kmp_xchg_fixed8(volatile kmp_int8 *p, kmp_int8 v); 933extern kmp_int16 __kmp_xchg_fixed16(volatile kmp_int16 *p, kmp_int16 v); 934extern kmp_int32 __kmp_xchg_fixed32(volatile kmp_int32 *p, kmp_int32 v); 935extern kmp_int64 __kmp_xchg_fixed64(volatile kmp_int64 *p, kmp_int64 v); 936extern kmp_real32 __kmp_xchg_real32(volatile kmp_real32 *p, kmp_real32 v); 937extern kmp_real64 __kmp_xchg_real64(volatile kmp_real64 *p, kmp_real64 v); 938 939#define KMP_TEST_THEN_INC32(p) \ 940 __kmp_test_then_add32((volatile kmp_int32 *)(p), 1) 941#define KMP_TEST_THEN_INC_ACQ32(p) \ 942 __kmp_test_then_add32((volatile kmp_int32 *)(p), 1) 943#define KMP_TEST_THEN_INC64(p) \ 944 __kmp_test_then_add64((volatile kmp_int64 *)(p), 1LL) 945#define KMP_TEST_THEN_INC_ACQ64(p) \ 946 __kmp_test_then_add64((volatile kmp_int64 *)(p), 1LL) 947#define KMP_TEST_THEN_ADD4_32(p) \ 948 __kmp_test_then_add32((volatile kmp_int32 *)(p), 4) 949#define KMP_TEST_THEN_ADD4_ACQ32(p) \ 950 __kmp_test_then_add32((volatile kmp_int32 *)(p), 4) 951#define KMP_TEST_THEN_ADD4_64(p) \ 952 __kmp_test_then_add64((volatile kmp_int64 *)(p), 4LL) 953#define KMP_TEST_THEN_ADD4_ACQ64(p) \ 954 __kmp_test_then_add64((volatile kmp_int64 *)(p), 4LL) 955#define KMP_TEST_THEN_DEC32(p) \ 956 __kmp_test_then_add32((volatile kmp_int32 *)(p), -1) 957#define KMP_TEST_THEN_DEC_ACQ32(p) \ 958 __kmp_test_then_add32((volatile kmp_int32 *)(p), -1) 959#define KMP_TEST_THEN_DEC64(p) \ 960 __kmp_test_then_add64((volatile kmp_int64 *)(p), -1LL) 961#define KMP_TEST_THEN_DEC_ACQ64(p) \ 962 __kmp_test_then_add64((volatile kmp_int64 *)(p), -1LL) 963#define KMP_TEST_THEN_ADD8(p, v) \ 964 __kmp_test_then_add8((volatile kmp_int8 *)(p), (kmp_int8)(v)) 965#define KMP_TEST_THEN_ADD32(p, v) \ 966 __kmp_test_then_add32((volatile kmp_int32 *)(p), (kmp_int32)(v)) 967#define KMP_TEST_THEN_ADD64(p, v) \ 968 __kmp_test_then_add64((volatile kmp_int64 *)(p), (kmp_int64)(v)) 969 970#define KMP_TEST_THEN_OR8(p, v) \ 971 __kmp_test_then_or8((volatile kmp_int8 *)(p), (kmp_int8)(v)) 972#define KMP_TEST_THEN_AND8(p, v) \ 973 __kmp_test_then_and8((volatile kmp_int8 *)(p), (kmp_int8)(v)) 974#define KMP_TEST_THEN_OR32(p, v) \ 975 __kmp_test_then_or32((volatile kmp_uint32 *)(p), (kmp_uint32)(v)) 976#define KMP_TEST_THEN_AND32(p, v) \ 977 __kmp_test_then_and32((volatile kmp_uint32 *)(p), (kmp_uint32)(v)) 978#define KMP_TEST_THEN_OR64(p, v) \ 979 __kmp_test_then_or64((volatile kmp_uint64 *)(p), (kmp_uint64)(v)) 980#define KMP_TEST_THEN_AND64(p, v) \ 981 __kmp_test_then_and64((volatile kmp_uint64 *)(p), (kmp_uint64)(v)) 982 983#define KMP_COMPARE_AND_STORE_ACQ8(p, cv, sv) \ 984 __kmp_compare_and_store8((volatile kmp_int8 *)(p), (kmp_int8)(cv), \ 985 (kmp_int8)(sv)) 986#define KMP_COMPARE_AND_STORE_REL8(p, cv, sv) \ 987 __kmp_compare_and_store8((volatile kmp_int8 *)(p), (kmp_int8)(cv), \ 988 (kmp_int8)(sv)) 989#define KMP_COMPARE_AND_STORE_ACQ16(p, cv, sv) \ 990 __kmp_compare_and_store16((volatile kmp_int16 *)(p), (kmp_int16)(cv), \ 991 (kmp_int16)(sv)) 992#define KMP_COMPARE_AND_STORE_REL16(p, cv, sv) \ 993 __kmp_compare_and_store16((volatile kmp_int16 *)(p), (kmp_int16)(cv), \ 994 (kmp_int16)(sv)) 995#define KMP_COMPARE_AND_STORE_ACQ32(p, cv, sv) \ 996 __kmp_compare_and_store32((volatile kmp_int32 *)(p), (kmp_int32)(cv), \ 997 (kmp_int32)(sv)) 998#define KMP_COMPARE_AND_STORE_REL32(p, cv, sv) \ 999 __kmp_compare_and_store32((volatile kmp_int32 *)(p), (kmp_int32)(cv), \ 1000 (kmp_int32)(sv)) 1001#define KMP_COMPARE_AND_STORE_ACQ64(p, cv, sv) \ 1002 __kmp_compare_and_store64((volatile kmp_int64 *)(p), (kmp_int64)(cv), \ 1003 (kmp_int64)(sv)) 1004#define KMP_COMPARE_AND_STORE_REL64(p, cv, sv) \ 1005 __kmp_compare_and_store64((volatile kmp_int64 *)(p), (kmp_int64)(cv), \ 1006 (kmp_int64)(sv)) 1007 1008#if KMP_ARCH_X86 1009#define KMP_COMPARE_AND_STORE_PTR(p, cv, sv) \ 1010 __kmp_compare_and_store32((volatile kmp_int32 *)(p), (kmp_int32)(cv), \ 1011 (kmp_int32)(sv)) 1012#else /* 64 bit pointers */ 1013#define KMP_COMPARE_AND_STORE_PTR(p, cv, sv) \ 1014 __kmp_compare_and_store64((volatile kmp_int64 *)(p), (kmp_int64)(cv), \ 1015 (kmp_int64)(sv)) 1016#endif /* KMP_ARCH_X86 */ 1017 1018#define KMP_COMPARE_AND_STORE_RET8(p, cv, sv) \ 1019 __kmp_compare_and_store_ret8((p), (cv), (sv)) 1020#define KMP_COMPARE_AND_STORE_RET16(p, cv, sv) \ 1021 __kmp_compare_and_store_ret16((p), (cv), (sv)) 1022#define KMP_COMPARE_AND_STORE_RET32(p, cv, sv) \ 1023 __kmp_compare_and_store_ret32((volatile kmp_int32 *)(p), (kmp_int32)(cv), \ 1024 (kmp_int32)(sv)) 1025#define KMP_COMPARE_AND_STORE_RET64(p, cv, sv) \ 1026 __kmp_compare_and_store_ret64((volatile kmp_int64 *)(p), (kmp_int64)(cv), \ 1027 (kmp_int64)(sv)) 1028 1029#define KMP_XCHG_FIXED8(p, v) \ 1030 __kmp_xchg_fixed8((volatile kmp_int8 *)(p), (kmp_int8)(v)); 1031#define KMP_XCHG_FIXED16(p, v) __kmp_xchg_fixed16((p), (v)); 1032#define KMP_XCHG_FIXED32(p, v) __kmp_xchg_fixed32((p), (v)); 1033#define KMP_XCHG_FIXED64(p, v) __kmp_xchg_fixed64((p), (v)); 1034#define KMP_XCHG_REAL32(p, v) __kmp_xchg_real32((p), (v)); 1035#define KMP_XCHG_REAL64(p, v) __kmp_xchg_real64((p), (v)); 1036 1037#endif /* KMP_ASM_INTRINS */ 1038 1039/* ------------- relaxed consistency memory model stuff ------------------ */ 1040 1041#if KMP_OS_WINDOWS 1042#ifdef __ABSOFT_WIN 1043#define KMP_MB() asm("nop") 1044#define KMP_IMB() asm("nop") 1045#else 1046#define KMP_MB() /* _asm{ nop } */ 1047#define KMP_IMB() /* _asm{ nop } */ 1048#endif 1049#endif /* KMP_OS_WINDOWS */ 1050 1051#if KMP_ARCH_PPC64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS || \ 1052 KMP_ARCH_MIPS64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 || \ 1053 KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_PPC 1054#if KMP_OS_WINDOWS 1055#undef KMP_MB 1056#define KMP_MB() std::atomic_thread_fence(std::memory_order_seq_cst) 1057#else /* !KMP_OS_WINDOWS */ 1058#define KMP_MB() __sync_synchronize() 1059#endif 1060#endif 1061 1062#ifndef KMP_MB 1063#define KMP_MB() /* nothing to do */ 1064#endif 1065 1066#if KMP_ARCH_X86 || KMP_ARCH_X86_64 1067#if KMP_MIC 1068// fence-style instructions do not exist, but lock; xaddl $0,(%rsp) can be used. 1069// We shouldn't need it, though, since the ABI rules require that 1070// * If the compiler generates NGO stores it also generates the fence 1071// * If users hand-code NGO stores they should insert the fence 1072// therefore no incomplete unordered stores should be visible. 1073#define KMP_MFENCE() /* Nothing */ 1074#define KMP_SFENCE() /* Nothing */ 1075#else 1076#if KMP_COMPILER_ICC || KMP_COMPILER_ICX 1077#define KMP_MFENCE_() _mm_mfence() 1078#define KMP_SFENCE_() _mm_sfence() 1079#elif KMP_COMPILER_MSVC 1080#define KMP_MFENCE_() MemoryBarrier() 1081#define KMP_SFENCE_() MemoryBarrier() 1082#else 1083#define KMP_MFENCE_() __sync_synchronize() 1084#define KMP_SFENCE_() __sync_synchronize() 1085#endif 1086#define KMP_MFENCE() \ 1087 if (UNLIKELY(!__kmp_cpuinfo.initialized)) { \ 1088 __kmp_query_cpuid(&__kmp_cpuinfo); \ 1089 } \ 1090 if (__kmp_cpuinfo.flags.sse2) { \ 1091 KMP_MFENCE_(); \ 1092 } 1093#define KMP_SFENCE() KMP_SFENCE_() 1094#endif 1095#else 1096#define KMP_MFENCE() KMP_MB() 1097#define KMP_SFENCE() KMP_MB() 1098#endif 1099 1100#ifndef KMP_IMB 1101#define KMP_IMB() /* nothing to do */ 1102#endif 1103 1104#ifndef KMP_ST_REL32 1105#define KMP_ST_REL32(A, D) (*(A) = (D)) 1106#endif 1107 1108#ifndef KMP_ST_REL64 1109#define KMP_ST_REL64(A, D) (*(A) = (D)) 1110#endif 1111 1112#ifndef KMP_LD_ACQ32 1113#define KMP_LD_ACQ32(A) (*(A)) 1114#endif 1115 1116#ifndef KMP_LD_ACQ64 1117#define KMP_LD_ACQ64(A) (*(A)) 1118#endif 1119 1120/* ------------------------------------------------------------------------ */ 1121// FIXME - maybe this should this be 1122// 1123// #define TCR_4(a) (*(volatile kmp_int32 *)(&a)) 1124// #define TCW_4(a,b) (a) = (*(volatile kmp_int32 *)&(b)) 1125// 1126// #define TCR_8(a) (*(volatile kmp_int64 *)(a)) 1127// #define TCW_8(a,b) (a) = (*(volatile kmp_int64 *)(&b)) 1128// 1129// I'm fairly certain this is the correct thing to do, but I'm afraid 1130// of performance regressions. 1131 1132#define TCR_1(a) (a) 1133#define TCW_1(a, b) (a) = (b) 1134#define TCR_4(a) (a) 1135#define TCW_4(a, b) (a) = (b) 1136#define TCI_4(a) (++(a)) 1137#define TCD_4(a) (--(a)) 1138#define TCR_8(a) (a) 1139#define TCW_8(a, b) (a) = (b) 1140#define TCI_8(a) (++(a)) 1141#define TCD_8(a) (--(a)) 1142#define TCR_SYNC_4(a) (a) 1143#define TCW_SYNC_4(a, b) (a) = (b) 1144#define TCX_SYNC_4(a, b, c) \ 1145 KMP_COMPARE_AND_STORE_REL32((volatile kmp_int32 *)(volatile void *)&(a), \ 1146 (kmp_int32)(b), (kmp_int32)(c)) 1147#define TCR_SYNC_8(a) (a) 1148#define TCW_SYNC_8(a, b) (a) = (b) 1149#define TCX_SYNC_8(a, b, c) \ 1150 KMP_COMPARE_AND_STORE_REL64((volatile kmp_int64 *)(volatile void *)&(a), \ 1151 (kmp_int64)(b), (kmp_int64)(c)) 1152 1153#if KMP_ARCH_X86 || KMP_ARCH_MIPS || KMP_ARCH_WASM || KMP_ARCH_PPC 1154// What about ARM? 1155#define TCR_PTR(a) ((void *)TCR_4(a)) 1156#define TCW_PTR(a, b) TCW_4((a), (b)) 1157#define TCR_SYNC_PTR(a) ((void *)TCR_SYNC_4(a)) 1158#define TCW_SYNC_PTR(a, b) TCW_SYNC_4((a), (b)) 1159#define TCX_SYNC_PTR(a, b, c) ((void *)TCX_SYNC_4((a), (b), (c))) 1160 1161#else /* 64 bit pointers */ 1162 1163#define TCR_PTR(a) ((void *)TCR_8(a)) 1164#define TCW_PTR(a, b) TCW_8((a), (b)) 1165#define TCR_SYNC_PTR(a) ((void *)TCR_SYNC_8(a)) 1166#define TCW_SYNC_PTR(a, b) TCW_SYNC_8((a), (b)) 1167#define TCX_SYNC_PTR(a, b, c) ((void *)TCX_SYNC_8((a), (b), (c))) 1168 1169#endif /* KMP_ARCH_X86 */ 1170 1171/* If these FTN_{TRUE,FALSE} values change, may need to change several places 1172 where they are used to check that language is Fortran, not C. */ 1173 1174#ifndef FTN_TRUE 1175#define FTN_TRUE TRUE 1176#endif 1177 1178#ifndef FTN_FALSE 1179#define FTN_FALSE FALSE 1180#endif 1181 1182typedef void (*microtask_t)(int *gtid, int *npr, ...); 1183 1184#ifdef USE_VOLATILE_CAST 1185#define VOLATILE_CAST(x) (volatile x) 1186#else 1187#define VOLATILE_CAST(x) (x) 1188#endif 1189 1190#define KMP_WAIT __kmp_wait_4 1191#define KMP_WAIT_PTR __kmp_wait_4_ptr 1192#define KMP_EQ __kmp_eq_4 1193#define KMP_NEQ __kmp_neq_4 1194#define KMP_LT __kmp_lt_4 1195#define KMP_GE __kmp_ge_4 1196#define KMP_LE __kmp_le_4 1197 1198/* Workaround for Intel(R) 64 code gen bug when taking address of static array 1199 * (Intel(R) 64 Tracker #138) */ 1200#if (KMP_ARCH_X86_64 || KMP_ARCH_PPC64) && KMP_OS_LINUX 1201#define STATIC_EFI2_WORKAROUND 1202#else 1203#define STATIC_EFI2_WORKAROUND static 1204#endif 1205 1206// Support of BGET usage 1207#ifndef KMP_USE_BGET 1208#define KMP_USE_BGET 1 1209#endif 1210 1211// Switches for OSS builds 1212#ifndef USE_CMPXCHG_FIX 1213#define USE_CMPXCHG_FIX 1 1214#endif 1215 1216// Enable dynamic user lock 1217#define KMP_USE_DYNAMIC_LOCK 1 1218 1219// Enable Intel(R) Transactional Synchronization Extensions (Intel(R) TSX) if 1220// dynamic user lock is turned on 1221#if KMP_USE_DYNAMIC_LOCK 1222// Visual studio can't handle the asm sections in this code 1223#define KMP_USE_TSX (KMP_ARCH_X86 || KMP_ARCH_X86_64) && !KMP_COMPILER_MSVC 1224#ifdef KMP_USE_ADAPTIVE_LOCKS 1225#undef KMP_USE_ADAPTIVE_LOCKS 1226#endif 1227#define KMP_USE_ADAPTIVE_LOCKS KMP_USE_TSX 1228#endif 1229 1230// Enable tick time conversion of ticks to seconds 1231#if KMP_STATS_ENABLED 1232#define KMP_HAVE_TICK_TIME \ 1233 (KMP_OS_LINUX && (KMP_MIC || KMP_ARCH_X86 || KMP_ARCH_X86_64)) 1234#endif 1235 1236// Warning levels 1237enum kmp_warnings_level { 1238 kmp_warnings_off = 0, /* No warnings */ 1239 kmp_warnings_low, /* Minimal warnings (default) */ 1240 kmp_warnings_explicit = 6, /* Explicitly set to ON - more warnings */ 1241 kmp_warnings_verbose /* reserved */ 1242}; 1243 1244#ifdef __cplusplus 1245} // extern "C" 1246#endif // __cplusplus 1247 1248// Safe C API 1249#include "kmp_safe_c_api.h" 1250 1251// Macros for C++11 atomic functions 1252#define KMP_ATOMIC_LD(p, order) (p)->load(std::memory_order_##order) 1253#define KMP_ATOMIC_OP(op, p, v, order) (p)->op(v, std::memory_order_##order) 1254 1255// For non-default load/store 1256#define KMP_ATOMIC_LD_ACQ(p) KMP_ATOMIC_LD(p, acquire) 1257#define KMP_ATOMIC_LD_RLX(p) KMP_ATOMIC_LD(p, relaxed) 1258#define KMP_ATOMIC_ST_REL(p, v) KMP_ATOMIC_OP(store, p, v, release) 1259#define KMP_ATOMIC_ST_RLX(p, v) KMP_ATOMIC_OP(store, p, v, relaxed) 1260 1261// For non-default fetch_<op> 1262#define KMP_ATOMIC_ADD(p, v) KMP_ATOMIC_OP(fetch_add, p, v, acq_rel) 1263#define KMP_ATOMIC_SUB(p, v) KMP_ATOMIC_OP(fetch_sub, p, v, acq_rel) 1264#define KMP_ATOMIC_AND(p, v) KMP_ATOMIC_OP(fetch_and, p, v, acq_rel) 1265#define KMP_ATOMIC_OR(p, v) KMP_ATOMIC_OP(fetch_or, p, v, acq_rel) 1266#define KMP_ATOMIC_INC(p) KMP_ATOMIC_OP(fetch_add, p, 1, acq_rel) 1267#define KMP_ATOMIC_DEC(p) KMP_ATOMIC_OP(fetch_sub, p, 1, acq_rel) 1268#define KMP_ATOMIC_ADD_RLX(p, v) KMP_ATOMIC_OP(fetch_add, p, v, relaxed) 1269#define KMP_ATOMIC_INC_RLX(p) KMP_ATOMIC_OP(fetch_add, p, 1, relaxed) 1270 1271// Callers of the following functions cannot see the side effect on "expected". 1272template <typename T> 1273bool __kmp_atomic_compare_store(std::atomic<T> *p, T expected, T desired) { 1274 return p->compare_exchange_strong( 1275 expected, desired, std::memory_order_acq_rel, std::memory_order_relaxed); 1276} 1277 1278template <typename T> 1279bool __kmp_atomic_compare_store_acq(std::atomic<T> *p, T expected, T desired) { 1280 return p->compare_exchange_strong( 1281 expected, desired, std::memory_order_acquire, std::memory_order_relaxed); 1282} 1283 1284template <typename T> 1285bool __kmp_atomic_compare_store_rel(std::atomic<T> *p, T expected, T desired) { 1286 return p->compare_exchange_strong( 1287 expected, desired, std::memory_order_release, std::memory_order_relaxed); 1288} 1289 1290// Symbol lookup on Linux/Windows 1291#if KMP_OS_WINDOWS 1292extern void *__kmp_lookup_symbol(const char *name, bool next = false); 1293#define KMP_DLSYM(name) __kmp_lookup_symbol(name) 1294#define KMP_DLSYM_NEXT(name) __kmp_lookup_symbol(name, true) 1295#elif KMP_OS_WASI 1296#define KMP_DLSYM(name) nullptr 1297#define KMP_DLSYM_NEXT(name) nullptr 1298#else 1299#define KMP_DLSYM(name) dlsym(RTLD_DEFAULT, name) 1300#define KMP_DLSYM_NEXT(name) dlsym(RTLD_NEXT, name) 1301#endif 1302 1303// MSVC doesn't have this, but clang/clang-cl does. 1304#ifndef __has_builtin 1305#define __has_builtin(x) 0 1306#endif 1307 1308// Same as LLVM_BUILTIN_UNREACHABLE. States that it is UB to reach this point. 1309#if __has_builtin(__builtin_unreachable) || defined(__GNUC__) 1310#define KMP_BUILTIN_UNREACHABLE __builtin_unreachable() 1311#elif defined(_MSC_VER) 1312#define KMP_BUILTIN_UNREACHABLE __assume(false) 1313#else 1314#define KMP_BUILTIN_UNREACHABLE 1315#endif 1316 1317#endif /* KMP_OS_H */ 1318