1234370Sjasone/******************************************************************************/ 2234370Sjasone#ifdef JEMALLOC_H_TYPES 3234370Sjasone 4234370Sjasone#endif /* JEMALLOC_H_TYPES */ 5234370Sjasone/******************************************************************************/ 6234370Sjasone#ifdef JEMALLOC_H_STRUCTS 7234370Sjasone 8234370Sjasone#endif /* JEMALLOC_H_STRUCTS */ 9234370Sjasone/******************************************************************************/ 10234370Sjasone#ifdef JEMALLOC_H_EXTERNS 11234370Sjasone 12234370Sjasone#define atomic_read_uint64(p) atomic_add_uint64(p, 0) 13234370Sjasone#define atomic_read_uint32(p) atomic_add_uint32(p, 0) 14234370Sjasone#define atomic_read_z(p) atomic_add_z(p, 0) 15234370Sjasone#define atomic_read_u(p) atomic_add_u(p, 0) 16234370Sjasone 17234370Sjasone#endif /* JEMALLOC_H_EXTERNS */ 18234370Sjasone/******************************************************************************/ 19234370Sjasone#ifdef JEMALLOC_H_INLINES 20234370Sjasone 21234370Sjasone#ifndef JEMALLOC_ENABLE_INLINE 22234370Sjasoneuint64_t atomic_add_uint64(uint64_t *p, uint64_t x); 23234370Sjasoneuint64_t atomic_sub_uint64(uint64_t *p, uint64_t x); 24234370Sjasoneuint32_t atomic_add_uint32(uint32_t *p, uint32_t x); 25234370Sjasoneuint32_t atomic_sub_uint32(uint32_t *p, uint32_t x); 26234370Sjasonesize_t atomic_add_z(size_t *p, size_t x); 27234370Sjasonesize_t atomic_sub_z(size_t *p, size_t x); 28234370Sjasoneunsigned atomic_add_u(unsigned *p, unsigned x); 29234370Sjasoneunsigned atomic_sub_u(unsigned *p, unsigned x); 30234370Sjasone#endif 31234370Sjasone 32234370Sjasone#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ATOMIC_C_)) 33234370Sjasone/******************************************************************************/ 34234370Sjasone/* 64-bit operations. */ 35234402Sjasone#if (LG_SIZEOF_PTR == 3 || LG_SIZEOF_INT == 3) 36234402Sjasone# ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 37234370SjasoneJEMALLOC_INLINE uint64_t 38234370Sjasoneatomic_add_uint64(uint64_t *p, uint64_t x) 39234370Sjasone{ 40234370Sjasone 41234370Sjasone return (__sync_add_and_fetch(p, x)); 42234370Sjasone} 43234370Sjasone 44234370SjasoneJEMALLOC_INLINE uint64_t 45234370Sjasoneatomic_sub_uint64(uint64_t *p, uint64_t x) 46234370Sjasone{ 47234370Sjasone 48234370Sjasone return (__sync_sub_and_fetch(p, x)); 49234370Sjasone} 50235238Sjasone#elif (defined(_MSC_VER)) 51235238SjasoneJEMALLOC_INLINE uint64_t 52235238Sjasoneatomic_add_uint64(uint64_t *p, uint64_t x) 53235238Sjasone{ 54235238Sjasone 55235238Sjasone return (InterlockedExchangeAdd64(p, x)); 56235238Sjasone} 57235238Sjasone 58235238SjasoneJEMALLOC_INLINE uint64_t 59235238Sjasoneatomic_sub_uint64(uint64_t *p, uint64_t x) 60235238Sjasone{ 61235238Sjasone 62235238Sjasone return (InterlockedExchangeAdd64(p, -((int64_t)x))); 63235238Sjasone} 64234370Sjasone#elif (defined(JEMALLOC_OSATOMIC)) 65234370SjasoneJEMALLOC_INLINE uint64_t 66234370Sjasoneatomic_add_uint64(uint64_t *p, uint64_t x) 67234370Sjasone{ 68234370Sjasone 69234370Sjasone return (OSAtomicAdd64((int64_t)x, (int64_t *)p)); 70234370Sjasone} 71234370Sjasone 72234370SjasoneJEMALLOC_INLINE uint64_t 73234370Sjasoneatomic_sub_uint64(uint64_t *p, uint64_t x) 74234370Sjasone{ 75234370Sjasone 76234370Sjasone return (OSAtomicAdd64(-((int64_t)x), (int64_t *)p)); 77234370Sjasone} 78234402Sjasone# elif (defined(__amd64__) || defined(__x86_64__)) 79234370SjasoneJEMALLOC_INLINE uint64_t 80234370Sjasoneatomic_add_uint64(uint64_t *p, uint64_t x) 81234370Sjasone{ 82234370Sjasone 83234370Sjasone asm volatile ( 84234370Sjasone "lock; xaddq %0, %1;" 85234370Sjasone : "+r" (x), "=m" (*p) /* Outputs. */ 86234370Sjasone : "m" (*p) /* Inputs. */ 87234370Sjasone ); 88234370Sjasone 89234370Sjasone return (x); 90234370Sjasone} 91234370Sjasone 92234370SjasoneJEMALLOC_INLINE uint64_t 93234370Sjasoneatomic_sub_uint64(uint64_t *p, uint64_t x) 94234370Sjasone{ 95234370Sjasone 96234370Sjasone x = (uint64_t)(-(int64_t)x); 97234370Sjasone asm volatile ( 98234370Sjasone "lock; xaddq %0, %1;" 99234370Sjasone : "+r" (x), "=m" (*p) /* Outputs. */ 100234370Sjasone : "m" (*p) /* Inputs. */ 101234370Sjasone ); 102234370Sjasone 103234370Sjasone return (x); 104234370Sjasone} 105234402Sjasone# elif (defined(JEMALLOC_ATOMIC9)) 106234370SjasoneJEMALLOC_INLINE uint64_t 107234370Sjasoneatomic_add_uint64(uint64_t *p, uint64_t x) 108234370Sjasone{ 109234370Sjasone 110234402Sjasone /* 111234402Sjasone * atomic_fetchadd_64() doesn't exist, but we only ever use this 112234402Sjasone * function on LP64 systems, so atomic_fetchadd_long() will do. 113234402Sjasone */ 114234402Sjasone assert(sizeof(uint64_t) == sizeof(unsigned long)); 115234402Sjasone 116234402Sjasone return (atomic_fetchadd_long(p, (unsigned long)x) + x); 117234402Sjasone} 118234402Sjasone 119234402SjasoneJEMALLOC_INLINE uint64_t 120234402Sjasoneatomic_sub_uint64(uint64_t *p, uint64_t x) 121234402Sjasone{ 122234402Sjasone 123234402Sjasone assert(sizeof(uint64_t) == sizeof(unsigned long)); 124234402Sjasone 125234402Sjasone return (atomic_fetchadd_long(p, (unsigned long)(-(long)x)) - x); 126234402Sjasone} 127234402Sjasone# elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_8)) 128234402SjasoneJEMALLOC_INLINE uint64_t 129234402Sjasoneatomic_add_uint64(uint64_t *p, uint64_t x) 130234402Sjasone{ 131234402Sjasone 132234370Sjasone return (__sync_add_and_fetch(p, x)); 133234370Sjasone} 134234370Sjasone 135234370SjasoneJEMALLOC_INLINE uint64_t 136234370Sjasoneatomic_sub_uint64(uint64_t *p, uint64_t x) 137234370Sjasone{ 138234370Sjasone 139234370Sjasone return (__sync_sub_and_fetch(p, x)); 140234370Sjasone} 141234402Sjasone# else 142234370Sjasone# error "Missing implementation for 64-bit atomic operations" 143234370Sjasone# endif 144234370Sjasone#endif 145234370Sjasone 146234370Sjasone/******************************************************************************/ 147234370Sjasone/* 32-bit operations. */ 148234370Sjasone#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 149234370SjasoneJEMALLOC_INLINE uint32_t 150234370Sjasoneatomic_add_uint32(uint32_t *p, uint32_t x) 151234370Sjasone{ 152234370Sjasone 153234370Sjasone return (__sync_add_and_fetch(p, x)); 154234370Sjasone} 155234370Sjasone 156234370SjasoneJEMALLOC_INLINE uint32_t 157234370Sjasoneatomic_sub_uint32(uint32_t *p, uint32_t x) 158234370Sjasone{ 159234370Sjasone 160234370Sjasone return (__sync_sub_and_fetch(p, x)); 161234370Sjasone} 162235238Sjasone#elif (defined(_MSC_VER)) 163235238SjasoneJEMALLOC_INLINE uint32_t 164235238Sjasoneatomic_add_uint32(uint32_t *p, uint32_t x) 165235238Sjasone{ 166235238Sjasone 167235238Sjasone return (InterlockedExchangeAdd(p, x)); 168235238Sjasone} 169235238Sjasone 170235238SjasoneJEMALLOC_INLINE uint32_t 171235238Sjasoneatomic_sub_uint32(uint32_t *p, uint32_t x) 172235238Sjasone{ 173235238Sjasone 174235238Sjasone return (InterlockedExchangeAdd(p, -((int32_t)x))); 175235238Sjasone} 176234370Sjasone#elif (defined(JEMALLOC_OSATOMIC)) 177234370SjasoneJEMALLOC_INLINE uint32_t 178234370Sjasoneatomic_add_uint32(uint32_t *p, uint32_t x) 179234370Sjasone{ 180234370Sjasone 181234370Sjasone return (OSAtomicAdd32((int32_t)x, (int32_t *)p)); 182234370Sjasone} 183234370Sjasone 184234370SjasoneJEMALLOC_INLINE uint32_t 185234370Sjasoneatomic_sub_uint32(uint32_t *p, uint32_t x) 186234370Sjasone{ 187234370Sjasone 188234370Sjasone return (OSAtomicAdd32(-((int32_t)x), (int32_t *)p)); 189234370Sjasone} 190234370Sjasone#elif (defined(__i386__) || defined(__amd64__) || defined(__x86_64__)) 191234370SjasoneJEMALLOC_INLINE uint32_t 192234370Sjasoneatomic_add_uint32(uint32_t *p, uint32_t x) 193234370Sjasone{ 194234370Sjasone 195234370Sjasone asm volatile ( 196234370Sjasone "lock; xaddl %0, %1;" 197234370Sjasone : "+r" (x), "=m" (*p) /* Outputs. */ 198234370Sjasone : "m" (*p) /* Inputs. */ 199234370Sjasone ); 200234370Sjasone 201234370Sjasone return (x); 202234370Sjasone} 203234370Sjasone 204234370SjasoneJEMALLOC_INLINE uint32_t 205234370Sjasoneatomic_sub_uint32(uint32_t *p, uint32_t x) 206234370Sjasone{ 207234370Sjasone 208234370Sjasone x = (uint32_t)(-(int32_t)x); 209234370Sjasone asm volatile ( 210234370Sjasone "lock; xaddl %0, %1;" 211234370Sjasone : "+r" (x), "=m" (*p) /* Outputs. */ 212234370Sjasone : "m" (*p) /* Inputs. */ 213234370Sjasone ); 214234370Sjasone 215234370Sjasone return (x); 216234370Sjasone} 217234402Sjasone#elif (defined(JEMALLOC_ATOMIC9)) 218234402SjasoneJEMALLOC_INLINE uint32_t 219234402Sjasoneatomic_add_uint32(uint32_t *p, uint32_t x) 220234402Sjasone{ 221234402Sjasone 222234402Sjasone return (atomic_fetchadd_32(p, x) + x); 223234402Sjasone} 224234402Sjasone 225234402SjasoneJEMALLOC_INLINE uint32_t 226234402Sjasoneatomic_sub_uint32(uint32_t *p, uint32_t x) 227234402Sjasone{ 228234402Sjasone 229234402Sjasone return (atomic_fetchadd_32(p, (uint32_t)(-(int32_t)x)) - x); 230234402Sjasone} 231234370Sjasone#elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_4)) 232234370SjasoneJEMALLOC_INLINE uint32_t 233234370Sjasoneatomic_add_uint32(uint32_t *p, uint32_t x) 234234370Sjasone{ 235234370Sjasone 236234370Sjasone return (__sync_add_and_fetch(p, x)); 237234370Sjasone} 238234370Sjasone 239234370SjasoneJEMALLOC_INLINE uint32_t 240234370Sjasoneatomic_sub_uint32(uint32_t *p, uint32_t x) 241234370Sjasone{ 242234370Sjasone 243234370Sjasone return (__sync_sub_and_fetch(p, x)); 244234370Sjasone} 245234370Sjasone#else 246234370Sjasone# error "Missing implementation for 32-bit atomic operations" 247234370Sjasone#endif 248234370Sjasone 249234370Sjasone/******************************************************************************/ 250234370Sjasone/* size_t operations. */ 251234370SjasoneJEMALLOC_INLINE size_t 252234370Sjasoneatomic_add_z(size_t *p, size_t x) 253234370Sjasone{ 254234370Sjasone 255234370Sjasone#if (LG_SIZEOF_PTR == 3) 256234370Sjasone return ((size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)x)); 257234370Sjasone#elif (LG_SIZEOF_PTR == 2) 258234370Sjasone return ((size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)x)); 259234370Sjasone#endif 260234370Sjasone} 261234370Sjasone 262234370SjasoneJEMALLOC_INLINE size_t 263234370Sjasoneatomic_sub_z(size_t *p, size_t x) 264234370Sjasone{ 265234370Sjasone 266234370Sjasone#if (LG_SIZEOF_PTR == 3) 267234370Sjasone return ((size_t)atomic_add_uint64((uint64_t *)p, 268234370Sjasone (uint64_t)-((int64_t)x))); 269234370Sjasone#elif (LG_SIZEOF_PTR == 2) 270234370Sjasone return ((size_t)atomic_add_uint32((uint32_t *)p, 271234370Sjasone (uint32_t)-((int32_t)x))); 272234370Sjasone#endif 273234370Sjasone} 274234370Sjasone 275234370Sjasone/******************************************************************************/ 276234370Sjasone/* unsigned operations. */ 277234370SjasoneJEMALLOC_INLINE unsigned 278234370Sjasoneatomic_add_u(unsigned *p, unsigned x) 279234370Sjasone{ 280234370Sjasone 281234370Sjasone#if (LG_SIZEOF_INT == 3) 282234370Sjasone return ((unsigned)atomic_add_uint64((uint64_t *)p, (uint64_t)x)); 283234370Sjasone#elif (LG_SIZEOF_INT == 2) 284234370Sjasone return ((unsigned)atomic_add_uint32((uint32_t *)p, (uint32_t)x)); 285234370Sjasone#endif 286234370Sjasone} 287234370Sjasone 288234370SjasoneJEMALLOC_INLINE unsigned 289234370Sjasoneatomic_sub_u(unsigned *p, unsigned x) 290234370Sjasone{ 291234370Sjasone 292234370Sjasone#if (LG_SIZEOF_INT == 3) 293234370Sjasone return ((unsigned)atomic_add_uint64((uint64_t *)p, 294234370Sjasone (uint64_t)-((int64_t)x))); 295234370Sjasone#elif (LG_SIZEOF_INT == 2) 296234370Sjasone return ((unsigned)atomic_add_uint32((uint32_t *)p, 297234370Sjasone (uint32_t)-((int32_t)x))); 298234370Sjasone#endif 299234370Sjasone} 300234370Sjasone/******************************************************************************/ 301234370Sjasone#endif 302234370Sjasone 303234370Sjasone#endif /* JEMALLOC_H_INLINES */ 304234370Sjasone/******************************************************************************/ 305