atomic.h revision 264375
1/*- 2 * Copyright (c) 2008 Marcel Moolenaar 3 * Copyright (c) 2001 Benno Rice 4 * Copyright (c) 2001 David E. O'Brien 5 * Copyright (c) 1998 Doug Rabson 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD: stable/10/sys/powerpc/include/atomic.h 264375 2014-04-12 19:57:15Z andreast $ 30 */ 31 32#ifndef _MACHINE_ATOMIC_H_ 33#define _MACHINE_ATOMIC_H_ 34 35#ifndef _SYS_CDEFS_H_ 36#error this file needs sys/cdefs.h as a prerequisite 37#endif 38 39/* 40 * The __ATOMIC_REL/ACQ() macros provide memory barriers only in conjunction 41 * with the atomic lXarx/stXcx. sequences below. They are not exposed outside 42 * of this file. See also Appendix B.2 of Book II of the architecture manual. 43 * 44 * Note that not all Book-E processors accept the light-weight sync variant. 45 * In particular, early models of E500 cores are known to wedge. Bank on all 46 * 64-bit capable CPUs to accept lwsync properly and pressimize 32-bit CPUs 47 * to use the heavier-weight sync. 48 */ 49 50#ifdef __powerpc64__ 51#define mb() __asm __volatile("lwsync" : : : "memory") 52#define rmb() __asm __volatile("lwsync" : : : "memory") 53#define wmb() __asm __volatile("lwsync" : : : "memory") 54#define __ATOMIC_REL() __asm __volatile("lwsync" : : : "memory") 55#define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory") 56#else 57#define mb() __asm __volatile("sync" : : : "memory") 58#define rmb() __asm __volatile("sync" : : : "memory") 59#define wmb() __asm __volatile("sync" : : : "memory") 60#define __ATOMIC_REL() __asm __volatile("sync" : : : "memory") 61#define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory") 62#endif 63 64/* 65 * atomic_add(p, v) 66 * { *p += v; } 67 */ 68 69#define __atomic_add_int(p, v, t) \ 70 __asm __volatile( \ 71 "1: lwarx %0, 0, %2\n" \ 72 " add %0, %3, %0\n" \ 73 " stwcx. %0, 0, %2\n" \ 74 " bne- 1b\n" \ 75 : "=&r" (t), "=m" (*p) \ 76 : "r" (p), "r" (v), "m" (*p) \ 77 : "cc", "memory") \ 78 /* __atomic_add_int */ 79 80#ifdef __powerpc64__ 81#define __atomic_add_long(p, v, t) \ 82 __asm __volatile( \ 83 "1: ldarx %0, 0, %2\n" \ 84 " add %0, %3, %0\n" \ 85 " stdcx. %0, 0, %2\n" \ 86 " bne- 1b\n" \ 87 : "=&r" (t), "=m" (*p) \ 88 : "r" (p), "r" (v), "m" (*p) \ 89 : "cc", "memory") \ 90 /* __atomic_add_long */ 91#else 92#define __atomic_add_long(p, v, t) \ 93 __asm __volatile( \ 94 "1: lwarx %0, 0, %2\n" \ 95 " add %0, %3, %0\n" \ 96 " stwcx. %0, 0, %2\n" \ 97 " bne- 1b\n" \ 98 : "=&r" (t), "=m" (*p) \ 99 : "r" (p), "r" (v), "m" (*p) \ 100 : "cc", "memory") \ 101 /* __atomic_add_long */ 102#endif 103 104#define _ATOMIC_ADD(type) \ 105 static __inline void \ 106 atomic_add_##type(volatile u_##type *p, u_##type v) { \ 107 u_##type t; \ 108 __atomic_add_##type(p, v, t); \ 109 } \ 110 \ 111 static __inline void \ 112 atomic_add_acq_##type(volatile u_##type *p, u_##type v) { \ 113 u_##type t; \ 114 __atomic_add_##type(p, v, t); \ 115 __ATOMIC_ACQ(); \ 116 } \ 117 \ 118 static __inline void \ 119 atomic_add_rel_##type(volatile u_##type *p, u_##type v) { \ 120 u_##type t; \ 121 __ATOMIC_REL(); \ 122 __atomic_add_##type(p, v, t); \ 123 } \ 124 /* _ATOMIC_ADD */ 125 126_ATOMIC_ADD(int) 127_ATOMIC_ADD(long) 128 129#define atomic_add_32 atomic_add_int 130#define atomic_add_acq_32 atomic_add_acq_int 131#define atomic_add_rel_32 atomic_add_rel_int 132 133#ifdef __powerpc64__ 134#define atomic_add_64 atomic_add_long 135#define atomic_add_acq_64 atomic_add_acq_long 136#define atomic_add_rel_64 atomic_add_rel_long 137 138#define atomic_add_ptr atomic_add_long 139#define atomic_add_acq_ptr atomic_add_acq_long 140#define atomic_add_rel_ptr atomic_add_rel_long 141#else 142#define atomic_add_ptr atomic_add_int 143#define atomic_add_acq_ptr atomic_add_acq_int 144#define atomic_add_rel_ptr atomic_add_rel_int 145#endif 146#undef _ATOMIC_ADD 147#undef __atomic_add_long 148#undef __atomic_add_int 149 150/* 151 * atomic_clear(p, v) 152 * { *p &= ~v; } 153 */ 154 155#define __atomic_clear_int(p, v, t) \ 156 __asm __volatile( \ 157 "1: lwarx %0, 0, %2\n" \ 158 " andc %0, %0, %3\n" \ 159 " stwcx. %0, 0, %2\n" \ 160 " bne- 1b\n" \ 161 : "=&r" (t), "=m" (*p) \ 162 : "r" (p), "r" (v), "m" (*p) \ 163 : "cc", "memory") \ 164 /* __atomic_clear_int */ 165 166#ifdef __powerpc64__ 167#define __atomic_clear_long(p, v, t) \ 168 __asm __volatile( \ 169 "1: ldarx %0, 0, %2\n" \ 170 " andc %0, %0, %3\n" \ 171 " stdcx. %0, 0, %2\n" \ 172 " bne- 1b\n" \ 173 : "=&r" (t), "=m" (*p) \ 174 : "r" (p), "r" (v), "m" (*p) \ 175 : "cc", "memory") \ 176 /* __atomic_clear_long */ 177#else 178#define __atomic_clear_long(p, v, t) \ 179 __asm __volatile( \ 180 "1: lwarx %0, 0, %2\n" \ 181 " andc %0, %0, %3\n" \ 182 " stwcx. %0, 0, %2\n" \ 183 " bne- 1b\n" \ 184 : "=&r" (t), "=m" (*p) \ 185 : "r" (p), "r" (v), "m" (*p) \ 186 : "cc", "memory") \ 187 /* __atomic_clear_long */ 188#endif 189 190#define _ATOMIC_CLEAR(type) \ 191 static __inline void \ 192 atomic_clear_##type(volatile u_##type *p, u_##type v) { \ 193 u_##type t; \ 194 __atomic_clear_##type(p, v, t); \ 195 } \ 196 \ 197 static __inline void \ 198 atomic_clear_acq_##type(volatile u_##type *p, u_##type v) { \ 199 u_##type t; \ 200 __atomic_clear_##type(p, v, t); \ 201 __ATOMIC_ACQ(); \ 202 } \ 203 \ 204 static __inline void \ 205 atomic_clear_rel_##type(volatile u_##type *p, u_##type v) { \ 206 u_##type t; \ 207 __ATOMIC_REL(); \ 208 __atomic_clear_##type(p, v, t); \ 209 } \ 210 /* _ATOMIC_CLEAR */ 211 212 213_ATOMIC_CLEAR(int) 214_ATOMIC_CLEAR(long) 215 216#define atomic_clear_32 atomic_clear_int 217#define atomic_clear_acq_32 atomic_clear_acq_int 218#define atomic_clear_rel_32 atomic_clear_rel_int 219 220#ifdef __powerpc64__ 221#define atomic_clear_64 atomic_clear_long 222#define atomic_clear_acq_64 atomic_clear_acq_long 223#define atomic_clear_rel_64 atomic_clear_rel_long 224 225#define atomic_clear_ptr atomic_clear_long 226#define atomic_clear_acq_ptr atomic_clear_acq_long 227#define atomic_clear_rel_ptr atomic_clear_rel_long 228#else 229#define atomic_clear_ptr atomic_clear_int 230#define atomic_clear_acq_ptr atomic_clear_acq_int 231#define atomic_clear_rel_ptr atomic_clear_rel_int 232#endif 233#undef _ATOMIC_CLEAR 234#undef __atomic_clear_long 235#undef __atomic_clear_int 236 237/* 238 * atomic_cmpset(p, o, n) 239 */ 240/* TODO -- see below */ 241 242/* 243 * atomic_load_acq(p) 244 */ 245/* TODO -- see below */ 246 247/* 248 * atomic_readandclear(p) 249 */ 250/* TODO -- see below */ 251 252/* 253 * atomic_set(p, v) 254 * { *p |= v; } 255 */ 256 257#define __atomic_set_int(p, v, t) \ 258 __asm __volatile( \ 259 "1: lwarx %0, 0, %2\n" \ 260 " or %0, %3, %0\n" \ 261 " stwcx. %0, 0, %2\n" \ 262 " bne- 1b\n" \ 263 : "=&r" (t), "=m" (*p) \ 264 : "r" (p), "r" (v), "m" (*p) \ 265 : "cc", "memory") \ 266 /* __atomic_set_int */ 267 268#ifdef __powerpc64__ 269#define __atomic_set_long(p, v, t) \ 270 __asm __volatile( \ 271 "1: ldarx %0, 0, %2\n" \ 272 " or %0, %3, %0\n" \ 273 " stdcx. %0, 0, %2\n" \ 274 " bne- 1b\n" \ 275 : "=&r" (t), "=m" (*p) \ 276 : "r" (p), "r" (v), "m" (*p) \ 277 : "cc", "memory") \ 278 /* __atomic_set_long */ 279#else 280#define __atomic_set_long(p, v, t) \ 281 __asm __volatile( \ 282 "1: lwarx %0, 0, %2\n" \ 283 " or %0, %3, %0\n" \ 284 " stwcx. %0, 0, %2\n" \ 285 " bne- 1b\n" \ 286 : "=&r" (t), "=m" (*p) \ 287 : "r" (p), "r" (v), "m" (*p) \ 288 : "cc", "memory") \ 289 /* __atomic_set_long */ 290#endif 291 292#define _ATOMIC_SET(type) \ 293 static __inline void \ 294 atomic_set_##type(volatile u_##type *p, u_##type v) { \ 295 u_##type t; \ 296 __atomic_set_##type(p, v, t); \ 297 } \ 298 \ 299 static __inline void \ 300 atomic_set_acq_##type(volatile u_##type *p, u_##type v) { \ 301 u_##type t; \ 302 __atomic_set_##type(p, v, t); \ 303 __ATOMIC_ACQ(); \ 304 } \ 305 \ 306 static __inline void \ 307 atomic_set_rel_##type(volatile u_##type *p, u_##type v) { \ 308 u_##type t; \ 309 __ATOMIC_REL(); \ 310 __atomic_set_##type(p, v, t); \ 311 } \ 312 /* _ATOMIC_SET */ 313 314_ATOMIC_SET(int) 315_ATOMIC_SET(long) 316 317#define atomic_set_32 atomic_set_int 318#define atomic_set_acq_32 atomic_set_acq_int 319#define atomic_set_rel_32 atomic_set_rel_int 320 321#ifdef __powerpc64__ 322#define atomic_set_64 atomic_set_long 323#define atomic_set_acq_64 atomic_set_acq_long 324#define atomic_set_rel_64 atomic_set_rel_long 325 326#define atomic_set_ptr atomic_set_long 327#define atomic_set_acq_ptr atomic_set_acq_long 328#define atomic_set_rel_ptr atomic_set_rel_long 329#else 330#define atomic_set_ptr atomic_set_int 331#define atomic_set_acq_ptr atomic_set_acq_int 332#define atomic_set_rel_ptr atomic_set_rel_int 333#endif 334#undef _ATOMIC_SET 335#undef __atomic_set_long 336#undef __atomic_set_int 337 338/* 339 * atomic_subtract(p, v) 340 * { *p -= v; } 341 */ 342 343#define __atomic_subtract_int(p, v, t) \ 344 __asm __volatile( \ 345 "1: lwarx %0, 0, %2\n" \ 346 " subf %0, %3, %0\n" \ 347 " stwcx. %0, 0, %2\n" \ 348 " bne- 1b\n" \ 349 : "=&r" (t), "=m" (*p) \ 350 : "r" (p), "r" (v), "m" (*p) \ 351 : "cc", "memory") \ 352 /* __atomic_subtract_int */ 353 354#ifdef __powerpc64__ 355#define __atomic_subtract_long(p, v, t) \ 356 __asm __volatile( \ 357 "1: ldarx %0, 0, %2\n" \ 358 " subf %0, %3, %0\n" \ 359 " stdcx. %0, 0, %2\n" \ 360 " bne- 1b\n" \ 361 : "=&r" (t), "=m" (*p) \ 362 : "r" (p), "r" (v), "m" (*p) \ 363 : "cc", "memory") \ 364 /* __atomic_subtract_long */ 365#else 366#define __atomic_subtract_long(p, v, t) \ 367 __asm __volatile( \ 368 "1: lwarx %0, 0, %2\n" \ 369 " subf %0, %3, %0\n" \ 370 " stwcx. %0, 0, %2\n" \ 371 " bne- 1b\n" \ 372 : "=&r" (t), "=m" (*p) \ 373 : "r" (p), "r" (v), "m" (*p) \ 374 : "cc", "memory") \ 375 /* __atomic_subtract_long */ 376#endif 377 378#define _ATOMIC_SUBTRACT(type) \ 379 static __inline void \ 380 atomic_subtract_##type(volatile u_##type *p, u_##type v) { \ 381 u_##type t; \ 382 __atomic_subtract_##type(p, v, t); \ 383 } \ 384 \ 385 static __inline void \ 386 atomic_subtract_acq_##type(volatile u_##type *p, u_##type v) { \ 387 u_##type t; \ 388 __atomic_subtract_##type(p, v, t); \ 389 __ATOMIC_ACQ(); \ 390 } \ 391 \ 392 static __inline void \ 393 atomic_subtract_rel_##type(volatile u_##type *p, u_##type v) { \ 394 u_##type t; \ 395 __ATOMIC_REL(); \ 396 __atomic_subtract_##type(p, v, t); \ 397 } \ 398 /* _ATOMIC_SUBTRACT */ 399 400_ATOMIC_SUBTRACT(int) 401_ATOMIC_SUBTRACT(long) 402 403#define atomic_subtract_32 atomic_subtract_int 404#define atomic_subtract_acq_32 atomic_subtract_acq_int 405#define atomic_subtract_rel_32 atomic_subtract_rel_int 406 407#ifdef __powerpc64__ 408#define atomic_subtract_64 atomic_subtract_long 409#define atomic_subtract_acq_64 atomic_subract_acq_long 410#define atomic_subtract_rel_64 atomic_subtract_rel_long 411 412#define atomic_subtract_ptr atomic_subtract_long 413#define atomic_subtract_acq_ptr atomic_subtract_acq_long 414#define atomic_subtract_rel_ptr atomic_subtract_rel_long 415#else 416#define atomic_subtract_ptr atomic_subtract_int 417#define atomic_subtract_acq_ptr atomic_subtract_acq_int 418#define atomic_subtract_rel_ptr atomic_subtract_rel_int 419#endif 420#undef _ATOMIC_SUBTRACT 421#undef __atomic_subtract_long 422#undef __atomic_subtract_int 423 424/* 425 * atomic_store_rel(p, v) 426 */ 427/* TODO -- see below */ 428 429/* 430 * Old/original implementations that still need revisiting. 431 */ 432 433static __inline u_int 434atomic_readandclear_int(volatile u_int *addr) 435{ 436 u_int result,temp; 437 438#ifdef __GNUCLIKE_ASM 439 __asm __volatile ( 440 "\tsync\n" /* drain writes */ 441 "1:\tlwarx %0, 0, %3\n\t" /* load old value */ 442 "li %1, 0\n\t" /* load new value */ 443 "stwcx. %1, 0, %3\n\t" /* attempt to store */ 444 "bne- 1b\n\t" /* spin if failed */ 445 : "=&r"(result), "=&r"(temp), "=m" (*addr) 446 : "r" (addr), "m" (*addr) 447 : "cc", "memory"); 448#endif 449 450 return (result); 451} 452 453#ifdef __powerpc64__ 454static __inline u_long 455atomic_readandclear_long(volatile u_long *addr) 456{ 457 u_long result,temp; 458 459#ifdef __GNUCLIKE_ASM 460 __asm __volatile ( 461 "\tsync\n" /* drain writes */ 462 "1:\tldarx %0, 0, %3\n\t" /* load old value */ 463 "li %1, 0\n\t" /* load new value */ 464 "stdcx. %1, 0, %3\n\t" /* attempt to store */ 465 "bne- 1b\n\t" /* spin if failed */ 466 : "=&r"(result), "=&r"(temp), "=m" (*addr) 467 : "r" (addr), "m" (*addr) 468 : "cc", "memory"); 469#endif 470 471 return (result); 472} 473#endif 474 475#define atomic_readandclear_32 atomic_readandclear_int 476 477#ifdef __powerpc64__ 478#define atomic_readandclear_64 atomic_readandclear_long 479 480#define atomic_readandclear_ptr atomic_readandclear_long 481#else 482static __inline u_long 483atomic_readandclear_long(volatile u_long *addr) 484{ 485 486 return ((u_long)atomic_readandclear_int((volatile u_int *)addr)); 487} 488 489#define atomic_readandclear_ptr atomic_readandclear_int 490#endif 491 492/* 493 * We assume that a = b will do atomic loads and stores. 494 */ 495#define ATOMIC_STORE_LOAD(TYPE) \ 496static __inline u_##TYPE \ 497atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ 498{ \ 499 u_##TYPE v; \ 500 \ 501 v = *p; \ 502 mb(); \ 503 return (v); \ 504} \ 505 \ 506static __inline void \ 507atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ 508{ \ 509 mb(); \ 510 *p = v; \ 511} 512 513ATOMIC_STORE_LOAD(int) 514 515#define atomic_load_acq_32 atomic_load_acq_int 516#define atomic_store_rel_32 atomic_store_rel_int 517 518#ifdef __powerpc64__ 519ATOMIC_STORE_LOAD(long) 520 521#define atomic_load_acq_64 atomic_load_acq_long 522#define atomic_store_rel_64 atomic_store_rel_long 523 524#define atomic_load_acq_ptr atomic_load_acq_long 525#define atomic_store_rel_ptr atomic_store_rel_long 526#else 527static __inline u_long 528atomic_load_acq_long(volatile u_long *addr) 529{ 530 531 return ((u_long)atomic_load_acq_int((volatile u_int *)addr)); 532} 533 534static __inline void 535atomic_store_rel_long(volatile u_long *addr, u_long val) 536{ 537 538 atomic_store_rel_int((volatile u_int *)addr, (u_int)val); 539} 540 541#define atomic_load_acq_ptr atomic_load_acq_int 542#define atomic_store_rel_ptr atomic_store_rel_int 543#endif 544#undef ATOMIC_STORE_LOAD 545 546/* 547 * Atomically compare the value stored at *p with cmpval and if the 548 * two values are equal, update the value of *p with newval. Returns 549 * zero if the compare failed, nonzero otherwise. 550 */ 551static __inline int 552atomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval) 553{ 554 int ret; 555 556#ifdef __GNUCLIKE_ASM 557 __asm __volatile ( 558 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 559 "cmplw %3, %0\n\t" /* compare */ 560 "bne 2f\n\t" /* exit if not equal */ 561 "stwcx. %4, 0, %2\n\t" /* attempt to store */ 562 "bne- 1b\n\t" /* spin if failed */ 563 "li %0, 1\n\t" /* success - retval = 1 */ 564 "b 3f\n\t" /* we've succeeded */ 565 "2:\n\t" 566 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 567 "li %0, 0\n\t" /* failure - retval = 0 */ 568 "3:\n\t" 569 : "=&r" (ret), "=m" (*p) 570 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 571 : "cc", "memory"); 572#endif 573 574 return (ret); 575} 576static __inline int 577atomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval) 578{ 579 int ret; 580 581#ifdef __GNUCLIKE_ASM 582 __asm __volatile ( 583 #ifdef __powerpc64__ 584 "1:\tldarx %0, 0, %2\n\t" /* load old value */ 585 "cmpld %3, %0\n\t" /* compare */ 586 "bne 2f\n\t" /* exit if not equal */ 587 "stdcx. %4, 0, %2\n\t" /* attempt to store */ 588 #else 589 "1:\tlwarx %0, 0, %2\n\t" /* load old value */ 590 "cmplw %3, %0\n\t" /* compare */ 591 "bne 2f\n\t" /* exit if not equal */ 592 "stwcx. %4, 0, %2\n\t" /* attempt to store */ 593 #endif 594 "bne- 1b\n\t" /* spin if failed */ 595 "li %0, 1\n\t" /* success - retval = 1 */ 596 "b 3f\n\t" /* we've succeeded */ 597 "2:\n\t" 598 #ifdef __powerpc64__ 599 "stdcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 600 #else 601 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ 602 #endif 603 "li %0, 0\n\t" /* failure - retval = 0 */ 604 "3:\n\t" 605 : "=&r" (ret), "=m" (*p) 606 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) 607 : "cc", "memory"); 608#endif 609 610 return (ret); 611} 612 613static __inline int 614atomic_cmpset_acq_int(volatile u_int *p, u_int cmpval, u_int newval) 615{ 616 int retval; 617 618 retval = atomic_cmpset_int(p, cmpval, newval); 619 __ATOMIC_ACQ(); 620 return (retval); 621} 622 623static __inline int 624atomic_cmpset_rel_int(volatile u_int *p, u_int cmpval, u_int newval) 625{ 626 __ATOMIC_REL(); 627 return (atomic_cmpset_int(p, cmpval, newval)); 628} 629 630static __inline int 631atomic_cmpset_acq_long(volatile u_long *p, u_long cmpval, u_long newval) 632{ 633 u_long retval; 634 635 retval = atomic_cmpset_long(p, cmpval, newval); 636 __ATOMIC_ACQ(); 637 return (retval); 638} 639 640static __inline int 641atomic_cmpset_rel_long(volatile u_long *p, u_long cmpval, u_long newval) 642{ 643 __ATOMIC_REL(); 644 return (atomic_cmpset_long(p, cmpval, newval)); 645} 646 647#define atomic_cmpset_32 atomic_cmpset_int 648#define atomic_cmpset_acq_32 atomic_cmpset_acq_int 649#define atomic_cmpset_rel_32 atomic_cmpset_rel_int 650 651#ifdef __powerpc64__ 652#define atomic_cmpset_64 atomic_cmpset_long 653#define atomic_cmpset_acq_64 atomic_cmpset_acq_long 654#define atomic_cmpset_rel_64 atomic_cmpset_rel_long 655 656#define atomic_cmpset_ptr atomic_cmpset_long 657#define atomic_cmpset_acq_ptr atomic_cmpset_acq_long 658#define atomic_cmpset_rel_ptr atomic_cmpset_rel_long 659#else 660#define atomic_cmpset_ptr atomic_cmpset_int 661#define atomic_cmpset_acq_ptr atomic_cmpset_acq_int 662#define atomic_cmpset_rel_ptr atomic_cmpset_rel_int 663#endif 664 665static __inline u_int 666atomic_fetchadd_int(volatile u_int *p, u_int v) 667{ 668 u_int value; 669 670 do { 671 value = *p; 672 } while (!atomic_cmpset_int(p, value, value + v)); 673 return (value); 674} 675 676static __inline u_long 677atomic_fetchadd_long(volatile u_long *p, u_long v) 678{ 679 u_long value; 680 681 do { 682 value = *p; 683 } while (!atomic_cmpset_long(p, value, value + v)); 684 return (value); 685} 686 687static __inline u_int 688atomic_swap_32(volatile u_int *p, u_int v) 689{ 690 u_int prev; 691 692 __asm __volatile( 693 "1: lwarx %0,0,%2\n" 694 " stwcx. %3,0,%2\n" 695 " bne- 1b\n" 696 : "=&r" (prev), "+m" (*(volatile u_int *)p) 697 : "r" (p), "r" (v) 698 : "cc", "memory"); 699 700 return (prev); 701} 702 703#ifdef __powerpc64__ 704static __inline u_long 705atomic_swap_64(volatile u_long *p, u_long v) 706{ 707 u_long prev; 708 709 __asm __volatile( 710 "1: ldarx %0,0,%2\n" 711 " stdcx. %3,0,%2\n" 712 " bne- 1b\n" 713 : "=&r" (prev), "+m" (*(volatile u_long *)p) 714 : "r" (p), "r" (v) 715 : "cc", "memory"); 716 717 return (prev); 718} 719#endif 720 721#define atomic_fetchadd_32 atomic_fetchadd_int 722#define atomic_swap_int atomic_swap_32 723 724#ifdef __powerpc64__ 725#define atomic_fetchadd_64 atomic_fetchadd_long 726#define atomic_swap_long atomic_swap_64 727#define atomic_swap_ptr atomic_swap_64 728#endif 729 730#undef __ATOMIC_REL 731#undef __ATOMIC_ACQ 732 733#endif /* ! _MACHINE_ATOMIC_H_ */ 734