1/***********************license start***************
2 * Copyright (c) 2003-2010  Cavium Networks (support@cavium.com). All rights
3 * reserved.
4 *
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are
8 * met:
9 *
10 *   * Redistributions of source code must retain the above copyright
11 *     notice, this list of conditions and the following disclaimer.
12 *
13 *   * Redistributions in binary form must reproduce the above
14 *     copyright notice, this list of conditions and the following
15 *     disclaimer in the documentation and/or other materials provided
16 *     with the distribution.
17
18 *   * Neither the name of Cavium Networks nor the names of
19 *     its contributors may be used to endorse or promote products
20 *     derived from this software without specific prior written
21 *     permission.
22
23 * This Software, including technical data, may be subject to U.S. export  control
24 * laws, including the U.S. Export Administration Act and its  associated
25 * regulations, and may be subject to export or import  regulations in other
26 * countries.
27
28 * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
29 * AND WITH ALL FAULTS AND CAVIUM  NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR
30 * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO
31 * THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR
32 * DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
33 * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
34 * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
35 * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
36 * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE  RISK ARISING OUT OF USE OR
37 * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
38 ***********************license end**************************************/
39
40
41
42
43
44
45
46/**
47 * @file
48 *
49 * This file provides atomic operations
50 *
51 * <hr>$Revision: 49448 $<hr>
52 *
53 *
54 */
55
56
57#ifndef __CVMX_ATOMIC_H__
58#define __CVMX_ATOMIC_H__
59
60#ifdef	__cplusplus
61extern "C" {
62#endif
63
64
65/**
66 * Atomically adds a signed value to a 32 bit (aligned) memory location.
67 *
68 * This version does not perform 'sync' operations to enforce memory
69 * operations.  This should only be used when there are no memory operation
70 * ordering constraints.  (This should NOT be used for reference counting -
71 * use the standard version instead.)
72 *
73 * @param ptr    address in memory to add incr to
74 * @param incr   amount to increment memory location by (signed)
75 */
76static inline void cvmx_atomic_add32_nosync(int32_t *ptr, int32_t incr)
77{
78    if (OCTEON_IS_MODEL(OCTEON_CN3XXX))
79    {
80	uint32_t tmp;
81
82        __asm__ __volatile__(
83        ".set noreorder         \n"
84        "1: ll   %[tmp], %[val] \n"
85        "   addu %[tmp], %[inc] \n"
86        "   sc   %[tmp], %[val] \n"
87        "   beqz %[tmp], 1b     \n"
88        "   nop                 \n"
89        ".set reorder           \n"
90        : [val] "+m" (*ptr), [tmp] "=&r" (tmp)
91        : [inc] "r" (incr)
92        : "memory");
93    }
94    else
95    {
96        __asm__ __volatile__(
97        "   saa %[inc], (%[base]) \n"
98        : "+m" (*ptr)
99        : [inc] "r" (incr), [base] "r" (ptr)
100        : "memory");
101    }
102}
103
104/**
105 * Atomically adds a signed value to a 32 bit (aligned) memory location.
106 *
107 * Memory access ordering is enforced before/after the atomic operation,
108 * so no additional 'sync' instructions are required.
109 *
110 *
111 * @param ptr    address in memory to add incr to
112 * @param incr   amount to increment memory location by (signed)
113 */
114static inline void cvmx_atomic_add32(int32_t *ptr, int32_t incr)
115{
116    CVMX_SYNCWS;
117    cvmx_atomic_add32_nosync(ptr, incr);
118    CVMX_SYNCWS;
119}
120
121/**
122 * Atomically sets a 32 bit (aligned) memory location to a value
123 *
124 * @param ptr    address of memory to set
125 * @param value  value to set memory location to.
126 */
127static inline void cvmx_atomic_set32(int32_t *ptr, int32_t value)
128{
129    CVMX_SYNCWS;
130    *ptr = value;
131    CVMX_SYNCWS;
132}
133
134/**
135 * Returns the current value of a 32 bit (aligned) memory
136 * location.
137 *
138 * @param ptr    Address of memory to get
139 * @return Value of the memory
140 */
141static inline int32_t cvmx_atomic_get32(int32_t *ptr)
142{
143    return *(volatile int32_t *)ptr;
144}
145
146/**
147 * Atomically adds a signed value to a 64 bit (aligned) memory location.
148 *
149 * This version does not perform 'sync' operations to enforce memory
150 * operations.  This should only be used when there are no memory operation
151 * ordering constraints.  (This should NOT be used for reference counting -
152 * use the standard version instead.)
153 *
154 * @param ptr    address in memory to add incr to
155 * @param incr   amount to increment memory location by (signed)
156 */
157static inline void cvmx_atomic_add64_nosync(int64_t *ptr, int64_t incr)
158{
159    if (OCTEON_IS_MODEL(OCTEON_CN3XXX))
160    {
161	uint64_t tmp;
162        __asm__ __volatile__(
163        ".set noreorder         \n"
164        "1: lld  %[tmp], %[val] \n"
165        "   daddu %[tmp], %[inc] \n"
166        "   scd  %[tmp], %[val] \n"
167        "   beqz %[tmp], 1b     \n"
168        "   nop                 \n"
169        ".set reorder           \n"
170        : [val] "+m" (*ptr), [tmp] "=&r" (tmp)
171        : [inc] "r" (incr)
172        : "memory");
173    }
174    else
175    {
176        __asm__ __volatile__(
177        "   saad %[inc], (%[base])  \n"
178        : "+m" (*ptr)
179        : [inc] "r" (incr), [base] "r" (ptr)
180        : "memory");
181    }
182}
183
184/**
185 * Atomically adds a signed value to a 64 bit (aligned) memory location.
186 *
187 * Memory access ordering is enforced before/after the atomic operation,
188 * so no additional 'sync' instructions are required.
189 *
190 *
191 * @param ptr    address in memory to add incr to
192 * @param incr   amount to increment memory location by (signed)
193 */
194static inline void cvmx_atomic_add64(int64_t *ptr, int64_t incr)
195{
196    CVMX_SYNCWS;
197    cvmx_atomic_add64_nosync(ptr, incr);
198    CVMX_SYNCWS;
199}
200
201/**
202 * Atomically sets a 64 bit (aligned) memory location to a value
203 *
204 * @param ptr    address of memory to set
205 * @param value  value to set memory location to.
206 */
207static inline void cvmx_atomic_set64(int64_t *ptr, int64_t value)
208{
209    CVMX_SYNCWS;
210    *ptr = value;
211    CVMX_SYNCWS;
212}
213
214/**
215 * Returns the current value of a 64 bit (aligned) memory
216 * location.
217 *
218 * @param ptr    Address of memory to get
219 * @return Value of the memory
220 */
221static inline int64_t cvmx_atomic_get64(int64_t *ptr)
222{
223    return *(volatile int64_t *)ptr;
224}
225
226/**
227 * Atomically compares the old value with the value at ptr, and if they match,
228 * stores new_val to ptr.
229 * If *ptr and old don't match, function returns failure immediately.
230 * If *ptr and old match, function spins until *ptr updated to new atomically, or
231 *  until *ptr and old no longer match
232 *
233 * Does no memory synchronization.
234 *
235 * @return 1 on success (match and store)
236 *         0 on no match
237 */
238static inline uint32_t cvmx_atomic_compare_and_store32_nosync(uint32_t *ptr, uint32_t old_val, uint32_t new_val)
239{
240    uint32_t tmp, ret;
241
242    __asm__ __volatile__(
243    ".set noreorder         \n"
244    "1: ll   %[tmp], %[val] \n"
245    "   li   %[ret], 0     \n"
246    "   bne  %[tmp], %[old], 2f \n"
247    "   move %[tmp], %[new_val] \n"
248    "   sc   %[tmp], %[val] \n"
249    "   beqz %[tmp], 1b     \n"
250    "   li   %[ret], 1      \n"
251    "2: nop               \n"
252    ".set reorder           \n"
253    : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
254    : [old] "r" (old_val), [new_val] "r" (new_val)
255    : "memory");
256
257    return(ret);
258
259}
260
261/**
262 * Atomically compares the old value with the value at ptr, and if they match,
263 * stores new_val to ptr.
264 * If *ptr and old don't match, function returns failure immediately.
265 * If *ptr and old match, function spins until *ptr updated to new atomically, or
266 *  until *ptr and old no longer match
267 *
268 * Does memory synchronization that is required to use this as a locking primitive.
269 *
270 * @return 1 on success (match and store)
271 *         0 on no match
272 */
273static inline uint32_t cvmx_atomic_compare_and_store32(uint32_t *ptr, uint32_t old_val, uint32_t new_val)
274{
275    uint32_t ret;
276    CVMX_SYNCWS;
277    ret = cvmx_atomic_compare_and_store32_nosync(ptr, old_val, new_val);
278    CVMX_SYNCWS;
279    return ret;
280
281
282}
283
284/**
285 * Atomically compares the old value with the value at ptr, and if they match,
286 * stores new_val to ptr.
287 * If *ptr and old don't match, function returns failure immediately.
288 * If *ptr and old match, function spins until *ptr updated to new atomically, or
289 *  until *ptr and old no longer match
290 *
291 * Does no memory synchronization.
292 *
293 * @return 1 on success (match and store)
294 *         0 on no match
295 */
296static inline uint64_t cvmx_atomic_compare_and_store64_nosync(uint64_t *ptr, uint64_t old_val, uint64_t new_val)
297{
298    uint64_t tmp, ret;
299
300    __asm__ __volatile__(
301    ".set noreorder         \n"
302    "1: lld  %[tmp], %[val] \n"
303    "   li   %[ret], 0     \n"
304    "   bne  %[tmp], %[old], 2f \n"
305    "   move %[tmp], %[new_val] \n"
306    "   scd  %[tmp], %[val] \n"
307    "   beqz %[tmp], 1b     \n"
308    "   li   %[ret], 1      \n"
309    "2: nop               \n"
310    ".set reorder           \n"
311    : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
312    : [old] "r" (old_val), [new_val] "r" (new_val)
313    : "memory");
314
315    return(ret);
316
317}
318
319/**
320 * Atomically compares the old value with the value at ptr, and if they match,
321 * stores new_val to ptr.
322 * If *ptr and old don't match, function returns failure immediately.
323 * If *ptr and old match, function spins until *ptr updated to new atomically, or
324 *  until *ptr and old no longer match
325 *
326 * Does memory synchronization that is required to use this as a locking primitive.
327 *
328 * @return 1 on success (match and store)
329 *         0 on no match
330 */
331static inline uint64_t cvmx_atomic_compare_and_store64(uint64_t *ptr, uint64_t old_val, uint64_t new_val)
332{
333    uint64_t ret;
334    CVMX_SYNCWS;
335    ret = cvmx_atomic_compare_and_store64_nosync(ptr, old_val, new_val);
336    CVMX_SYNCWS;
337    return ret;
338}
339
340/**
341 * Atomically adds a signed value to a 64 bit (aligned) memory location,
342 * and returns previous value.
343 *
344 * This version does not perform 'sync' operations to enforce memory
345 * operations.  This should only be used when there are no memory operation
346 * ordering constraints.  (This should NOT be used for reference counting -
347 * use the standard version instead.)
348 *
349 * @param ptr    address in memory to add incr to
350 * @param incr   amount to increment memory location by (signed)
351 *
352 * @return Value of memory location before increment
353 */
354static inline int64_t cvmx_atomic_fetch_and_add64_nosync(int64_t *ptr, int64_t incr)
355{
356    uint64_t tmp, ret;
357
358#if !defined(__FreeBSD__) || !defined(_KERNEL)
359    if (OCTEON_IS_MODEL(OCTEON_CN6XXX))
360    {
361	CVMX_PUSH_OCTEON2;
362	if (__builtin_constant_p(incr) && incr == 1)
363	{
364	    __asm__ __volatile__(
365		"laid  %0,(%2)"
366		: "=r" (ret), "+m" (ptr) : "r" (ptr) : "memory");
367	}
368	else if (__builtin_constant_p(incr) && incr == -1)
369        {
370	    __asm__ __volatile__(
371		"ladd  %0,(%2)"
372		: "=r" (ret), "+m" (ptr) : "r" (ptr) : "memory");
373        }
374        else
375        {
376	    __asm__ __volatile__(
377		"laad  %0,(%2),%3"
378		: "=r" (ret), "+m" (ptr) : "r" (ptr), "r" (incr) : "memory");
379        }
380	CVMX_POP_OCTEON2;
381    }
382    else
383    {
384#endif
385        __asm__ __volatile__(
386            ".set noreorder          \n"
387            "1: lld   %[tmp], %[val] \n"
388            "   move  %[ret], %[tmp] \n"
389            "   daddu %[tmp], %[inc] \n"
390            "   scd   %[tmp], %[val] \n"
391            "   beqz  %[tmp], 1b     \n"
392            "   nop                  \n"
393            ".set reorder            \n"
394            : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
395            : [inc] "r" (incr)
396            : "memory");
397#if !defined(__FreeBSD__) || !defined(_KERNEL)
398    }
399#endif
400
401    return (ret);
402}
403
404/**
405 * Atomically adds a signed value to a 64 bit (aligned) memory location,
406 * and returns previous value.
407 *
408 * Memory access ordering is enforced before/after the atomic operation,
409 * so no additional 'sync' instructions are required.
410 *
411 * @param ptr    address in memory to add incr to
412 * @param incr   amount to increment memory location by (signed)
413 *
414 * @return Value of memory location before increment
415 */
416static inline int64_t cvmx_atomic_fetch_and_add64(int64_t *ptr, int64_t incr)
417{
418    uint64_t ret;
419    CVMX_SYNCWS;
420    ret = cvmx_atomic_fetch_and_add64_nosync(ptr, incr);
421    CVMX_SYNCWS;
422    return ret;
423}
424
425/**
426 * Atomically adds a signed value to a 32 bit (aligned) memory location,
427 * and returns previous value.
428 *
429 * This version does not perform 'sync' operations to enforce memory
430 * operations.  This should only be used when there are no memory operation
431 * ordering constraints.  (This should NOT be used for reference counting -
432 * use the standard version instead.)
433 *
434 * @param ptr    address in memory to add incr to
435 * @param incr   amount to increment memory location by (signed)
436 *
437 * @return Value of memory location before increment
438 */
439static inline int32_t cvmx_atomic_fetch_and_add32_nosync(int32_t *ptr, int32_t incr)
440{
441    uint32_t tmp, ret;
442
443#if !defined(__FreeBSD__) || !defined(_KERNEL)
444    if (OCTEON_IS_MODEL(OCTEON_CN6XXX))
445    {
446	CVMX_PUSH_OCTEON2;
447	if (__builtin_constant_p(incr) && incr == 1)
448	{
449	    __asm__ __volatile__(
450		"lai  %0,(%2)"
451		: "=r" (ret), "+m" (ptr) : "r" (ptr) : "memory");
452	}
453	else if (__builtin_constant_p(incr) && incr == -1)
454        {
455	    __asm__ __volatile__(
456		"lad  %0,(%2)"
457		: "=r" (ret), "+m" (ptr) : "r" (ptr) : "memory");
458        }
459        else
460        {
461	    __asm__ __volatile__(
462		"laa  %0,(%2),%3"
463		: "=r" (ret), "+m" (ptr) : "r" (ptr), "r" (incr) : "memory");
464        }
465	CVMX_POP_OCTEON2;
466    }
467    else
468    {
469#endif
470        __asm__ __volatile__(
471            ".set noreorder         \n"
472            "1: ll   %[tmp], %[val] \n"
473            "   move %[ret], %[tmp] \n"
474            "   addu %[tmp], %[inc] \n"
475            "   sc   %[tmp], %[val] \n"
476            "   beqz %[tmp], 1b     \n"
477            "   nop                 \n"
478            ".set reorder           \n"
479            : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
480            : [inc] "r" (incr)
481            : "memory");
482#if !defined(__FreeBSD__) || !defined(_KERNEL)
483    }
484#endif
485
486    return (ret);
487}
488
489/**
490 * Atomically adds a signed value to a 32 bit (aligned) memory location,
491 * and returns previous value.
492 *
493 * Memory access ordering is enforced before/after the atomic operation,
494 * so no additional 'sync' instructions are required.
495 *
496 * @param ptr    address in memory to add incr to
497 * @param incr   amount to increment memory location by (signed)
498 *
499 * @return Value of memory location before increment
500 */
501static inline int32_t cvmx_atomic_fetch_and_add32(int32_t *ptr, int32_t incr)
502{
503    uint32_t ret;
504    CVMX_SYNCWS;
505    ret = cvmx_atomic_fetch_and_add32_nosync(ptr, incr);
506    CVMX_SYNCWS;
507    return ret;
508}
509
510/**
511 * Atomically set bits in a 64 bit (aligned) memory location,
512 * and returns previous value.
513 *
514 * This version does not perform 'sync' operations to enforce memory
515 * operations.  This should only be used when there are no memory operation
516 * ordering constraints.
517 *
518 * @param ptr    address in memory
519 * @param mask   mask of bits to set
520 *
521 * @return Value of memory location before setting bits
522 */
523static inline uint64_t cvmx_atomic_fetch_and_bset64_nosync(uint64_t *ptr, uint64_t mask)
524{
525    uint64_t tmp, ret;
526
527    __asm__ __volatile__(
528    ".set noreorder         \n"
529    "1: lld  %[tmp], %[val] \n"
530    "   move %[ret], %[tmp] \n"
531    "   or   %[tmp], %[msk] \n"
532    "   scd  %[tmp], %[val] \n"
533    "   beqz %[tmp], 1b     \n"
534    "   nop                 \n"
535    ".set reorder           \n"
536    : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
537    : [msk] "r" (mask)
538    : "memory");
539
540    return (ret);
541}
542
543/**
544 * Atomically set bits in a 32 bit (aligned) memory location,
545 * and returns previous value.
546 *
547 * This version does not perform 'sync' operations to enforce memory
548 * operations.  This should only be used when there are no memory operation
549 * ordering constraints.
550 *
551 * @param ptr    address in memory
552 * @param mask   mask of bits to set
553 *
554 * @return Value of memory location before setting bits
555 */
556static inline uint32_t cvmx_atomic_fetch_and_bset32_nosync(uint32_t *ptr, uint32_t mask)
557{
558    uint32_t tmp, ret;
559
560    __asm__ __volatile__(
561    ".set noreorder         \n"
562    "1: ll   %[tmp], %[val] \n"
563    "   move %[ret], %[tmp] \n"
564    "   or   %[tmp], %[msk] \n"
565    "   sc   %[tmp], %[val] \n"
566    "   beqz %[tmp], 1b     \n"
567    "   nop                 \n"
568    ".set reorder           \n"
569    : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
570    : [msk] "r" (mask)
571    : "memory");
572
573    return (ret);
574}
575
576/**
577 * Atomically clear bits in a 64 bit (aligned) memory location,
578 * and returns previous value.
579 *
580 * This version does not perform 'sync' operations to enforce memory
581 * operations.  This should only be used when there are no memory operation
582 * ordering constraints.
583 *
584 * @param ptr    address in memory
585 * @param mask   mask of bits to clear
586 *
587 * @return Value of memory location before clearing bits
588 */
589static inline uint64_t cvmx_atomic_fetch_and_bclr64_nosync(uint64_t *ptr, uint64_t mask)
590{
591    uint64_t tmp, ret;
592
593    __asm__ __volatile__(
594    ".set noreorder         \n"
595    "   nor  %[msk], 0      \n"
596    "1: lld  %[tmp], %[val] \n"
597    "   move %[ret], %[tmp] \n"
598    "   and  %[tmp], %[msk] \n"
599    "   scd  %[tmp], %[val] \n"
600    "   beqz %[tmp], 1b     \n"
601    "   nop                 \n"
602    ".set reorder           \n"
603    : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret), [msk] "+r" (mask)
604    : : "memory");
605
606    return (ret);
607}
608
609/**
610 * Atomically clear bits in a 32 bit (aligned) memory location,
611 * and returns previous value.
612 *
613 * This version does not perform 'sync' operations to enforce memory
614 * operations.  This should only be used when there are no memory operation
615 * ordering constraints.
616 *
617 * @param ptr    address in memory
618 * @param mask   mask of bits to clear
619 *
620 * @return Value of memory location before clearing bits
621 */
622static inline uint32_t cvmx_atomic_fetch_and_bclr32_nosync(uint32_t *ptr, uint32_t mask)
623{
624    uint32_t tmp, ret;
625
626    __asm__ __volatile__(
627    ".set noreorder         \n"
628    "   nor  %[msk], 0      \n"
629    "1: ll   %[tmp], %[val] \n"
630    "   move %[ret], %[tmp] \n"
631    "   and  %[tmp], %[msk] \n"
632    "   sc   %[tmp], %[val] \n"
633    "   beqz %[tmp], 1b     \n"
634    "   nop                 \n"
635    ".set reorder           \n"
636    : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret), [msk] "+r" (mask)
637    : : "memory");
638
639    return (ret);
640}
641
642/**
643 * Atomically swaps value in 64 bit (aligned) memory location,
644 * and returns previous value.
645 *
646 * This version does not perform 'sync' operations to enforce memory
647 * operations.  This should only be used when there are no memory operation
648 * ordering constraints.
649 *
650 * @param ptr       address in memory
651 * @param new_val   new value to write
652 *
653 * @return Value of memory location before swap operation
654 */
655static inline uint64_t cvmx_atomic_swap64_nosync(uint64_t *ptr, uint64_t new_val)
656{
657    uint64_t tmp, ret;
658
659#if !defined(__FreeBSD__) || !defined(_KERNEL)
660    if (OCTEON_IS_MODEL(OCTEON_CN6XXX))
661    {
662	CVMX_PUSH_OCTEON2;
663	if (__builtin_constant_p(new_val) && new_val == 0)
664	{
665	    __asm__ __volatile__(
666		"lacd  %0,(%1)"
667		: "=r" (ret) : "r" (ptr) : "memory");
668	}
669	else if (__builtin_constant_p(new_val) && new_val == ~0ull)
670        {
671	    __asm__ __volatile__(
672		"lasd  %0,(%1)"
673		: "=r" (ret) : "r" (ptr) : "memory");
674        }
675        else
676        {
677	    __asm__ __volatile__(
678		"lawd  %0,(%1),%2"
679		: "=r" (ret) : "r" (ptr), "r" (new_val) : "memory");
680        }
681	CVMX_POP_OCTEON2;
682    }
683    else
684    {
685#endif
686        __asm__ __volatile__(
687            ".set noreorder         \n"
688            "1: lld  %[ret], %[val] \n"
689            "   move %[tmp], %[new_val] \n"
690            "   scd  %[tmp], %[val] \n"
691            "   beqz %[tmp],  1b    \n"
692            "   nop                 \n"
693            ".set reorder           \n"
694            : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
695            : [new_val] "r"  (new_val)
696            : "memory");
697#if !defined(__FreeBSD__) || !defined(_KERNEL)
698    }
699#endif
700
701    return (ret);
702}
703
704/**
705 * Atomically swaps value in 32 bit (aligned) memory location,
706 * and returns previous value.
707 *
708 * This version does not perform 'sync' operations to enforce memory
709 * operations.  This should only be used when there are no memory operation
710 * ordering constraints.
711 *
712 * @param ptr       address in memory
713 * @param new_val   new value to write
714 *
715 * @return Value of memory location before swap operation
716 */
717static inline uint32_t cvmx_atomic_swap32_nosync(uint32_t *ptr, uint32_t new_val)
718{
719    uint32_t tmp, ret;
720
721#if !defined(__FreeBSD__) || !defined(_KERNEL)
722    if (OCTEON_IS_MODEL(OCTEON_CN6XXX))
723    {
724	CVMX_PUSH_OCTEON2;
725	if (__builtin_constant_p(new_val) && new_val == 0)
726	{
727	    __asm__ __volatile__(
728		"lac  %0,(%1)"
729		: "=r" (ret) : "r" (ptr) : "memory");
730	}
731	else if (__builtin_constant_p(new_val) && new_val == ~0u)
732        {
733	    __asm__ __volatile__(
734		"las  %0,(%1)"
735		: "=r" (ret) : "r" (ptr) : "memory");
736        }
737        else
738        {
739	    __asm__ __volatile__(
740		"law  %0,(%1),%2"
741		: "=r" (ret) : "r" (ptr), "r" (new_val) : "memory");
742        }
743	CVMX_POP_OCTEON2;
744    }
745    else
746    {
747#endif
748        __asm__ __volatile__(
749        ".set noreorder         \n"
750        "1: ll   %[ret], %[val] \n"
751        "   move %[tmp], %[new_val] \n"
752        "   sc   %[tmp], %[val] \n"
753        "   beqz %[tmp],  1b    \n"
754        "   nop                 \n"
755        ".set reorder           \n"
756        : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret)
757        : [new_val] "r"  (new_val)
758        : "memory");
759#if !defined(__FreeBSD__) || !defined(_KERNEL)
760    }
761#endif
762
763    return (ret);
764}
765
766/**
767 * This atomic operation is now named cvmx_atomic_compare_and_store32_nosync
768 * and the (deprecated) macro is provided for backward compatibility.
769 * @deprecated
770 */
771#define cvmx_atomic_compare_and_store_nosync32 cvmx_atomic_compare_and_store32_nosync
772
773/**
774 * This atomic operation is now named cvmx_atomic_compare_and_store64_nosync
775 * and the (deprecated) macro is provided for backward compatibility.
776 * @deprecated
777 */
778#define cvmx_atomic_compare_and_store_nosync64 cvmx_atomic_compare_and_store64_nosync
779
780
781
782#ifdef	__cplusplus
783}
784#endif
785
786#endif /* __CVMX_ATOMIC_H__ */
787