atomic.h revision 327195 
1/*-
2 * Copyright (c) 2013 Andrew Turner <andrew@freebsd.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 * $FreeBSD: stable/11/sys/arm64/include/atomic.h 327195 2017-12-26 10:07:17Z kib $
27 */
28
29#ifndef	_MACHINE_ATOMIC_H_
30#define	_MACHINE_ATOMIC_H_
31
32#include <sys/atomic_common.h>
33
34#define	isb()		__asm __volatile("isb" : : : "memory")
35
36/*
37 * Options for DMB and DSB:
38 *	oshld	Outer Shareable, load
39 *	oshst	Outer Shareable, store
40 *	osh	Outer Shareable, all
41 *	nshld	Non-shareable, load
42 *	nshst	Non-shareable, store
43 *	nsh	Non-shareable, all
44 *	ishld	Inner Shareable, load
45 *	ishst	Inner Shareable, store
46 *	ish	Inner Shareable, all
47 *	ld	Full system, load
48 *	st	Full system, store
49 *	sy	Full system, all
50 */
51#define	dsb(opt)	__asm __volatile("dsb " __STRING(opt) : : : "memory")
52#define	dmb(opt)	__asm __volatile("dmb " __STRING(opt) : : : "memory")
53
54#define	mb()	dmb(sy)	/* Full system memory barrier all */
55#define	wmb()	dmb(st)	/* Full system memory barrier store */
56#define	rmb()	dmb(ld)	/* Full system memory barrier load */
57
58#define	ATOMIC_OP(op, asm_op, bar, a, l)				\
59static __inline void							\
60atomic_##op##_##bar##32(volatile uint32_t *p, uint32_t val)		\
61{									\
62	uint32_t tmp;							\
63	int res;							\
64									\
65	__asm __volatile(						\
66	    "1: ld"#a"xr   %w0, [%2]      \n"				\
67	    "   "#asm_op"  %w0, %w0, %w3  \n"				\
68	    "   st"#l"xr   %w1, %w0, [%2] \n"				\
69            "   cbnz       %w1, 1b        \n"				\
70	    : "=&r"(tmp), "=&r"(res)					\
71	    : "r" (p), "r" (val)					\
72	    : "memory"							\
73	);								\
74}									\
75									\
76static __inline void							\
77atomic_##op##_##bar##64(volatile uint64_t *p, uint64_t val)		\
78{									\
79	uint64_t tmp;							\
80	int res;							\
81									\
82	__asm __volatile(						\
83	    "1: ld"#a"xr   %0, [%2]      \n"				\
84	    "   "#asm_op"  %0, %0, %3    \n"				\
85	    "   st"#l"xr   %w1, %0, [%2] \n"				\
86            "   cbnz       %w1, 1b       \n"				\
87	    : "=&r"(tmp), "=&r"(res)					\
88	    : "r" (p), "r" (val)					\
89	    : "memory"							\
90	);								\
91}
92
93#define	ATOMIC(op, asm_op)						\
94    ATOMIC_OP(op, asm_op,     ,  ,  )					\
95    ATOMIC_OP(op, asm_op, acq_, a,  )					\
96    ATOMIC_OP(op, asm_op, rel_,  , l)					\
97
98ATOMIC(add,      add)
99ATOMIC(clear,    bic)
100ATOMIC(set,      orr)
101ATOMIC(subtract, sub)
102
103#define	ATOMIC_FCMPSET(bar, a, l)					\
104static __inline int							\
105atomic_fcmpset_##bar##32(volatile uint32_t *p, uint32_t *cmpval,	\
106    uint32_t newval)		 					\
107{									\
108	uint32_t tmp;							\
109	uint32_t _cmpval = *cmpval;					\
110	int res;							\
111									\
112	__asm __volatile(						\
113	    "1: mov      %w1, #1        \n"				\
114	    "   ld"#a"xr %w0, [%2]      \n"				\
115	    "   cmp      %w0, %w3       \n"				\
116	    "   b.ne     2f             \n"				\
117	    "   st"#l"xr %w1, %w4, [%2] \n"				\
118	    "2:"							\
119	    : "=&r"(tmp), "=&r"(res)					\
120	    : "r" (p), "r" (_cmpval), "r" (newval)			\
121	    : "cc", "memory"						\
122	);								\
123	*cmpval = tmp;							\
124									\
125	return (!res);							\
126}									\
127									\
128static __inline int							\
129atomic_fcmpset_##bar##64(volatile uint64_t *p, uint64_t *cmpval,	\
130    uint64_t newval)							\
131{									\
132	uint64_t tmp;							\
133	uint64_t _cmpval = *cmpval;					\
134	int res;							\
135									\
136	__asm __volatile(						\
137	    "1: mov      %w1, #1       \n"				\
138	    "   ld"#a"xr %0, [%2]      \n"				\
139	    "   cmp      %0, %3        \n"				\
140	    "   b.ne     2f            \n"				\
141	    "   st"#l"xr %w1, %4, [%2] \n"				\
142	    "2:"							\
143	    : "=&r"(tmp), "=&r"(res)					\
144	    : "r" (p), "r" (_cmpval), "r" (newval)			\
145	    : "cc", "memory"						\
146	);								\
147	*cmpval = tmp;							\
148									\
149	return (!res);							\
150}
151
152ATOMIC_FCMPSET(    ,  , )
153ATOMIC_FCMPSET(acq_, a, )
154ATOMIC_FCMPSET(rel_,  ,l)
155
156#undef ATOMIC_FCMPSET
157
158#define	ATOMIC_CMPSET(bar, a, l)					\
159static __inline int							\
160atomic_cmpset_##bar##32(volatile uint32_t *p, uint32_t cmpval,		\
161    uint32_t newval)							\
162{									\
163	uint32_t tmp;							\
164	int res;							\
165									\
166	__asm __volatile(						\
167	    "1: mov      %w1, #1        \n"				\
168	    "   ld"#a"xr %w0, [%2]      \n"				\
169	    "   cmp      %w0, %w3       \n"				\
170	    "   b.ne     2f             \n"				\
171	    "   st"#l"xr %w1, %w4, [%2] \n"				\
172            "   cbnz     %w1, 1b        \n"				\
173	    "2:"							\
174	    : "=&r"(tmp), "=&r"(res)					\
175	    : "r" (p), "r" (cmpval), "r" (newval)			\
176	    : "cc", "memory"							\
177	);								\
178									\
179	return (!res);							\
180}									\
181									\
182static __inline int							\
183atomic_cmpset_##bar##64(volatile uint64_t *p, uint64_t cmpval,		\
184    uint64_t newval)							\
185{									\
186	uint64_t tmp;							\
187	int res;							\
188									\
189	__asm __volatile(						\
190	    "1: mov      %w1, #1       \n"				\
191	    "   ld"#a"xr %0, [%2]      \n"				\
192	    "   cmp      %0, %3        \n"				\
193	    "   b.ne     2f            \n"				\
194	    "   st"#l"xr %w1, %4, [%2] \n"				\
195            "   cbnz     %w1, 1b       \n"				\
196	    "2:"							\
197	    : "=&r"(tmp), "=&r"(res)					\
198	    : "r" (p), "r" (cmpval), "r" (newval)			\
199	    : "cc", "memory"							\
200	);								\
201									\
202	return (!res);							\
203}
204
205ATOMIC_CMPSET(    ,  , )
206ATOMIC_CMPSET(acq_, a, )
207ATOMIC_CMPSET(rel_,  ,l)
208
209static __inline uint32_t
210atomic_fetchadd_32(volatile uint32_t *p, uint32_t val)
211{
212	uint32_t tmp, ret;
213	int res;
214
215	__asm __volatile(
216	    "1: ldxr	%w2, [%3]      \n"
217	    "   add	%w0, %w2, %w4  \n"
218	    "   stxr	%w1, %w0, [%3] \n"
219            "   cbnz	%w1, 1b        \n"
220	    : "=&r"(tmp), "=&r"(res), "=&r"(ret)
221	    : "r" (p), "r" (val)
222	    : "memory"
223	);
224
225	return (ret);
226}
227
228static __inline uint64_t
229atomic_fetchadd_64(volatile uint64_t *p, uint64_t val)
230{
231	uint64_t tmp, ret;
232	int res;
233
234	__asm __volatile(
235	    "1: ldxr	%2, [%3]      \n"
236	    "   add	%0, %2, %4    \n"
237	    "   stxr	%w1, %0, [%3] \n"
238            "   cbnz	%w1, 1b       \n"
239	    : "=&r"(tmp), "=&r"(res), "=&r"(ret)
240	    : "r" (p), "r" (val)
241	    : "memory"
242	);
243
244	return (ret);
245}
246
247static __inline uint32_t
248atomic_readandclear_32(volatile uint32_t *p)
249{
250	uint32_t ret;
251	int res;
252
253	__asm __volatile(
254	    "1: ldxr	%w1, [%2]      \n"
255	    "   stxr	%w0, wzr, [%2] \n"
256            "   cbnz	%w0, 1b        \n"
257	    : "=&r"(res), "=&r"(ret)
258	    : "r" (p)
259	    : "memory"
260	);
261
262	return (ret);
263}
264
265static __inline uint64_t
266atomic_readandclear_64(volatile uint64_t *p)
267{
268	uint64_t ret;
269	int res;
270
271	__asm __volatile(
272	    "1: ldxr	%1, [%2]      \n"
273	    "   stxr	%w0, xzr, [%2] \n"
274            "   cbnz	%w0, 1b        \n"
275	    : "=&r"(res), "=&r"(ret)
276	    : "r" (p)
277	    : "memory"
278	);
279
280	return (ret);
281}
282
283static __inline uint32_t
284atomic_swap_32(volatile uint32_t *p, uint32_t val)
285{
286	uint32_t ret;
287	int res;
288
289	__asm __volatile(
290	    "1: ldxr	%w0, [%2]      \n"
291	    "   stxr	%w1, %w3, [%2] \n"
292	    "   cbnz	%w1, 1b        \n"
293	    : "=&r"(ret), "=&r"(res)
294	    : "r" (p), "r" (val)
295	    : "memory"
296	);
297
298	return (ret);
299}
300
301static __inline uint64_t
302atomic_swap_64(volatile uint64_t *p, uint64_t val)
303{
304	uint64_t ret;
305	int res;
306
307	__asm __volatile(
308	    "1: ldxr	%0, [%2]      \n"
309	    "   stxr	%w1, %3, [%2] \n"
310	    "   cbnz	%w1, 1b       \n"
311	    : "=&r"(ret), "=&r"(res)
312	    : "r" (p), "r" (val)
313	    : "memory"
314	);
315
316	return (ret);
317}
318
319static __inline uint32_t
320atomic_load_acq_32(volatile uint32_t *p)
321{
322	uint32_t ret;
323
324	__asm __volatile(
325	    "ldar	%w0, [%1] \n"
326	    : "=&r" (ret)
327	    : "r" (p)
328	    : "memory");
329
330	return (ret);
331}
332
333static __inline uint64_t
334atomic_load_acq_64(volatile uint64_t *p)
335{
336	uint64_t ret;
337
338	__asm __volatile(
339	    "ldar	%0, [%1] \n"
340	    : "=&r" (ret)
341	    : "r" (p)
342	    : "memory");
343
344	return (ret);
345}
346
347static __inline void
348atomic_store_rel_32(volatile uint32_t *p, uint32_t val)
349{
350
351	__asm __volatile(
352	    "stlr	%w0, [%1] \n"
353	    :
354	    : "r" (val), "r" (p)
355	    : "memory");
356}
357
358static __inline void
359atomic_store_rel_64(volatile uint64_t *p, uint64_t val)
360{
361
362	__asm __volatile(
363	    "stlr	%0, [%1] \n"
364	    :
365	    : "r" (val), "r" (p)
366	    : "memory");
367}
368
369
370#define	atomic_add_int			atomic_add_32
371#define	atomic_fcmpset_int		atomic_fcmpset_32
372#define	atomic_clear_int		atomic_clear_32
373#define	atomic_cmpset_int		atomic_cmpset_32
374#define	atomic_fetchadd_int		atomic_fetchadd_32
375#define	atomic_readandclear_int		atomic_readandclear_32
376#define	atomic_set_int			atomic_set_32
377#define	atomic_swap_int			atomic_swap_32
378#define	atomic_subtract_int		atomic_subtract_32
379
380#define	atomic_add_acq_int		atomic_add_acq_32
381#define	atomic_fcmpset_acq_int		atomic_fcmpset_acq_32
382#define	atomic_clear_acq_int		atomic_clear_acq_32
383#define	atomic_cmpset_acq_int		atomic_cmpset_acq_32
384#define	atomic_load_acq_int		atomic_load_acq_32
385#define	atomic_set_acq_int		atomic_set_acq_32
386#define	atomic_subtract_acq_int		atomic_subtract_acq_32
387
388#define	atomic_add_rel_int		atomic_add_rel_32
389#define	atomic_fcmpset_rel_int		atomic_fcmpset_rel_32
390#define	atomic_clear_rel_int		atomic_clear_rel_32
391#define	atomic_cmpset_rel_int		atomic_cmpset_rel_32
392#define	atomic_set_rel_int		atomic_set_rel_32
393#define	atomic_subtract_rel_int		atomic_subtract_rel_32
394#define	atomic_store_rel_int		atomic_store_rel_32
395
396#define	atomic_add_long			atomic_add_64
397#define	atomic_fcmpset_long		atomic_fcmpset_64
398#define	atomic_clear_long		atomic_clear_64
399#define	atomic_cmpset_long		atomic_cmpset_64
400#define	atomic_fetchadd_long		atomic_fetchadd_64
401#define	atomic_readandclear_long	atomic_readandclear_64
402#define	atomic_set_long			atomic_set_64
403#define	atomic_swap_long		atomic_swap_64
404#define	atomic_subtract_long		atomic_subtract_64
405
406#define	atomic_add_ptr			atomic_add_64
407#define	atomic_fcmpset_ptr		atomic_fcmpset_64
408#define	atomic_clear_ptr		atomic_clear_64
409#define	atomic_cmpset_ptr		atomic_cmpset_64
410#define	atomic_fetchadd_ptr		atomic_fetchadd_64
411#define	atomic_readandclear_ptr		atomic_readandclear_64
412#define	atomic_set_ptr			atomic_set_64
413#define	atomic_swap_ptr			atomic_swap_64
414#define	atomic_subtract_ptr		atomic_subtract_64
415
416#define	atomic_add_acq_long		atomic_add_acq_64
417#define	atomic_fcmpset_acq_long		atomic_fcmpset_acq_64
418#define	atomic_clear_acq_long		atomic_clear_acq_64
419#define	atomic_cmpset_acq_long		atomic_cmpset_acq_64
420#define	atomic_load_acq_long		atomic_load_acq_64
421#define	atomic_set_acq_long		atomic_set_acq_64
422#define	atomic_subtract_acq_long	atomic_subtract_acq_64
423
424#define	atomic_add_acq_ptr		atomic_add_acq_64
425#define	atomic_fcmpset_acq_ptr		atomic_fcmpset_acq_64
426#define	atomic_clear_acq_ptr		atomic_clear_acq_64
427#define	atomic_cmpset_acq_ptr		atomic_cmpset_acq_64
428#define	atomic_load_acq_ptr		atomic_load_acq_64
429#define	atomic_set_acq_ptr		atomic_set_acq_64
430#define	atomic_subtract_acq_ptr		atomic_subtract_acq_64
431
432#define	atomic_add_rel_long		atomic_add_rel_64
433#define	atomic_fcmpset_rel_long		atomic_fcmpset_rel_64
434#define	atomic_clear_rel_long		atomic_clear_rel_64
435#define	atomic_cmpset_rel_long		atomic_cmpset_rel_64
436#define	atomic_set_rel_long		atomic_set_rel_64
437#define	atomic_subtract_rel_long	atomic_subtract_rel_64
438#define	atomic_store_rel_long		atomic_store_rel_64
439
440#define	atomic_add_rel_ptr		atomic_add_rel_64
441#define	atomic_fcmpset_rel_ptr		atomic_fcmpset_rel_64
442#define	atomic_clear_rel_ptr		atomic_clear_rel_64
443#define	atomic_cmpset_rel_ptr		atomic_cmpset_rel_64
444#define	atomic_set_rel_ptr		atomic_set_rel_64
445#define	atomic_subtract_rel_ptr		atomic_subtract_rel_64
446#define	atomic_store_rel_ptr		atomic_store_rel_64
447
448static __inline void
449atomic_thread_fence_acq(void)
450{
451
452	dmb(ld);
453}
454
455static __inline void
456atomic_thread_fence_rel(void)
457{
458
459	dmb(sy);
460}
461
462static __inline void
463atomic_thread_fence_acq_rel(void)
464{
465
466	dmb(sy);
467}
468
469static __inline void
470atomic_thread_fence_seq_cst(void)
471{
472
473	dmb(sy);
474}
475
476#endif /* _MACHINE_ATOMIC_H_ */
477
478