atomic.h revision 315371 
1/*-
2 * Copyright (c) 2013 Andrew Turner <andrew@freebsd.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 * $FreeBSD: stable/11/sys/arm64/include/atomic.h 315371 2017-03-16 06:00:27Z mjg $
27 */
28
29#ifndef	_MACHINE_ATOMIC_H_
30#define	_MACHINE_ATOMIC_H_
31
32#define	isb()		__asm __volatile("isb" : : : "memory")
33
34/*
35 * Options for DMB and DSB:
36 *	oshld	Outer Shareable, load
37 *	oshst	Outer Shareable, store
38 *	osh	Outer Shareable, all
39 *	nshld	Non-shareable, load
40 *	nshst	Non-shareable, store
41 *	nsh	Non-shareable, all
42 *	ishld	Inner Shareable, load
43 *	ishst	Inner Shareable, store
44 *	ish	Inner Shareable, all
45 *	ld	Full system, load
46 *	st	Full system, store
47 *	sy	Full system, all
48 */
49#define	dsb(opt)	__asm __volatile("dsb " __STRING(opt) : : : "memory")
50#define	dmb(opt)	__asm __volatile("dmb " __STRING(opt) : : : "memory")
51
52#define	mb()	dmb(sy)	/* Full system memory barrier all */
53#define	wmb()	dmb(st)	/* Full system memory barrier store */
54#define	rmb()	dmb(ld)	/* Full system memory barrier load */
55
56#define	ATOMIC_OP(op, asm_op, bar, a, l)				\
57static __inline void							\
58atomic_##op##_##bar##32(volatile uint32_t *p, uint32_t val)		\
59{									\
60	uint32_t tmp;							\
61	int res;							\
62									\
63	__asm __volatile(						\
64	    "1: ld"#a"xr   %w0, [%2]      \n"				\
65	    "   "#asm_op"  %w0, %w0, %w3  \n"				\
66	    "   st"#l"xr   %w1, %w0, [%2] \n"				\
67            "   cbnz       %w1, 1b        \n"				\
68	    : "=&r"(tmp), "=&r"(res)					\
69	    : "r" (p), "r" (val)					\
70	    : "memory"							\
71	);								\
72}									\
73									\
74static __inline void							\
75atomic_##op##_##bar##64(volatile uint64_t *p, uint64_t val)		\
76{									\
77	uint64_t tmp;							\
78	int res;							\
79									\
80	__asm __volatile(						\
81	    "1: ld"#a"xr   %0, [%2]      \n"				\
82	    "   "#asm_op"  %0, %0, %3    \n"				\
83	    "   st"#l"xr   %w1, %0, [%2] \n"				\
84            "   cbnz       %w1, 1b       \n"				\
85	    : "=&r"(tmp), "=&r"(res)					\
86	    : "r" (p), "r" (val)					\
87	    : "memory"							\
88	);								\
89}
90
91#define	ATOMIC(op, asm_op)						\
92    ATOMIC_OP(op, asm_op,     ,  ,  )					\
93    ATOMIC_OP(op, asm_op, acq_, a,  )					\
94    ATOMIC_OP(op, asm_op, rel_,  , l)					\
95
96ATOMIC(add,      add)
97ATOMIC(clear,    bic)
98ATOMIC(set,      orr)
99ATOMIC(subtract, sub)
100
101#define	ATOMIC_FCMPSET(bar, a, l)					\
102static __inline int							\
103atomic_fcmpset_##bar##32(volatile uint32_t *p, uint32_t *cmpval,	\
104    uint32_t newval)		 					\
105{									\
106	uint32_t tmp;							\
107	uint32_t _cmpval = *cmpval;					\
108	int res;							\
109									\
110	__asm __volatile(						\
111	    "1: mov      %w1, #1        \n"				\
112	    "   ld"#a"xr %w0, [%2]      \n"				\
113	    "   cmp      %w0, %w3       \n"				\
114	    "   b.ne     2f             \n"				\
115	    "   st"#l"xr %w1, %w4, [%2] \n"				\
116	    "2:"							\
117	    : "=&r"(tmp), "=&r"(res)					\
118	    : "r" (p), "r" (_cmpval), "r" (newval)			\
119	    : "cc", "memory"						\
120	);								\
121	*cmpval = tmp;							\
122									\
123	return (!res);							\
124}									\
125									\
126static __inline int							\
127atomic_fcmpset_##bar##64(volatile uint64_t *p, uint64_t *cmpval,	\
128    uint64_t newval)							\
129{									\
130	uint64_t tmp;							\
131	uint64_t _cmpval = *cmpval;					\
132	int res;							\
133									\
134	__asm __volatile(						\
135	    "1: mov      %w1, #1       \n"				\
136	    "   ld"#a"xr %0, [%2]      \n"				\
137	    "   cmp      %0, %3        \n"				\
138	    "   b.ne     2f            \n"				\
139	    "   st"#l"xr %w1, %4, [%2] \n"				\
140	    "2:"							\
141	    : "=&r"(tmp), "=&r"(res)					\
142	    : "r" (p), "r" (_cmpval), "r" (newval)			\
143	    : "cc", "memory"						\
144	);								\
145	*cmpval = tmp;							\
146									\
147	return (!res);							\
148}
149
150ATOMIC_FCMPSET(    ,  , )
151ATOMIC_FCMPSET(acq_, a, )
152ATOMIC_FCMPSET(rel_,  ,l)
153
154#undef ATOMIC_FCMPSET
155
156#define	ATOMIC_CMPSET(bar, a, l)					\
157static __inline int							\
158atomic_cmpset_##bar##32(volatile uint32_t *p, uint32_t cmpval,		\
159    uint32_t newval)							\
160{									\
161	uint32_t tmp;							\
162	int res;							\
163									\
164	__asm __volatile(						\
165	    "1: mov      %w1, #1        \n"				\
166	    "   ld"#a"xr %w0, [%2]      \n"				\
167	    "   cmp      %w0, %w3       \n"				\
168	    "   b.ne     2f             \n"				\
169	    "   st"#l"xr %w1, %w4, [%2] \n"				\
170            "   cbnz     %w1, 1b        \n"				\
171	    "2:"							\
172	    : "=&r"(tmp), "=&r"(res)					\
173	    : "r" (p), "r" (cmpval), "r" (newval)			\
174	    : "cc", "memory"							\
175	);								\
176									\
177	return (!res);							\
178}									\
179									\
180static __inline int							\
181atomic_cmpset_##bar##64(volatile uint64_t *p, uint64_t cmpval,		\
182    uint64_t newval)							\
183{									\
184	uint64_t tmp;							\
185	int res;							\
186									\
187	__asm __volatile(						\
188	    "1: mov      %w1, #1       \n"				\
189	    "   ld"#a"xr %0, [%2]      \n"				\
190	    "   cmp      %0, %3        \n"				\
191	    "   b.ne     2f            \n"				\
192	    "   st"#l"xr %w1, %4, [%2] \n"				\
193            "   cbnz     %w1, 1b       \n"				\
194	    "2:"							\
195	    : "=&r"(tmp), "=&r"(res)					\
196	    : "r" (p), "r" (cmpval), "r" (newval)			\
197	    : "cc", "memory"							\
198	);								\
199									\
200	return (!res);							\
201}
202
203ATOMIC_CMPSET(    ,  , )
204ATOMIC_CMPSET(acq_, a, )
205ATOMIC_CMPSET(rel_,  ,l)
206
207static __inline uint32_t
208atomic_fetchadd_32(volatile uint32_t *p, uint32_t val)
209{
210	uint32_t tmp, ret;
211	int res;
212
213	__asm __volatile(
214	    "1: ldxr	%w2, [%3]      \n"
215	    "   add	%w0, %w2, %w4  \n"
216	    "   stxr	%w1, %w0, [%3] \n"
217            "   cbnz	%w1, 1b        \n"
218	    : "=&r"(tmp), "=&r"(res), "=&r"(ret)
219	    : "r" (p), "r" (val)
220	    : "memory"
221	);
222
223	return (ret);
224}
225
226static __inline uint64_t
227atomic_fetchadd_64(volatile uint64_t *p, uint64_t val)
228{
229	uint64_t tmp, ret;
230	int res;
231
232	__asm __volatile(
233	    "1: ldxr	%2, [%3]      \n"
234	    "   add	%0, %2, %4    \n"
235	    "   stxr	%w1, %0, [%3] \n"
236            "   cbnz	%w1, 1b       \n"
237	    : "=&r"(tmp), "=&r"(res), "=&r"(ret)
238	    : "r" (p), "r" (val)
239	    : "memory"
240	);
241
242	return (ret);
243}
244
245static __inline uint32_t
246atomic_readandclear_32(volatile uint32_t *p)
247{
248	uint32_t ret;
249	int res;
250
251	__asm __volatile(
252	    "1: ldxr	%w1, [%2]      \n"
253	    "   stxr	%w0, wzr, [%2] \n"
254            "   cbnz	%w0, 1b        \n"
255	    : "=&r"(res), "=&r"(ret)
256	    : "r" (p)
257	    : "memory"
258	);
259
260	return (ret);
261}
262
263static __inline uint64_t
264atomic_readandclear_64(volatile uint64_t *p)
265{
266	uint64_t ret;
267	int res;
268
269	__asm __volatile(
270	    "1: ldxr	%1, [%2]      \n"
271	    "   stxr	%w0, xzr, [%2] \n"
272            "   cbnz	%w0, 1b        \n"
273	    : "=&r"(res), "=&r"(ret)
274	    : "r" (p)
275	    : "memory"
276	);
277
278	return (ret);
279}
280
281static __inline uint32_t
282atomic_swap_32(volatile uint32_t *p, uint32_t val)
283{
284	uint32_t ret;
285	int res;
286
287	__asm __volatile(
288	    "1: ldxr	%w0, [%2]      \n"
289	    "   stxr	%w1, %w3, [%2] \n"
290	    "   cbnz	%w1, 1b        \n"
291	    : "=&r"(ret), "=&r"(res)
292	    : "r" (p), "r" (val)
293	    : "memory"
294	);
295
296	return (ret);
297}
298
299static __inline uint64_t
300atomic_swap_64(volatile uint64_t *p, uint64_t val)
301{
302	uint64_t ret;
303	int res;
304
305	__asm __volatile(
306	    "1: ldxr	%0, [%2]      \n"
307	    "   stxr	%w1, %3, [%2] \n"
308	    "   cbnz	%w1, 1b       \n"
309	    : "=&r"(ret), "=&r"(res)
310	    : "r" (p), "r" (val)
311	    : "memory"
312	);
313
314	return (ret);
315}
316
317static __inline uint32_t
318atomic_load_acq_32(volatile uint32_t *p)
319{
320	uint32_t ret;
321
322	__asm __volatile(
323	    "ldar	%w0, [%1] \n"
324	    : "=&r" (ret)
325	    : "r" (p)
326	    : "memory");
327
328	return (ret);
329}
330
331static __inline uint64_t
332atomic_load_acq_64(volatile uint64_t *p)
333{
334	uint64_t ret;
335
336	__asm __volatile(
337	    "ldar	%0, [%1] \n"
338	    : "=&r" (ret)
339	    : "r" (p)
340	    : "memory");
341
342	return (ret);
343}
344
345static __inline void
346atomic_store_rel_32(volatile uint32_t *p, uint32_t val)
347{
348
349	__asm __volatile(
350	    "stlr	%w0, [%1] \n"
351	    :
352	    : "r" (val), "r" (p)
353	    : "memory");
354}
355
356static __inline void
357atomic_store_rel_64(volatile uint64_t *p, uint64_t val)
358{
359
360	__asm __volatile(
361	    "stlr	%0, [%1] \n"
362	    :
363	    : "r" (val), "r" (p)
364	    : "memory");
365}
366
367
368#define	atomic_add_int			atomic_add_32
369#define	atomic_fcmpset_int		atomic_fcmpset_32
370#define	atomic_clear_int		atomic_clear_32
371#define	atomic_cmpset_int		atomic_cmpset_32
372#define	atomic_fetchadd_int		atomic_fetchadd_32
373#define	atomic_readandclear_int		atomic_readandclear_32
374#define	atomic_set_int			atomic_set_32
375#define	atomic_swap_int			atomic_swap_32
376#define	atomic_subtract_int		atomic_subtract_32
377
378#define	atomic_add_acq_int		atomic_add_acq_32
379#define	atomic_fcmpset_acq_int		atomic_fcmpset_acq_32
380#define	atomic_clear_acq_int		atomic_clear_acq_32
381#define	atomic_cmpset_acq_int		atomic_cmpset_acq_32
382#define	atomic_load_acq_int		atomic_load_acq_32
383#define	atomic_set_acq_int		atomic_set_acq_32
384#define	atomic_subtract_acq_int		atomic_subtract_acq_32
385
386#define	atomic_add_rel_int		atomic_add_rel_32
387#define	atomic_fcmpset_rel_int		atomic_fcmpset_rel_32
388#define	atomic_clear_rel_int		atomic_add_rel_32
389#define	atomic_cmpset_rel_int		atomic_cmpset_rel_32
390#define	atomic_set_rel_int		atomic_set_rel_32
391#define	atomic_subtract_rel_int		atomic_subtract_rel_32
392#define	atomic_store_rel_int		atomic_store_rel_32
393
394#define	atomic_add_long			atomic_add_64
395#define	atomic_fcmpset_long		atomic_fcmpset_64
396#define	atomic_clear_long		atomic_clear_64
397#define	atomic_cmpset_long		atomic_cmpset_64
398#define	atomic_fetchadd_long		atomic_fetchadd_64
399#define	atomic_readandclear_long	atomic_readandclear_64
400#define	atomic_set_long			atomic_set_64
401#define	atomic_swap_long		atomic_swap_64
402#define	atomic_subtract_long		atomic_subtract_64
403
404#define	atomic_add_ptr			atomic_add_64
405#define	atomic_fcmpset_ptr		atomic_fcmpset_64
406#define	atomic_clear_ptr		atomic_clear_64
407#define	atomic_cmpset_ptr		atomic_cmpset_64
408#define	atomic_fetchadd_ptr		atomic_fetchadd_64
409#define	atomic_readandclear_ptr		atomic_readandclear_64
410#define	atomic_set_ptr			atomic_set_64
411#define	atomic_swap_ptr			atomic_swap_64
412#define	atomic_subtract_ptr		atomic_subtract_64
413
414#define	atomic_add_acq_long		atomic_add_acq_64
415#define	atomic_fcmpset_acq_long		atomic_fcmpset_acq_64
416#define	atomic_clear_acq_long		atomic_add_acq_64
417#define	atomic_cmpset_acq_long		atomic_cmpset_acq_64
418#define	atomic_load_acq_long		atomic_load_acq_64
419#define	atomic_set_acq_long		atomic_set_acq_64
420#define	atomic_subtract_acq_long	atomic_subtract_acq_64
421
422#define	atomic_add_acq_ptr		atomic_add_acq_64
423#define	atomic_fcmpset_acq_ptr		atomic_fcmpset_acq_64
424#define	atomic_clear_acq_ptr		atomic_add_acq_64
425#define	atomic_cmpset_acq_ptr		atomic_cmpset_acq_64
426#define	atomic_load_acq_ptr		atomic_load_acq_64
427#define	atomic_set_acq_ptr		atomic_set_acq_64
428#define	atomic_subtract_acq_ptr		atomic_subtract_acq_64
429
430#define	atomic_add_rel_long		atomic_add_rel_64
431#define	atomic_fcmpset_rel_long		atomic_fcmpset_rel_64
432#define	atomic_clear_rel_long		atomic_clear_rel_64
433#define	atomic_cmpset_rel_long		atomic_cmpset_rel_64
434#define	atomic_set_rel_long		atomic_set_rel_64
435#define	atomic_subtract_rel_long	atomic_subtract_rel_64
436#define	atomic_store_rel_long		atomic_store_rel_64
437
438#define	atomic_add_rel_ptr		atomic_add_rel_64
439#define	atomic_fcmpset_rel_ptr		atomic_fcmpset_rel_64
440#define	atomic_clear_rel_ptr		atomic_clear_rel_64
441#define	atomic_cmpset_rel_ptr		atomic_cmpset_rel_64
442#define	atomic_set_rel_ptr		atomic_set_rel_64
443#define	atomic_subtract_rel_ptr		atomic_subtract_rel_64
444#define	atomic_store_rel_ptr		atomic_store_rel_64
445
446static __inline void
447atomic_thread_fence_acq(void)
448{
449
450	dmb(ld);
451}
452
453static __inline void
454atomic_thread_fence_rel(void)
455{
456
457	dmb(sy);
458}
459
460static __inline void
461atomic_thread_fence_acq_rel(void)
462{
463
464	dmb(sy);
465}
466
467static __inline void
468atomic_thread_fence_seq_cst(void)
469{
470
471	dmb(sy);
472}
473
474#endif /* _MACHINE_ATOMIC_H_ */
475
476