atomic.h revision 315371 
1/*-
2 * Copyright (c) 1998 Doug Rabson.
3 * Copyright (c) 2001 Jake Burkholder.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 *
27 *	from: FreeBSD: src/sys/i386/include/atomic.h,v 1.20 2001/02/11
28 * $FreeBSD: stable/11/sys/sparc64/include/atomic.h 315371 2017-03-16 06:00:27Z mjg $
29 */
30
31#ifndef	_MACHINE_ATOMIC_H_
32#define	_MACHINE_ATOMIC_H_
33
34#include <machine/cpufunc.h>
35
36#define	mb()	__asm__ __volatile__ ("membar #MemIssue": : :"memory")
37#define	wmb()	mb()
38#define	rmb()	mb()
39
40/* Userland needs different ASI's. */
41#ifdef _KERNEL
42#define	__ASI_ATOMIC	ASI_N
43#else
44#define	__ASI_ATOMIC	ASI_P
45#endif
46
47/*
48 * Various simple arithmetic on memory which is atomic in the presence
49 * of interrupts and multiple processors.  See atomic(9) for details.
50 * Note that efficient hardware support exists only for the 32 and 64
51 * bit variants; the 8 and 16 bit versions are not provided and should
52 * not be used in MI code.
53 *
54 * This implementation takes advantage of the fact that the sparc64
55 * cas instruction is both a load and a store.  The loop is often coded
56 * as follows:
57 *
58 *	do {
59 *		expect = *p;
60 *		new = expect + 1;
61 *	} while (cas(p, expect, new) != expect);
62 *
63 * which performs an unnnecessary load on each iteration that the cas
64 * operation fails.  Modified as follows:
65 *
66 *	expect = *p;
67 *	for (;;) {
68 *		new = expect + 1;
69 *		result = cas(p, expect, new);
70 *		if (result == expect)
71 *			break;
72 *		expect = result;
73 *	}
74 *
75 * the return value of cas is used to avoid the extra reload.
76 *
77 * We only include a memory barrier in the rel variants as in total store
78 * order which we use for running the kernel and all of the userland atomic
79 * loads and stores behave as if the were followed by a membar with a mask
80 * of #LoadLoad | #LoadStore | #StoreStore.  In order to be also sufficient
81 * for use of relaxed memory ordering, the atomic_cas() in the acq variants
82 * additionally would have to be followed by a membar #LoadLoad | #LoadStore.
83 * Due to the suggested assembly syntax of the membar operands containing a
84 * # character, they cannot be used in macros.  The cmask and mmask bits thus
85 * are hard coded in machine/cpufunc.h and used here through macros.
86 * Hopefully the bit numbers won't change in the future.
87 */
88
89#define	itype(sz)	uint ## sz ## _t
90
91#define	atomic_cas_32(p, e, s)	casa((p), (e), (s), __ASI_ATOMIC)
92#define	atomic_cas_64(p, e, s)	casxa((p), (e), (s), __ASI_ATOMIC)
93
94#define	atomic_cas(p, e, s, sz)						\
95	atomic_cas_ ## sz((p), (e), (s))
96
97#define	atomic_cas_acq(p, e, s, sz) ({					\
98	itype(sz) v;							\
99	v = atomic_cas((p), (e), (s), sz);				\
100	__compiler_membar();						\
101	v;								\
102})
103
104#define	atomic_cas_rel(p, e, s, sz) ({					\
105	itype(sz) v;							\
106	membar(LoadStore | StoreStore);					\
107	v = atomic_cas((p), (e), (s), sz);				\
108	v;								\
109})
110
111#define	atomic_op(p, op, v, sz) ({					\
112	itype(sz) e, r, s;						\
113	for (e = *(volatile itype(sz) *)(p);; e = r) {			\
114		s = e op (v);						\
115		r = atomic_cas_ ## sz((p), e, s);			\
116		if (r == e)						\
117			break;						\
118	}								\
119	e;								\
120})
121
122#define	atomic_op_acq(p, op, v, sz) ({					\
123	itype(sz) t;							\
124	t = atomic_op((p), op, (v), sz);				\
125	__compiler_membar();						\
126	t;								\
127})
128
129#define	atomic_op_rel(p, op, v, sz) ({					\
130	itype(sz) t;							\
131	membar(LoadStore | StoreStore);					\
132	t = atomic_op((p), op, (v), sz);				\
133	t;								\
134})
135
136#define	atomic_ld_acq(p, sz) ({						\
137	itype(sz) v;							\
138	v = atomic_cas((p), 0, 0, sz);					\
139	__compiler_membar();						\
140	v;								\
141})
142
143#define	atomic_ld_clear(p, sz) ({					\
144	itype(sz) e, r;							\
145	for (e = *(volatile itype(sz) *)(p);; e = r) {			\
146		r = atomic_cas((p), e, 0, sz);				\
147		if (r == e)						\
148			break;						\
149	}								\
150	e;								\
151})
152
153#define	atomic_st(p, v, sz) do {					\
154	itype(sz) e, r;							\
155	for (e = *(volatile itype(sz) *)(p);; e = r) {			\
156		r = atomic_cas((p), e, (v), sz);			\
157		if (r == e)						\
158			break;						\
159	}								\
160} while (0)
161
162#define	atomic_st_acq(p, v, sz) do {					\
163	atomic_st((p), (v), sz);					\
164	__compiler_membar();						\
165} while (0)
166
167#define	atomic_st_rel(p, v, sz) do {					\
168	membar(LoadStore | StoreStore);					\
169	atomic_st((p), (v), sz);					\
170} while (0)
171
172#define	ATOMIC_GEN(name, ptype, vtype, atype, sz)			\
173									\
174static __inline vtype							\
175atomic_add_ ## name(volatile ptype p, atype v)				\
176{									\
177	return ((vtype)atomic_op((p), +, (v), sz));			\
178}									\
179static __inline vtype							\
180atomic_add_acq_ ## name(volatile ptype p, atype v)			\
181{									\
182	return ((vtype)atomic_op_acq((p), +, (v), sz));			\
183}									\
184static __inline vtype							\
185atomic_add_rel_ ## name(volatile ptype p, atype v)			\
186{									\
187	return ((vtype)atomic_op_rel((p), +, (v), sz));			\
188}									\
189									\
190static __inline vtype							\
191atomic_clear_ ## name(volatile ptype p, atype v)			\
192{									\
193	return ((vtype)atomic_op((p), &, ~(v), sz));			\
194}									\
195static __inline vtype							\
196atomic_clear_acq_ ## name(volatile ptype p, atype v)			\
197{									\
198	return ((vtype)atomic_op_acq((p), &, ~(v), sz));		\
199}									\
200static __inline vtype							\
201atomic_clear_rel_ ## name(volatile ptype p, atype v)			\
202{									\
203	return ((vtype)atomic_op_rel((p), &, ~(v), sz));		\
204}									\
205									\
206static __inline int							\
207atomic_cmpset_ ## name(volatile ptype p, vtype e, vtype s)		\
208{									\
209	return (((vtype)atomic_cas((p), (e), (s), sz)) == (e));		\
210}									\
211static __inline int							\
212atomic_cmpset_acq_ ## name(volatile ptype p, vtype e, vtype s)		\
213{									\
214	return (((vtype)atomic_cas_acq((p), (e), (s), sz)) == (e));	\
215}									\
216static __inline int							\
217atomic_cmpset_rel_ ## name(volatile ptype p, vtype e, vtype s)		\
218{									\
219	return (((vtype)atomic_cas_rel((p), (e), (s), sz)) == (e));	\
220}									\
221									\
222static __inline int							\
223atomic_fcmpset_ ## name(volatile ptype p, vtype *ep, vtype s)		\
224{									\
225	vtype t;							\
226									\
227	t = (vtype)atomic_cas((p), (*ep), (s), sz);			\
228	if (t == (*ep))	 						\
229		return (1);						\
230	*ep = t;							\
231	return (0);							\
232}									\
233static __inline int							\
234atomic_fcmpset_acq_ ## name(volatile ptype p, vtype *ep, vtype s)	\
235{									\
236	vtype t;							\
237									\
238	t = (vtype)atomic_cas_acq((p), (*ep), (s), sz);			\
239	if (t == (*ep))	 						\
240		return (1);						\
241	*ep = t;							\
242	return (0);							\
243}									\
244static __inline int							\
245atomic_fcmpset_rel_ ## name(volatile ptype p, vtype *ep, vtype s)	\
246{									\
247	vtype t;							\
248									\
249	t = (vtype)atomic_cas_rel((p), (*ep), (s), sz);			\
250	if (t == (*ep))	 						\
251		return (1);						\
252	*ep = t;							\
253	return (0);							\
254}									\
255									\
256static __inline vtype							\
257atomic_load_ ## name(volatile ptype p)					\
258{									\
259	return ((vtype)atomic_cas((p), 0, 0, sz));			\
260}									\
261static __inline vtype							\
262atomic_load_acq_ ## name(volatile ptype p)				\
263{									\
264	return ((vtype)atomic_cas_acq((p), 0, 0, sz));			\
265}									\
266									\
267static __inline vtype							\
268atomic_readandclear_ ## name(volatile ptype p)				\
269{									\
270	return ((vtype)atomic_ld_clear((p), sz));			\
271}									\
272									\
273static __inline vtype							\
274atomic_set_ ## name(volatile ptype p, atype v)				\
275{									\
276	return ((vtype)atomic_op((p), |, (v), sz));			\
277}									\
278static __inline vtype							\
279atomic_set_acq_ ## name(volatile ptype p, atype v)			\
280{									\
281	return ((vtype)atomic_op_acq((p), |, (v), sz));			\
282}									\
283static __inline vtype							\
284atomic_set_rel_ ## name(volatile ptype p, atype v)			\
285{									\
286	return ((vtype)atomic_op_rel((p), |, (v), sz));			\
287}									\
288									\
289static __inline vtype							\
290atomic_subtract_ ## name(volatile ptype p, atype v)			\
291{									\
292	return ((vtype)atomic_op((p), -, (v), sz));			\
293}									\
294static __inline vtype							\
295atomic_subtract_acq_ ## name(volatile ptype p, atype v)			\
296{									\
297	return ((vtype)atomic_op_acq((p), -, (v), sz));			\
298}									\
299static __inline vtype							\
300atomic_subtract_rel_ ## name(volatile ptype p, atype v)			\
301{									\
302	return ((vtype)atomic_op_rel((p), -, (v), sz));			\
303}									\
304									\
305static __inline void							\
306atomic_store_acq_ ## name(volatile ptype p, vtype v)			\
307{									\
308	atomic_st_acq((p), (v), sz);					\
309}									\
310static __inline void							\
311atomic_store_rel_ ## name(volatile ptype p, vtype v)			\
312{									\
313	atomic_st_rel((p), (v), sz);					\
314}
315
316static __inline void
317atomic_thread_fence_acq(void)
318{
319
320	__compiler_membar();
321}
322
323static __inline void
324atomic_thread_fence_rel(void)
325{
326
327	__compiler_membar();
328}
329
330static __inline void
331atomic_thread_fence_acq_rel(void)
332{
333
334	__compiler_membar();
335}
336
337static __inline void
338atomic_thread_fence_seq_cst(void)
339{
340
341	membar(LoadLoad | LoadStore | StoreStore | StoreLoad);
342}
343
344
345ATOMIC_GEN(int, u_int *, u_int, u_int, 32);
346ATOMIC_GEN(32, uint32_t *, uint32_t, uint32_t, 32);
347
348ATOMIC_GEN(long, u_long *, u_long, u_long, 64);
349ATOMIC_GEN(64, uint64_t *, uint64_t, uint64_t, 64);
350
351ATOMIC_GEN(ptr, uintptr_t *, uintptr_t, uintptr_t, 64);
352
353#define	atomic_fetchadd_int	atomic_add_int
354#define	atomic_fetchadd_32	atomic_add_32
355#define	atomic_fetchadd_long	atomic_add_long
356#define	atomic_fetchadd_64	atomic_add_64
357
358#undef ATOMIC_GEN
359#undef atomic_cas
360#undef atomic_cas_acq
361#undef atomic_cas_rel
362#undef atomic_op
363#undef atomic_op_acq
364#undef atomic_op_rel
365#undef atomic_ld_acq
366#undef atomic_ld_clear
367#undef atomic_st
368#undef atomic_st_acq
369#undef atomic_st_rel
370
371#endif /* !_MACHINE_ATOMIC_H_ */
372