1/*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2019 Kyle Evans <kevans@FreeBSD.org>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27#ifndef _SYS__ATOMIC_SUBWORD_H_
28#define	_SYS__ATOMIC_SUBWORD_H_
29
30/*
31 * This header is specifically for platforms that either do not have ways to or
32 * simply do not do sub-word atomic operations.  These are not ideal as they
33 * require a little more effort to make sure our atomic operations are failing
34 * because of the bits of the word we're trying to write rather than the rest
35 * of the word.
36 */
37#ifndef _MACHINE_ATOMIC_H_
38#error do not include this header, use machine/atomic.h
39#endif
40
41#include <machine/endian.h>
42#ifndef _KERNEL
43#include <stdbool.h>
44#endif
45
46#ifndef NBBY
47#define	NBBY	8
48#endif
49
50#define	_ATOMIC_WORD_ALIGNED(p)		\
51    (uint32_t *)((__uintptr_t)(p) - ((__uintptr_t)(p) % 4))
52
53#if _BYTE_ORDER == _BIG_ENDIAN
54#define	_ATOMIC_BYTE_SHIFT(p)		\
55    ((3 - ((__uintptr_t)(p) % 4)) * NBBY)
56
57#define	_ATOMIC_HWORD_SHIFT(p)		\
58    ((2 - ((__uintptr_t)(p) % 4)) * NBBY)
59#else
60#define	_ATOMIC_BYTE_SHIFT(p)		\
61    ((((__uintptr_t)(p) % 4)) * NBBY)
62
63#define	_ATOMIC_HWORD_SHIFT(p)		\
64    ((((__uintptr_t)(p) % 4)) * NBBY)
65#endif
66
67#ifndef	_atomic_cmpset_masked_word
68/*
69 * Pass these bad boys a couple words and a mask of the bits you care about,
70 * they'll loop until we either succeed or fail because of those bits rather
71 * than the ones we're not masking.  old and val should already be preshifted to
72 * the proper position.
73 */
74static __inline int
75_atomic_cmpset_masked_word(uint32_t *addr, uint32_t old, uint32_t val,
76    uint32_t mask)
77{
78	int ret;
79	uint32_t wcomp;
80
81	wcomp = old;
82
83	/*
84	 * We'll attempt the cmpset on the entire word.  Loop here in case the
85	 * operation fails due to the other half-word resident in that word,
86	 * rather than the half-word we're trying to operate on.  Ideally we
87	 * only take one trip through here.  We'll have to recalculate the old
88	 * value since it's the other part of the word changing.
89	 */
90	do {
91		old = (*addr & ~mask) | wcomp;
92		ret = atomic_fcmpset_32(addr, &old, (old & ~mask) | val);
93	} while (ret == 0 && (old & mask) == wcomp);
94
95	return (ret);
96}
97#endif
98
99#ifndef	_atomic_fcmpset_masked_word
100static __inline int
101_atomic_fcmpset_masked_word(uint32_t *addr, uint32_t *old, uint32_t val,
102    uint32_t mask)
103{
104
105	/*
106	 * fcmpset_* is documented in atomic(9) to allow spurious failures where
107	 * *old == val on ll/sc architectures because the sc may fail due to
108	 * parallel writes or other reasons.  We take advantage of that here
109	 * and only attempt once, because the caller should be compensating for
110	 * that possibility.
111	 */
112	*old = (*addr & ~mask) | *old;
113	return (atomic_fcmpset_32(addr, old, (*old & ~mask) | val));
114}
115#endif
116
117#ifndef atomic_cmpset_8
118static __inline int
119atomic_cmpset_8(__volatile uint8_t *addr, uint8_t old, uint8_t val)
120{
121	int shift;
122
123	shift = _ATOMIC_BYTE_SHIFT(addr);
124
125	return (_atomic_cmpset_masked_word(_ATOMIC_WORD_ALIGNED(addr),
126	    old << shift, val << shift, 0xff << shift));
127}
128#endif
129
130#ifndef atomic_fcmpset_8
131static __inline int
132atomic_fcmpset_8(__volatile uint8_t *addr, uint8_t *old, uint8_t val)
133{
134	int ret, shift;
135	uint32_t wold;
136
137	shift = _ATOMIC_BYTE_SHIFT(addr);
138	wold = *old << shift;
139	ret = _atomic_fcmpset_masked_word(_ATOMIC_WORD_ALIGNED(addr),
140	    &wold, val << shift, 0xff << shift);
141	if (ret == 0)
142		*old = (wold >> shift) & 0xff;
143	return (ret);
144}
145#endif
146
147#ifndef atomic_cmpset_16
148static __inline int
149atomic_cmpset_16(__volatile uint16_t *addr, uint16_t old, uint16_t val)
150{
151	int shift;
152
153	shift = _ATOMIC_HWORD_SHIFT(addr);
154
155	return (_atomic_cmpset_masked_word(_ATOMIC_WORD_ALIGNED(addr),
156	    old << shift, val << shift, 0xffff << shift));
157}
158#endif
159
160#ifndef atomic_fcmpset_16
161static __inline int
162atomic_fcmpset_16(__volatile uint16_t *addr, uint16_t *old, uint16_t val)
163{
164	int ret, shift;
165	uint32_t wold;
166
167	shift = _ATOMIC_HWORD_SHIFT(addr);
168	wold = *old << shift;
169	ret = _atomic_fcmpset_masked_word(_ATOMIC_WORD_ALIGNED(addr),
170	    &wold, val << shift, 0xffff << shift);
171	if (ret == 0)
172		*old = (wold >> shift) & 0xffff;
173	return (ret);
174}
175#endif
176
177#ifndef atomic_load_acq_8
178static __inline uint8_t
179atomic_load_acq_8(volatile uint8_t *p)
180{
181	int shift;
182	uint8_t ret;
183
184	shift = _ATOMIC_BYTE_SHIFT(p);
185	ret = (atomic_load_acq_32(_ATOMIC_WORD_ALIGNED(p)) >> shift) & 0xff;
186	return (ret);
187}
188#endif
189
190#ifndef atomic_load_acq_16
191static __inline uint16_t
192atomic_load_acq_16(volatile uint16_t *p)
193{
194	int shift;
195	uint16_t ret;
196
197	shift = _ATOMIC_HWORD_SHIFT(p);
198	ret = (atomic_load_acq_32(_ATOMIC_WORD_ALIGNED(p)) >> shift) &
199	    0xffff;
200	return (ret);
201}
202#endif
203
204#undef _ATOMIC_WORD_ALIGNED
205#undef _ATOMIC_BYTE_SHIFT
206#undef _ATOMIC_HWORD_SHIFT
207
208/*
209 * Provide generic testandset_long implementation based on fcmpset long
210 * primitive.  It may not be ideal for any given arch, so machine/atomic.h
211 * should define the macro atomic_testandset_long to override with an
212 * MD-specific version.
213 *
214 * (Organizationally, this isn't really subword atomics.  But atomic_common is
215 * included too early in machine/atomic.h, so it isn't a good place for derived
216 * primitives like this.)
217 */
218#ifndef atomic_testandset_acq_long
219static __inline int
220atomic_testandset_acq_long(volatile u_long *p, u_int v)
221{
222	u_long bit, old;
223	bool ret;
224
225	bit = (1ul << (v % (sizeof(*p) * NBBY)));
226
227	old = atomic_load_acq_long(p);
228	ret = false;
229	while (!ret && (old & bit) == 0)
230		ret = atomic_fcmpset_acq_long(p, &old, old | bit);
231
232	return (!ret);
233}
234#endif
235
236#ifndef atomic_testandset_long
237static __inline int
238atomic_testandset_long(volatile u_long *p, u_int v)
239{
240	u_long bit, old;
241	bool ret;
242
243	bit = (1ul << (v % (sizeof(*p) * NBBY)));
244
245	old = atomic_load_long(p);
246	ret = false;
247	while (!ret && (old & bit) == 0)
248		ret = atomic_fcmpset_long(p, &old, old | bit);
249
250	return (!ret);
251}
252#endif
253
254#ifndef atomic_testandclear_long
255static __inline int
256atomic_testandclear_long(volatile u_long *p, u_int v)
257{
258	u_long bit, old;
259	bool ret;
260
261	bit = (1ul << (v % (sizeof(*p) * NBBY)));
262
263	old = atomic_load_long(p);
264	ret = false;
265	while (!ret && (old & bit) != 0)
266		ret = atomic_fcmpset_long(p, &old, old & ~bit);
267
268	return (ret);
269}
270#endif
271
272#endif	/* _SYS__ATOMIC_SUBWORD_H_ */
273