1/*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2005 John Baldwin <jhb@FreeBSD.org>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#ifndef __SYS_REFCOUNT_H__
29#define __SYS_REFCOUNT_H__
30
31#include <machine/atomic.h>
32
33#if defined(_KERNEL) || defined(_STANDALONE)
34#include <sys/systm.h>
35#else
36#include <stdbool.h>
37#define	KASSERT(exp, msg)	/* */
38#endif
39
40#define	REFCOUNT_SATURATED(val)		(((val) & (1U << 31)) != 0)
41#define	REFCOUNT_SATURATION_VALUE	(3U << 30)
42
43/*
44 * Attempt to handle reference count overflow and underflow.  Force the counter
45 * to stay at the saturation value so that a counter overflow cannot trigger
46 * destruction of the containing object and instead leads to a less harmful
47 * memory leak.
48 */
49static __inline void
50_refcount_update_saturated(volatile u_int *count)
51{
52#ifdef INVARIANTS
53	panic("refcount %p wraparound", count);
54#else
55	atomic_store_int(count, REFCOUNT_SATURATION_VALUE);
56#endif
57}
58
59static __inline void
60refcount_init(volatile u_int *count, u_int value)
61{
62	KASSERT(!REFCOUNT_SATURATED(value),
63	    ("invalid initial refcount value %u", value));
64	atomic_store_int(count, value);
65}
66
67static __inline u_int
68refcount_load(volatile u_int *count)
69{
70	return (atomic_load_int(count));
71}
72
73static __inline u_int
74refcount_acquire(volatile u_int *count)
75{
76	u_int old;
77
78	old = atomic_fetchadd_int(count, 1);
79	if (__predict_false(REFCOUNT_SATURATED(old)))
80		_refcount_update_saturated(count);
81
82	return (old);
83}
84
85static __inline u_int
86refcount_acquiren(volatile u_int *count, u_int n)
87{
88	u_int old;
89
90	KASSERT(n < REFCOUNT_SATURATION_VALUE / 2,
91	    ("refcount_acquiren: n=%u too large", n));
92	old = atomic_fetchadd_int(count, n);
93	if (__predict_false(REFCOUNT_SATURATED(old)))
94		_refcount_update_saturated(count);
95
96	return (old);
97}
98
99static __inline __result_use_check bool
100refcount_acquire_checked(volatile u_int *count)
101{
102	u_int old;
103
104	old = atomic_load_int(count);
105	for (;;) {
106		if (__predict_false(REFCOUNT_SATURATED(old + 1)))
107			return (false);
108		if (__predict_true(atomic_fcmpset_int(count, &old,
109		    old + 1) == 1))
110			return (true);
111	}
112}
113
114/*
115 * This functions returns non-zero if the refcount was
116 * incremented. Else zero is returned.
117 */
118static __inline __result_use_check bool
119refcount_acquire_if_gt(volatile u_int *count, u_int n)
120{
121	u_int old;
122
123	old = atomic_load_int(count);
124	for (;;) {
125		if (old <= n)
126			return (false);
127		if (__predict_false(REFCOUNT_SATURATED(old)))
128			return (true);
129		if (atomic_fcmpset_int(count, &old, old + 1))
130			return (true);
131	}
132}
133
134static __inline __result_use_check bool
135refcount_acquire_if_not_zero(volatile u_int *count)
136{
137
138	return (refcount_acquire_if_gt(count, 0));
139}
140
141static __inline bool
142refcount_releasen(volatile u_int *count, u_int n)
143{
144	u_int old;
145
146	KASSERT(n < REFCOUNT_SATURATION_VALUE / 2,
147	    ("refcount_releasen: n=%u too large", n));
148
149	atomic_thread_fence_rel();
150	old = atomic_fetchadd_int(count, -n);
151	if (__predict_false(old < n || REFCOUNT_SATURATED(old))) {
152		_refcount_update_saturated(count);
153		return (false);
154	}
155	if (old > n)
156		return (false);
157
158	/*
159	 * Last reference.  Signal the user to call the destructor.
160	 *
161	 * Ensure that the destructor sees all updates. This synchronizes with
162	 * release fences from all routines which drop the count.
163	 */
164	atomic_thread_fence_acq();
165	return (true);
166}
167
168static __inline bool
169refcount_release(volatile u_int *count)
170{
171
172	return (refcount_releasen(count, 1));
173}
174
175#define	_refcount_release_if_cond(cond, name)				\
176static __inline __result_use_check bool					\
177_refcount_release_if_##name(volatile u_int *count, u_int n)		\
178{									\
179	u_int old;							\
180									\
181	KASSERT(n > 0, ("%s: zero increment", __func__));		\
182	old = atomic_load_int(count);					\
183	for (;;) {							\
184		if (!(cond))						\
185			return (false);					\
186		if (__predict_false(REFCOUNT_SATURATED(old)))		\
187			return (false);					\
188		if (atomic_fcmpset_rel_int(count, &old, old - 1))	\
189			return (true);					\
190	}								\
191}
192_refcount_release_if_cond(old > n, gt)
193_refcount_release_if_cond(old == n, eq)
194
195static __inline __result_use_check bool
196refcount_release_if_gt(volatile u_int *count, u_int n)
197{
198
199	return (_refcount_release_if_gt(count, n));
200}
201
202static __inline __result_use_check bool
203refcount_release_if_last(volatile u_int *count)
204{
205
206	if (_refcount_release_if_eq(count, 1)) {
207		/* See the comment in refcount_releasen(). */
208		atomic_thread_fence_acq();
209		return (true);
210	}
211	return (false);
212}
213
214static __inline __result_use_check bool
215refcount_release_if_not_last(volatile u_int *count)
216{
217
218	return (_refcount_release_if_gt(count, 1));
219}
220
221#endif /* !__SYS_REFCOUNT_H__ */
222