1/*	$OpenBSD: percpu.h,v 1.9 2023/09/16 09:33:27 mpi Exp $ */
2
3/*
4 * Copyright (c) 2016 David Gwynne <dlg@openbsd.org>
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19#ifndef _SYS_PERCPU_H_
20#define _SYS_PERCPU_H_
21
22#ifndef CACHELINESIZE
23#define CACHELINESIZE 64
24#endif
25
26#ifndef __upunused /* this should go in param.h */
27#ifdef MULTIPROCESSOR
28#define __upunused
29#else
30#define __upunused __attribute__((__unused__))
31#endif
32#endif
33
34struct cpumem {
35	void		*mem;
36};
37
38struct cpumem_iter {
39	unsigned int	cpu;
40} __upunused;
41
42struct counters_ref {
43	uint64_t	 g;
44	uint64_t	*c;
45};
46
47#ifdef _KERNEL
48
49#include <sys/atomic.h>
50
51struct pool;
52
53struct cpumem	*cpumem_get(struct pool *);
54void		 cpumem_put(struct pool *, struct cpumem *);
55
56struct cpumem	*cpumem_malloc(size_t, int);
57struct cpumem	*cpumem_malloc_ncpus(struct cpumem *, size_t, int);
58void		 cpumem_free(struct cpumem *, int, size_t);
59
60void		*cpumem_first(struct cpumem_iter *, struct cpumem *);
61void		*cpumem_next(struct cpumem_iter *, struct cpumem *);
62
63static inline void *
64cpumem_enter(struct cpumem *cm)
65{
66#ifdef MULTIPROCESSOR
67	return (cm[cpu_number()].mem);
68#else
69	return (cm);
70#endif
71}
72
73static inline void
74cpumem_leave(struct cpumem *cm, void *mem)
75{
76	/* KDASSERT? */
77}
78
79#ifdef MULTIPROCESSOR
80
81#define CPUMEM_BOOT_MEMORY(_name, _sz)					\
82static struct {								\
83	unsigned char	mem[_sz];					\
84	struct cpumem	cpumem;						\
85} __aligned(CACHELINESIZE) _name##_boot_cpumem = {			\
86	.cpumem = { _name##_boot_cpumem.mem }				\
87}
88
89#define CPUMEM_BOOT_INITIALIZER(_name)					\
90	{ &_name##_boot_cpumem.cpumem }
91
92#else /* MULTIPROCESSOR */
93
94#define CPUMEM_BOOT_MEMORY(_name, _sz)					\
95static struct {								\
96	unsigned char	mem[_sz];					\
97} __aligned(sizeof(uint64_t)) _name##_boot_cpumem
98
99#define CPUMEM_BOOT_INITIALIZER(_name)					\
100	{ (struct cpumem *)&_name##_boot_cpumem.mem }
101
102#endif /* MULTIPROCESSOR */
103
104#define CPUMEM_FOREACH(_var, _iter, _cpumem)				\
105	for ((_var) = cpumem_first((_iter), (_cpumem));			\
106	    (_var) != NULL;						\
107	    (_var) = cpumem_next((_iter), (_cpumem)))
108
109/*
110 * per cpu counters
111 */
112
113struct cpumem	*counters_alloc(unsigned int);
114struct cpumem	*counters_alloc_ncpus(struct cpumem *, unsigned int);
115void		 counters_free(struct cpumem *, unsigned int);
116void		 counters_read(struct cpumem *, uint64_t *, unsigned int,
117		     uint64_t *);
118void		 counters_zero(struct cpumem *, unsigned int);
119
120static inline uint64_t *
121counters_enter(struct counters_ref *ref, struct cpumem *cm)
122{
123	ref->c = cpumem_enter(cm);
124#ifdef MULTIPROCESSOR
125	ref->g = ++(*ref->c); /* make the generation number odd */
126	membar_producer();
127	return (ref->c + 1);
128#else
129	return (ref->c);
130#endif
131}
132
133static inline void
134counters_leave(struct counters_ref *ref, struct cpumem *cm)
135{
136#ifdef MULTIPROCESSOR
137	membar_producer();
138	(*ref->c) = ++ref->g; /* make the generation number even again */
139#endif
140	cpumem_leave(cm, ref->c);
141}
142
143static inline void
144counters_inc(struct cpumem *cm, unsigned int c)
145{
146	struct counters_ref ref;
147	uint64_t *counters;
148
149	counters = counters_enter(&ref, cm);
150	counters[c]++;
151	counters_leave(&ref, cm);
152}
153
154static inline void
155counters_dec(struct cpumem *cm, unsigned int c)
156{
157	struct counters_ref ref;
158	uint64_t *counters;
159
160	counters = counters_enter(&ref, cm);
161	counters[c]--;
162	counters_leave(&ref, cm);
163}
164
165static inline void
166counters_add(struct cpumem *cm, unsigned int c, uint64_t v)
167{
168	struct counters_ref ref;
169	uint64_t *counters;
170
171	counters = counters_enter(&ref, cm);
172	counters[c] += v;
173	counters_leave(&ref, cm);
174}
175
176static inline void
177counters_pkt(struct cpumem *cm, unsigned int c, unsigned int b, uint64_t v)
178{
179	struct counters_ref ref;
180	uint64_t *counters;
181
182	counters = counters_enter(&ref, cm);
183	counters[c]++;
184	counters[b] += v;
185	counters_leave(&ref, cm);
186}
187
188#ifdef MULTIPROCESSOR
189#define COUNTERS_BOOT_MEMORY(_name, _n)					\
190	CPUMEM_BOOT_MEMORY(_name, ((_n) + 1) * sizeof(uint64_t))
191#else
192#define COUNTERS_BOOT_MEMORY(_name, _n)					\
193	CPUMEM_BOOT_MEMORY(_name, (_n) * sizeof(uint64_t))
194#endif
195
196#define COUNTERS_BOOT_INITIALIZER(_name)	CPUMEM_BOOT_INITIALIZER(_name)
197
198#endif /* _KERNEL */
199#endif /* _SYS_PERCPU_H_ */
200