1/*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 2001 Wind River Systems, Inc.
5 * All rights reserved.
6 * Written by: John Baldwin <jhb@FreeBSD.org>
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 *    notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 *    notice, this list of conditions and the following disclaimer in the
15 *    documentation and/or other materials provided with the distribution.
16 * 3. Neither the name of the author nor the names of any co-contributors
17 *    may be used to endorse or promote products derived from this software
18 *    without specific prior written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 */
32
33#ifndef _SYS_PCPU_H_
34#define	_SYS_PCPU_H_
35
36#ifdef LOCORE
37#error "no assembler-serviceable parts inside"
38#endif
39
40#include <sys/param.h>
41#include <sys/_cpuset.h>
42#include <sys/_lock.h>
43#include <sys/_mutex.h>
44#include <sys/_sx.h>
45#include <sys/queue.h>
46#include <sys/_rmlock.h>
47#include <sys/resource.h>
48#include <machine/pcpu.h>
49
50#define	DPCPU_SETNAME		"set_pcpu"
51#define	DPCPU_SYMPREFIX		"pcpu_entry_"
52
53#ifdef _KERNEL
54
55/*
56 * Define a set for pcpu data.
57 */
58extern uintptr_t *__start_set_pcpu;
59__GLOBL(__start_set_pcpu);
60extern uintptr_t *__stop_set_pcpu;
61__GLOBL(__stop_set_pcpu);
62
63/*
64 * Array of dynamic pcpu base offsets.  Indexed by id.
65 */
66extern uintptr_t dpcpu_off[];
67
68/*
69 * Convenience defines.
70 */
71#define	DPCPU_START		((uintptr_t)&__start_set_pcpu)
72#define	DPCPU_STOP		((uintptr_t)&__stop_set_pcpu)
73#define	DPCPU_BYTES		(DPCPU_STOP - DPCPU_START)
74#define	DPCPU_MODMIN		2048
75#define	DPCPU_SIZE		roundup2(DPCPU_BYTES, PAGE_SIZE)
76#define	DPCPU_MODSIZE		(DPCPU_SIZE - (DPCPU_BYTES - DPCPU_MODMIN))
77
78/*
79 * Declaration and definition.
80 */
81#define	DPCPU_NAME(n)		pcpu_entry_##n
82#define	DPCPU_DECLARE(t, n)	extern t DPCPU_NAME(n)
83/* struct _hack is to stop this from being used with the static keyword. */
84#define	DPCPU_DEFINE(t, n)	\
85    struct _hack; t DPCPU_NAME(n) __section(DPCPU_SETNAME) __used
86#if defined(KLD_MODULE) && (defined(__aarch64__) || defined(__riscv) \
87		|| defined(__powerpc64__) || defined(__i386__))
88/*
89 * On some architectures the compiler will use PC-relative load to
90 * find the address of DPCPU data with the static keyword. We then
91 * use this to find the offset of the data in a per-CPU region.
92 * This works for in the kernel as we can allocate the space ahead
93 * of time, however modules need to allocate a sepatate space and
94 * then use relocations to fix the address of the data. As
95 * PC-relative data doesn't have a relocation there is nothing for
96 * the kernel module linker to fix so data is accessed from the
97 * wrong location.
98 *
99 * This is a workaround until a better solution can be found.
100 *
101 * VNET_DEFINE_STATIC also has the same workaround.
102 */
103#define	DPCPU_DEFINE_STATIC(t, n)	\
104    t DPCPU_NAME(n) __section(DPCPU_SETNAME) __used
105#else
106#define	DPCPU_DEFINE_STATIC(t, n)	\
107    static t DPCPU_NAME(n) __section(DPCPU_SETNAME) __used
108#endif
109
110/*
111 * Accessors with a given base.
112 */
113#define	_DPCPU_PTR(b, n)						\
114    (__typeof(DPCPU_NAME(n))*)((b) + (uintptr_t)&DPCPU_NAME(n))
115#define	_DPCPU_GET(b, n)	(*_DPCPU_PTR(b, n))
116#define	_DPCPU_SET(b, n, v)	(*_DPCPU_PTR(b, n) = v)
117
118/*
119 * Accessors for the current cpu.
120 */
121#define	DPCPU_PTR(n)		_DPCPU_PTR(PCPU_GET(dynamic), n)
122#define	DPCPU_GET(n)		(*DPCPU_PTR(n))
123#define	DPCPU_SET(n, v)		(*DPCPU_PTR(n) = v)
124
125/*
126 * Accessors for remote cpus.
127 */
128#define	DPCPU_ID_PTR(i, n)	_DPCPU_PTR(dpcpu_off[(i)], n)
129#define	DPCPU_ID_GET(i, n)	(*DPCPU_ID_PTR(i, n))
130#define	DPCPU_ID_SET(i, n, v)	(*DPCPU_ID_PTR(i, n) = v)
131
132/*
133 * Utility macros.
134 */
135#define	DPCPU_SUM(n) __extension__					\
136({									\
137	u_int _i;							\
138	__typeof(*DPCPU_PTR(n)) sum;					\
139									\
140	sum = 0;							\
141	CPU_FOREACH(_i) {						\
142		sum += *DPCPU_ID_PTR(_i, n);				\
143	}								\
144	sum;								\
145})
146
147#define	DPCPU_VARSUM(n, var) __extension__				\
148({									\
149	u_int _i;							\
150	__typeof((DPCPU_PTR(n))->var) sum;				\
151									\
152	sum = 0;							\
153	CPU_FOREACH(_i) {						\
154		sum += (DPCPU_ID_PTR(_i, n))->var;			\
155	}								\
156	sum;								\
157})
158
159#define	DPCPU_ZERO(n) do {						\
160	u_int _i;							\
161									\
162	CPU_FOREACH(_i) {						\
163		bzero(DPCPU_ID_PTR(_i, n), sizeof(*DPCPU_PTR(n)));	\
164	}								\
165} while (0)
166
167#endif /* _KERNEL */
168
169/*
170 * This structure maps out the global data that needs to be kept on a
171 * per-cpu basis.  The members are accessed via the PCPU_GET/SET/PTR
172 * macros defined in <machine/pcpu.h>.  Machine dependent fields are
173 * defined in the PCPU_MD_FIELDS macro defined in <machine/pcpu.h>.
174 */
175struct pcpu {
176	struct thread	*pc_curthread;		/* Current thread */
177	struct thread	*pc_idlethread;		/* Idle thread */
178	struct thread	*pc_fpcurthread;	/* Fp state owner */
179	struct thread	*pc_deadthread;		/* Zombie thread or NULL */
180	struct pcb	*pc_curpcb;		/* Current pcb */
181	void		*pc_sched;		/* Scheduler state */
182	uint64_t	pc_switchtime;		/* cpu_ticks() at last csw */
183	int		pc_switchticks;		/* `ticks' at last csw */
184	u_int		pc_cpuid;		/* This cpu number */
185	STAILQ_ENTRY(pcpu) pc_allcpu;
186	struct lock_list_entry *pc_spinlocks;
187	long		pc_cp_time[CPUSTATES];	/* statclock ticks */
188	struct _device	*pc_device;		/* CPU device handle */
189	void		*pc_netisr;		/* netisr SWI cookie */
190	int8_t		pc_vfs_freevnodes;	/* freevnodes counter */
191	char		pc_unused1[3];		/* unused pad */
192	int		pc_domain;		/* Memory domain. */
193	struct rm_queue	pc_rm_queue;		/* rmlock list of trackers */
194	uintptr_t	pc_dynamic;		/* Dynamic per-cpu data area */
195	uint64_t	pc_early_dummy_counter;	/* Startup time counter(9) */
196	uintptr_t	pc_zpcpu_offset;	/* Offset into zpcpu allocs */
197
198	/*
199	 * Keep MD fields last, so that CPU-specific variations on a
200	 * single architecture don't result in offset variations of
201	 * the machine-independent fields of the pcpu.  Even though
202	 * the pcpu structure is private to the kernel, some ports
203	 * (e.g., lsof, part of gtop) define _KERNEL and include this
204	 * header.  While strictly speaking this is wrong, there's no
205	 * reason not to keep the offsets of the MI fields constant
206	 * if only to make kernel debugging easier.
207	 */
208	PCPU_MD_FIELDS;
209} __aligned(CACHE_LINE_SIZE);
210
211#ifdef _KERNEL
212
213STAILQ_HEAD(cpuhead, pcpu);
214
215extern struct cpuhead cpuhead;
216extern struct pcpu *cpuid_to_pcpu[];
217
218#define	curcpu		PCPU_GET(cpuid)
219#define	curvidata	PCPU_GET(vidata)
220
221#define UMA_PCPU_ALLOC_SIZE		PAGE_SIZE
222
223#include <machine/pcpu_aux.h>
224
225#ifndef curthread
226#define	curthread	PCPU_GET(curthread)
227#endif
228#define	curproc		(curthread->td_proc)
229
230#ifndef ZPCPU_ASSERT_PROTECTED
231#define ZPCPU_ASSERT_PROTECTED() MPASS(curthread->td_critnest > 0)
232#endif
233
234#ifndef zpcpu_offset_cpu
235#define zpcpu_offset_cpu(cpu)	(UMA_PCPU_ALLOC_SIZE * cpu)
236#endif
237#ifndef zpcpu_offset
238#define zpcpu_offset()		(PCPU_GET(zpcpu_offset))
239#endif
240
241#ifndef zpcpu_base_to_offset
242#define zpcpu_base_to_offset(base) (base)
243#endif
244#ifndef zpcpu_offset_to_base
245#define zpcpu_offset_to_base(base) (base)
246#endif
247
248/* Accessor to elements allocated via UMA_ZONE_PCPU zone. */
249#define zpcpu_get(base) ({								\
250	__typeof(base) _ptr = (void *)((char *)(base) + zpcpu_offset());		\
251	_ptr;										\
252})
253
254#define zpcpu_get_cpu(base, cpu) ({							\
255	__typeof(base) _ptr = (void *)((char *)(base) +	zpcpu_offset_cpu(cpu));		\
256	_ptr;										\
257})
258
259/*
260 * This operation is NOT atomic and does not post any barriers.
261 * If you use this the assumption is that the target CPU will not
262 * be modifying this variable.
263 * If you need atomicity use xchg.
264 * */
265#define zpcpu_replace(base, val) ({					\
266	__typeof(val) *_ptr = zpcpu_get(base);				\
267	__typeof(val) _old;						\
268									\
269	_old = *_ptr;							\
270	*_ptr = val;							\
271	_old;								\
272})
273
274#define zpcpu_replace_cpu(base, val, cpu) ({				\
275	__typeof(val) *_ptr = zpcpu_get_cpu(base, cpu);			\
276	__typeof(val) _old;						\
277									\
278	_old = *_ptr;							\
279	*_ptr = val;							\
280	_old;								\
281})
282
283#ifndef zpcpu_set_protected
284#define zpcpu_set_protected(base, val) ({				\
285	ZPCPU_ASSERT_PROTECTED();					\
286	__typeof(val) *_ptr = zpcpu_get(base);				\
287									\
288	*_ptr = (val);							\
289})
290#endif
291
292#ifndef zpcpu_add_protected
293#define zpcpu_add_protected(base, val) ({				\
294	ZPCPU_ASSERT_PROTECTED();					\
295	__typeof(val) *_ptr = zpcpu_get(base);				\
296									\
297	*_ptr += (val);							\
298})
299#endif
300
301#ifndef zpcpu_sub_protected
302#define zpcpu_sub_protected(base, val) ({				\
303	ZPCPU_ASSERT_PROTECTED();					\
304	__typeof(val) *_ptr = zpcpu_get(base);				\
305									\
306	*_ptr -= (val);							\
307})
308#endif
309
310/*
311 * Machine dependent callouts.  cpu_pcpu_init() is responsible for
312 * initializing machine dependent fields of struct pcpu, and
313 * db_show_mdpcpu() is responsible for handling machine dependent
314 * fields for the DDB 'show pcpu' command.
315 */
316void	cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size);
317void	db_show_mdpcpu(struct pcpu *pcpu);
318
319void	*dpcpu_alloc(int size);
320void	dpcpu_copy(void *s, int size);
321void	dpcpu_free(void *s, int size);
322void	dpcpu_init(void *dpcpu, int cpuid);
323void	pcpu_destroy(struct pcpu *pcpu);
324struct	pcpu *pcpu_find(u_int cpuid);
325void	pcpu_init(struct pcpu *pcpu, int cpuid, size_t size);
326
327#endif /* _KERNEL */
328
329#endif /* !_SYS_PCPU_H_ */
330