1139825Simp/*- 276440Sjhb * Copyright (c) 2001 Wind River Systems, Inc. 376440Sjhb * All rights reserved. 476440Sjhb * Written by: John Baldwin <jhb@FreeBSD.org> 576440Sjhb * 676440Sjhb * Redistribution and use in source and binary forms, with or without 776440Sjhb * modification, are permitted provided that the following conditions 876440Sjhb * are met: 976440Sjhb * 1. Redistributions of source code must retain the above copyright 1076440Sjhb * notice, this list of conditions and the following disclaimer. 1176440Sjhb * 2. Redistributions in binary form must reproduce the above copyright 1276440Sjhb * notice, this list of conditions and the following disclaimer in the 1376440Sjhb * documentation and/or other materials provided with the distribution. 1476440Sjhb * 4. Neither the name of the author nor the names of any co-contributors 1576440Sjhb * may be used to endorse or promote products derived from this software 1676440Sjhb * without specific prior written permission. 1776440Sjhb * 1876440Sjhb * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1976440Sjhb * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 2076440Sjhb * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2176440Sjhb * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 2276440Sjhb * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2376440Sjhb * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2476440Sjhb * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2576440Sjhb * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2676440Sjhb * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2776440Sjhb * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2876440Sjhb * SUCH DAMAGE. 2976440Sjhb * 3076440Sjhb * $FreeBSD$ 3176440Sjhb */ 3276440Sjhb 3376440Sjhb#ifndef _SYS_PCPU_H_ 34102191Sjhb#define _SYS_PCPU_H_ 3576440Sjhb 36102191Sjhb#ifdef LOCORE 37102191Sjhb#error "no assembler-serviceable parts inside" 38102191Sjhb#endif 39102191Sjhb 40222813Sattilio#include <sys/_cpuset.h> 41240624Sattilio#include <sys/_lock.h> 42240624Sattilio#include <sys/_mutex.h> 43240624Sattilio#include <sys/_sx.h> 4487702Sjhb#include <sys/queue.h> 45240624Sattilio#include <sys/_rmlock.h> 4693823Sdillon#include <sys/vmmeter.h> 47174070Speter#include <sys/resource.h> 4887702Sjhb#include <machine/pcpu.h> 4976440Sjhb 50215315Sdim#define DPCPU_SETNAME "set_pcpu" 51215315Sdim#define DPCPU_SYMPREFIX "pcpu_entry_" 52215315Sdim 53215138Sdim#ifdef _KERNEL 54215138Sdim 55194784Sjeff/* 56194784Sjeff * Define a set for pcpu data. 57194784Sjeff */ 58194784Sjeffextern uintptr_t *__start_set_pcpu; 59215701Sdim__GLOBL(__start_set_pcpu); 60194784Sjeffextern uintptr_t *__stop_set_pcpu; 61215701Sdim__GLOBL(__stop_set_pcpu); 62194784Sjeff 63194784Sjeff/* 64194784Sjeff * Array of dynamic pcpu base offsets. Indexed by id. 65194784Sjeff */ 66194784Sjeffextern uintptr_t dpcpu_off[]; 67194784Sjeff 68194784Sjeff/* 69194784Sjeff * Convenience defines. 70194784Sjeff */ 71205885Sjhb#define DPCPU_START ((uintptr_t)&__start_set_pcpu) 72205885Sjhb#define DPCPU_STOP ((uintptr_t)&__stop_set_pcpu) 73194784Sjeff#define DPCPU_BYTES (DPCPU_STOP - DPCPU_START) 74194784Sjeff#define DPCPU_MODMIN 2048 75194784Sjeff#define DPCPU_SIZE roundup2(DPCPU_BYTES, PAGE_SIZE) 76194784Sjeff#define DPCPU_MODSIZE (DPCPU_SIZE - (DPCPU_BYTES - DPCPU_MODMIN)) 77194784Sjeff 78194784Sjeff/* 79194784Sjeff * Declaration and definition. 80194784Sjeff */ 81194784Sjeff#define DPCPU_NAME(n) pcpu_entry_##n 82194784Sjeff#define DPCPU_DECLARE(t, n) extern t DPCPU_NAME(n) 83215701Sdim#define DPCPU_DEFINE(t, n) t DPCPU_NAME(n) __section(DPCPU_SETNAME) __used 84194784Sjeff 85194784Sjeff/* 86194784Sjeff * Accessors with a given base. 87194784Sjeff */ 88194784Sjeff#define _DPCPU_PTR(b, n) \ 89194784Sjeff (__typeof(DPCPU_NAME(n))*)((b) + (uintptr_t)&DPCPU_NAME(n)) 90194784Sjeff#define _DPCPU_GET(b, n) (*_DPCPU_PTR(b, n)) 91194784Sjeff#define _DPCPU_SET(b, n, v) (*_DPCPU_PTR(b, n) = v) 92194784Sjeff 93194784Sjeff/* 94194784Sjeff * Accessors for the current cpu. 95194784Sjeff */ 96194784Sjeff#define DPCPU_PTR(n) _DPCPU_PTR(PCPU_GET(dynamic), n) 97194784Sjeff#define DPCPU_GET(n) (*DPCPU_PTR(n)) 98194784Sjeff#define DPCPU_SET(n, v) (*DPCPU_PTR(n) = v) 99194784Sjeff 100194784Sjeff/* 101194784Sjeff * Accessors for remote cpus. 102194784Sjeff */ 103194784Sjeff#define DPCPU_ID_PTR(i, n) _DPCPU_PTR(dpcpu_off[(i)], n) 104194784Sjeff#define DPCPU_ID_GET(i, n) (*DPCPU_ID_PTR(i, n)) 105194784Sjeff#define DPCPU_ID_SET(i, n, v) (*DPCPU_ID_PTR(i, n) = v) 106194784Sjeff 107209119Slstewart/* 108209119Slstewart * Utility macros. 109209119Slstewart */ 110209978Slstewart#define DPCPU_SUM(n) __extension__ \ 111209325Slstewart({ \ 112209325Slstewart u_int _i; \ 113209987Slstewart __typeof(*DPCPU_PTR(n)) sum; \ 114209978Slstewart \ 115209978Slstewart sum = 0; \ 116209978Slstewart CPU_FOREACH(_i) { \ 117210184Slstewart sum += *DPCPU_ID_PTR(_i, n); \ 118209978Slstewart } \ 119209978Slstewart sum; \ 120209978Slstewart}) 121209978Slstewart 122209978Slstewart#define DPCPU_VARSUM(n, var) __extension__ \ 123209978Slstewart({ \ 124209978Slstewart u_int _i; \ 125209325Slstewart __typeof((DPCPU_PTR(n))->var) sum; \ 126209325Slstewart \ 127209325Slstewart sum = 0; \ 128209325Slstewart CPU_FOREACH(_i) { \ 129209325Slstewart sum += (DPCPU_ID_PTR(_i, n))->var; \ 130209325Slstewart } \ 131209325Slstewart sum; \ 132209325Slstewart}) 133209119Slstewart 134209981Slstewart#define DPCPU_ZERO(n) do { \ 135209981Slstewart u_int _i; \ 136209981Slstewart \ 137209981Slstewart CPU_FOREACH(_i) { \ 138209987Slstewart bzero(DPCPU_ID_PTR(_i, n), sizeof(*DPCPU_PTR(n))); \ 139209981Slstewart } \ 140209981Slstewart} while(0) 141209981Slstewart 142215138Sdim#endif /* _KERNEL */ 143215138Sdim 14487702Sjhb/* 14587702Sjhb * This structure maps out the global data that needs to be kept on a 14687702Sjhb * per-cpu basis. The members are accessed via the PCPU_GET/SET/PTR 14792808Sjhb * macros defined in <machine/pcpu.h>. Machine dependent fields are 14892808Sjhb * defined in the PCPU_MD_FIELDS macro defined in <machine/pcpu.h>. 14987702Sjhb */ 15087702Sjhbstruct pcpu { 15187702Sjhb struct thread *pc_curthread; /* Current thread */ 15287702Sjhb struct thread *pc_idlethread; /* Idle thread */ 15387702Sjhb struct thread *pc_fpcurthread; /* Fp state owner */ 154107719Sjulian struct thread *pc_deadthread; /* Zombie thread or NULL */ 15587702Sjhb struct pcb *pc_curpcb; /* Current pcb */ 156205885Sjhb uint64_t pc_switchtime; /* cpu_ticks() at last csw */ 157205885Sjhb int pc_switchticks; /* `ticks' at last csw */ 15887702Sjhb u_int pc_cpuid; /* This cpu number */ 159222531Snwhitehorn STAILQ_ENTRY(pcpu) pc_allcpu; 16087702Sjhb struct lock_list_entry *pc_spinlocks; 16193823Sdillon struct vmmeter pc_cnt; /* VM stats counters */ 162174598Sobrien long pc_cp_time[CPUSTATES]; /* statclock ticks */ 163126763Snjl struct device *pc_device; 164205885Sjhb void *pc_netisr; /* netisr SWI cookie */ 165208504Salc int pc_dnweight; /* vm_page_dontneed() */ 166210550Sjhb int pc_domain; /* Memory domain. */ 167240624Sattilio struct rm_queue pc_rm_queue; /* rmlock list of trackers */ 168205885Sjhb uintptr_t pc_dynamic; /* Dynamic per-cpu data area */ 169194784Sjeff 170194784Sjeff /* 171176878Smarcel * Keep MD fields last, so that CPU-specific variations on a 172176878Smarcel * single architecture don't result in offset variations of 173205885Sjhb * the machine-independent fields of the pcpu. Even though 174176878Smarcel * the pcpu structure is private to the kernel, some ports 175205885Sjhb * (e.g., lsof, part of gtop) define _KERNEL and include this 176205885Sjhb * header. While strictly speaking this is wrong, there's no 177205885Sjhb * reason not to keep the offsets of the MI fields constant 178205885Sjhb * if only to make kernel debugging easier. 179176878Smarcel */ 180176878Smarcel PCPU_MD_FIELDS; 181205915Sjhb} __aligned(CACHE_LINE_SIZE); 18287702Sjhb 183249265Sglebius#ifdef CTASSERT 184249265Sglebius/* 185249265Sglebius * To minimize memory waste in per-cpu UMA zones, size of struct pcpu 186249265Sglebius * should be denominator of PAGE_SIZE. 187249265Sglebius */ 188249265SglebiusCTASSERT((PAGE_SIZE / sizeof(struct pcpu)) * sizeof(struct pcpu) == PAGE_SIZE); 189249265Sglebius#endif 190249265Sglebius 191181875Sjhb#ifdef _KERNEL 192181875Sjhb 193222531SnwhitehornSTAILQ_HEAD(cpuhead, pcpu); 19487702Sjhb 19576440Sjhbextern struct cpuhead cpuhead; 196224221Sattilioextern struct pcpu *cpuid_to_pcpu[]; 19776440Sjhb 198145544Srwatson#define curcpu PCPU_GET(cpuid) 19987702Sjhb#define curproc (curthread->td_proc) 200122929Speter#ifndef curthread 201102191Sjhb#define curthread PCPU_GET(curthread) 202122929Speter#endif 203195699Srwatson#define curvidata PCPU_GET(vidata) 20476440Sjhb 205249265Sglebius/* Accessor to elements allocated via UMA_ZONE_PCPU zone. */ 206249265Sglebiusstatic inline void * 207249265Sglebiuszpcpu_get(void *base) 208249265Sglebius{ 209249265Sglebius 210249265Sglebius return ((char *)(base) + sizeof(struct pcpu) * curcpu); 211249265Sglebius} 212249265Sglebius 213262739Sglebiusstatic inline void * 214262739Sglebiuszpcpu_get_cpu(void *base, int cpu) 215262739Sglebius{ 216262739Sglebius 217262739Sglebius return ((char *)(base) + sizeof(struct pcpu) * cpu); 218262739Sglebius} 219262739Sglebius 22087702Sjhb/* 22187702Sjhb * Machine dependent callouts. cpu_pcpu_init() is responsible for 22287702Sjhb * initializing machine dependent fields of struct pcpu, and 22387702Sjhb * db_show_mdpcpu() is responsible for handling machine dependent 22487702Sjhb * fields for the DDB 'show pcpu' command. 22587702Sjhb */ 22687702Sjhbvoid cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size); 22787702Sjhbvoid db_show_mdpcpu(struct pcpu *pcpu); 22887702Sjhb 229194784Sjeffvoid *dpcpu_alloc(int size); 230194784Sjeffvoid dpcpu_copy(void *s, int size); 231194784Sjeffvoid dpcpu_free(void *s, int size); 232194784Sjeffvoid dpcpu_init(void *dpcpu, int cpuid); 233205885Sjhbvoid pcpu_destroy(struct pcpu *pcpu); 234205885Sjhbstruct pcpu *pcpu_find(u_int cpuid); 235205885Sjhbvoid pcpu_init(struct pcpu *pcpu, int cpuid, size_t size); 23687702Sjhb 237205885Sjhb#endif /* _KERNEL */ 238181875Sjhb 239102191Sjhb#endif /* !_SYS_PCPU_H_ */ 240