1139825Simp/*- 276440Sjhb * Copyright (c) 2001 Wind River Systems, Inc. 376440Sjhb * All rights reserved. 476440Sjhb * Written by: John Baldwin <jhb@FreeBSD.org> 576440Sjhb * 676440Sjhb * Redistribution and use in source and binary forms, with or without 776440Sjhb * modification, are permitted provided that the following conditions 876440Sjhb * are met: 976440Sjhb * 1. Redistributions of source code must retain the above copyright 1076440Sjhb * notice, this list of conditions and the following disclaimer. 1176440Sjhb * 2. Redistributions in binary form must reproduce the above copyright 1276440Sjhb * notice, this list of conditions and the following disclaimer in the 1376440Sjhb * documentation and/or other materials provided with the distribution. 1476440Sjhb * 4. Neither the name of the author nor the names of any co-contributors 1576440Sjhb * may be used to endorse or promote products derived from this software 1676440Sjhb * without specific prior written permission. 1776440Sjhb * 1876440Sjhb * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1976440Sjhb * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 2076440Sjhb * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2176440Sjhb * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 2276440Sjhb * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2376440Sjhb * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2476440Sjhb * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2576440Sjhb * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2676440Sjhb * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2776440Sjhb * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2876440Sjhb * SUCH DAMAGE. 2976440Sjhb * 3076440Sjhb * $FreeBSD$ 3176440Sjhb */ 3276440Sjhb 3376440Sjhb#ifndef _SYS_PCPU_H_ 34102191Sjhb#define _SYS_PCPU_H_ 3576440Sjhb 36102191Sjhb#ifdef LOCORE 37102191Sjhb#error "no assembler-serviceable parts inside" 38102191Sjhb#endif 39102191Sjhb 40222813Sattilio#include <sys/_cpuset.h> 4187702Sjhb#include <sys/queue.h> 4293823Sdillon#include <sys/vmmeter.h> 43174070Speter#include <sys/resource.h> 4487702Sjhb#include <machine/pcpu.h> 4576440Sjhb 46215315Sdim#define DPCPU_SETNAME "set_pcpu" 47215315Sdim#define DPCPU_SYMPREFIX "pcpu_entry_" 48215315Sdim 49215138Sdim#ifdef _KERNEL 50215138Sdim 51194784Sjeff/* 52194784Sjeff * Define a set for pcpu data. 53194784Sjeff */ 54194784Sjeffextern uintptr_t *__start_set_pcpu; 55215701Sdim__GLOBL(__start_set_pcpu); 56194784Sjeffextern uintptr_t *__stop_set_pcpu; 57215701Sdim__GLOBL(__stop_set_pcpu); 58194784Sjeff 59194784Sjeff/* 60194784Sjeff * Array of dynamic pcpu base offsets. Indexed by id. 61194784Sjeff */ 62194784Sjeffextern uintptr_t dpcpu_off[]; 63194784Sjeff 64194784Sjeff/* 65194784Sjeff * Convenience defines. 66194784Sjeff */ 67205885Sjhb#define DPCPU_START ((uintptr_t)&__start_set_pcpu) 68205885Sjhb#define DPCPU_STOP ((uintptr_t)&__stop_set_pcpu) 69194784Sjeff#define DPCPU_BYTES (DPCPU_STOP - DPCPU_START) 70194784Sjeff#define DPCPU_MODMIN 2048 71194784Sjeff#define DPCPU_SIZE roundup2(DPCPU_BYTES, PAGE_SIZE) 72194784Sjeff#define DPCPU_MODSIZE (DPCPU_SIZE - (DPCPU_BYTES - DPCPU_MODMIN)) 73194784Sjeff 74194784Sjeff/* 75194784Sjeff * Declaration and definition. 76194784Sjeff */ 77194784Sjeff#define DPCPU_NAME(n) pcpu_entry_##n 78194784Sjeff#define DPCPU_DECLARE(t, n) extern t DPCPU_NAME(n) 79215701Sdim#define DPCPU_DEFINE(t, n) t DPCPU_NAME(n) __section(DPCPU_SETNAME) __used 80194784Sjeff 81194784Sjeff/* 82194784Sjeff * Accessors with a given base. 83194784Sjeff */ 84194784Sjeff#define _DPCPU_PTR(b, n) \ 85194784Sjeff (__typeof(DPCPU_NAME(n))*)((b) + (uintptr_t)&DPCPU_NAME(n)) 86194784Sjeff#define _DPCPU_GET(b, n) (*_DPCPU_PTR(b, n)) 87194784Sjeff#define _DPCPU_SET(b, n, v) (*_DPCPU_PTR(b, n) = v) 88194784Sjeff 89194784Sjeff/* 90194784Sjeff * Accessors for the current cpu. 91194784Sjeff */ 92194784Sjeff#define DPCPU_PTR(n) _DPCPU_PTR(PCPU_GET(dynamic), n) 93194784Sjeff#define DPCPU_GET(n) (*DPCPU_PTR(n)) 94194784Sjeff#define DPCPU_SET(n, v) (*DPCPU_PTR(n) = v) 95194784Sjeff 96194784Sjeff/* 97194784Sjeff * Accessors for remote cpus. 98194784Sjeff */ 99194784Sjeff#define DPCPU_ID_PTR(i, n) _DPCPU_PTR(dpcpu_off[(i)], n) 100194784Sjeff#define DPCPU_ID_GET(i, n) (*DPCPU_ID_PTR(i, n)) 101194784Sjeff#define DPCPU_ID_SET(i, n, v) (*DPCPU_ID_PTR(i, n) = v) 102194784Sjeff 103209119Slstewart/* 104209119Slstewart * Utility macros. 105209119Slstewart */ 106209978Slstewart#define DPCPU_SUM(n) __extension__ \ 107209325Slstewart({ \ 108209325Slstewart u_int _i; \ 109209987Slstewart __typeof(*DPCPU_PTR(n)) sum; \ 110209978Slstewart \ 111209978Slstewart sum = 0; \ 112209978Slstewart CPU_FOREACH(_i) { \ 113210184Slstewart sum += *DPCPU_ID_PTR(_i, n); \ 114209978Slstewart } \ 115209978Slstewart sum; \ 116209978Slstewart}) 117209978Slstewart 118209978Slstewart#define DPCPU_VARSUM(n, var) __extension__ \ 119209978Slstewart({ \ 120209978Slstewart u_int _i; \ 121209325Slstewart __typeof((DPCPU_PTR(n))->var) sum; \ 122209325Slstewart \ 123209325Slstewart sum = 0; \ 124209325Slstewart CPU_FOREACH(_i) { \ 125209325Slstewart sum += (DPCPU_ID_PTR(_i, n))->var; \ 126209325Slstewart } \ 127209325Slstewart sum; \ 128209325Slstewart}) 129209119Slstewart 130209981Slstewart#define DPCPU_ZERO(n) do { \ 131209981Slstewart u_int _i; \ 132209981Slstewart \ 133209981Slstewart CPU_FOREACH(_i) { \ 134209987Slstewart bzero(DPCPU_ID_PTR(_i, n), sizeof(*DPCPU_PTR(n))); \ 135209981Slstewart } \ 136209981Slstewart} while(0) 137209981Slstewart 138215138Sdim#endif /* _KERNEL */ 139215138Sdim 140173444Sups/* 141173444Sups * XXXUPS remove as soon as we have per cpu variable 142205885Sjhb * linker sets and can define rm_queue in _rm_lock.h 143205885Sjhb */ 144173444Supsstruct rm_queue { 145173444Sups struct rm_queue* volatile rmq_next; 146173444Sups struct rm_queue* volatile rmq_prev; 147173444Sups}; 148173444Sups 14987702Sjhb/* 15087702Sjhb * This structure maps out the global data that needs to be kept on a 15187702Sjhb * per-cpu basis. The members are accessed via the PCPU_GET/SET/PTR 15292808Sjhb * macros defined in <machine/pcpu.h>. Machine dependent fields are 15392808Sjhb * defined in the PCPU_MD_FIELDS macro defined in <machine/pcpu.h>. 15487702Sjhb */ 15587702Sjhbstruct pcpu { 15687702Sjhb struct thread *pc_curthread; /* Current thread */ 15787702Sjhb struct thread *pc_idlethread; /* Idle thread */ 15887702Sjhb struct thread *pc_fpcurthread; /* Fp state owner */ 159107719Sjulian struct thread *pc_deadthread; /* Zombie thread or NULL */ 16087702Sjhb struct pcb *pc_curpcb; /* Current pcb */ 161205885Sjhb uint64_t pc_switchtime; /* cpu_ticks() at last csw */ 162205885Sjhb int pc_switchticks; /* `ticks' at last csw */ 16387702Sjhb u_int pc_cpuid; /* This cpu number */ 164222531Snwhitehorn STAILQ_ENTRY(pcpu) pc_allcpu; 16587702Sjhb struct lock_list_entry *pc_spinlocks; 16693823Sdillon struct vmmeter pc_cnt; /* VM stats counters */ 167174598Sobrien long pc_cp_time[CPUSTATES]; /* statclock ticks */ 168126763Snjl struct device *pc_device; 169205885Sjhb void *pc_netisr; /* netisr SWI cookie */ 170208504Salc int pc_dnweight; /* vm_page_dontneed() */ 171210550Sjhb int pc_domain; /* Memory domain. */ 172173444Sups 173205885Sjhb /* 174173444Sups * Stuff for read mostly lock 175205885Sjhb * 176173444Sups * XXXUPS remove as soon as we have per cpu variable 177173444Sups * linker sets. 178173444Sups */ 179205885Sjhb struct rm_queue pc_rm_queue; 180173444Sups 181205885Sjhb uintptr_t pc_dynamic; /* Dynamic per-cpu data area */ 182194784Sjeff 183194784Sjeff /* 184176878Smarcel * Keep MD fields last, so that CPU-specific variations on a 185176878Smarcel * single architecture don't result in offset variations of 186205885Sjhb * the machine-independent fields of the pcpu. Even though 187176878Smarcel * the pcpu structure is private to the kernel, some ports 188205885Sjhb * (e.g., lsof, part of gtop) define _KERNEL and include this 189205885Sjhb * header. While strictly speaking this is wrong, there's no 190205885Sjhb * reason not to keep the offsets of the MI fields constant 191205885Sjhb * if only to make kernel debugging easier. 192176878Smarcel */ 193176878Smarcel PCPU_MD_FIELDS; 194205915Sjhb} __aligned(CACHE_LINE_SIZE); 19587702Sjhb 196181875Sjhb#ifdef _KERNEL 197181875Sjhb 198222531SnwhitehornSTAILQ_HEAD(cpuhead, pcpu); 19987702Sjhb 20076440Sjhbextern struct cpuhead cpuhead; 201224221Sattilioextern struct pcpu *cpuid_to_pcpu[]; 20276440Sjhb 203145544Srwatson#define curcpu PCPU_GET(cpuid) 20487702Sjhb#define curproc (curthread->td_proc) 205122929Speter#ifndef curthread 206102191Sjhb#define curthread PCPU_GET(curthread) 207122929Speter#endif 208195699Srwatson#define curvidata PCPU_GET(vidata) 20976440Sjhb 21087702Sjhb/* 21187702Sjhb * Machine dependent callouts. cpu_pcpu_init() is responsible for 21287702Sjhb * initializing machine dependent fields of struct pcpu, and 21387702Sjhb * db_show_mdpcpu() is responsible for handling machine dependent 21487702Sjhb * fields for the DDB 'show pcpu' command. 21587702Sjhb */ 21687702Sjhbvoid cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size); 21787702Sjhbvoid db_show_mdpcpu(struct pcpu *pcpu); 21887702Sjhb 219194784Sjeffvoid *dpcpu_alloc(int size); 220194784Sjeffvoid dpcpu_copy(void *s, int size); 221194784Sjeffvoid dpcpu_free(void *s, int size); 222194784Sjeffvoid dpcpu_init(void *dpcpu, int cpuid); 223205885Sjhbvoid pcpu_destroy(struct pcpu *pcpu); 224205885Sjhbstruct pcpu *pcpu_find(u_int cpuid); 225205885Sjhbvoid pcpu_init(struct pcpu *pcpu, int cpuid, size_t size); 22687702Sjhb 227205885Sjhb#endif /* _KERNEL */ 228181875Sjhb 229102191Sjhb#endif /* !_SYS_PCPU_H_ */ 230