1145256Sjkoshy/*- 2174395Sjkoshy * Copyright (c) 2003-2007, Joseph Koshy 3174395Sjkoshy * Copyright (c) 2007 The FreeBSD Foundation 4145256Sjkoshy * All rights reserved. 5145256Sjkoshy * 6174395Sjkoshy * Portions of this software were developed by A. Joseph Koshy under 7174395Sjkoshy * sponsorship from the FreeBSD Foundation and Google, Inc. 8174395Sjkoshy * 9145256Sjkoshy * Redistribution and use in source and binary forms, with or without 10145256Sjkoshy * modification, are permitted provided that the following conditions 11145256Sjkoshy * are met: 12145256Sjkoshy * 1. Redistributions of source code must retain the above copyright 13145256Sjkoshy * notice, this list of conditions and the following disclaimer. 14145256Sjkoshy * 2. Redistributions in binary form must reproduce the above copyright 15145256Sjkoshy * notice, this list of conditions and the following disclaimer in the 16145256Sjkoshy * documentation and/or other materials provided with the distribution. 17145256Sjkoshy * 18145256Sjkoshy * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19145256Sjkoshy * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20145256Sjkoshy * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21145256Sjkoshy * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22145256Sjkoshy * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23145256Sjkoshy * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24145256Sjkoshy * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25145256Sjkoshy * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26145256Sjkoshy * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27145256Sjkoshy * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28145256Sjkoshy * SUCH DAMAGE. 29145256Sjkoshy * 30145256Sjkoshy * $FreeBSD$ 31145256Sjkoshy */ 32145256Sjkoshy 33145256Sjkoshy/* 34145256Sjkoshy * PMC interface used by the base kernel. 35145256Sjkoshy */ 36145256Sjkoshy 37145256Sjkoshy#ifndef _SYS_PMCKERN_H_ 38145256Sjkoshy#define _SYS_PMCKERN_H_ 39145256Sjkoshy 40145256Sjkoshy#include <sys/param.h> 41236238Sfabient#include <sys/systm.h> 42145256Sjkoshy#include <sys/kernel.h> 43145256Sjkoshy#include <sys/lock.h> 44145256Sjkoshy#include <sys/proc.h> 45145256Sjkoshy#include <sys/sx.h> 46236238Sfabient#include <sys/pmc.h> 47145256Sjkoshy 48236238Sfabient#include <machine/cpufunc.h> 49236238Sfabient 50146799Sjkoshy#define PMC_FN_PROCESS_EXEC 1 51146799Sjkoshy#define PMC_FN_CSW_IN 2 52146799Sjkoshy#define PMC_FN_CSW_OUT 3 53146799Sjkoshy#define PMC_FN_DO_SAMPLES 4 54262038Savg#define PMC_FN_UNUSED1 5 55262038Savg#define PMC_FN_UNUSED2 6 56157144Sjkoshy#define PMC_FN_MMAP 7 57157144Sjkoshy#define PMC_FN_MUNMAP 8 58174395Sjkoshy#define PMC_FN_USER_CALLCHAIN 9 59236238Sfabient#define PMC_FN_USER_CALLCHAIN_SOFT 10 60236238Sfabient#define PMC_FN_SOFT_SAMPLING 11 61145256Sjkoshy 62236238Sfabient#define PMC_HR 0 /* Hardware ring buffer */ 63236238Sfabient#define PMC_SR 1 /* Software ring buffer */ 64236238Sfabient 65147708Sjkoshystruct pmckern_procexec { 66147708Sjkoshy int pm_credentialschanged; 67157144Sjkoshy uintfptr_t pm_entryaddr; 68147708Sjkoshy}; 69147708Sjkoshy 70157144Sjkoshystruct pmckern_map_in { 71157144Sjkoshy void *pm_file; /* filename or vnode pointer */ 72157144Sjkoshy uintfptr_t pm_address; /* address object is loaded at */ 73157144Sjkoshy}; 74157144Sjkoshy 75157144Sjkoshystruct pmckern_map_out { 76157144Sjkoshy uintfptr_t pm_address; /* start address of region */ 77157144Sjkoshy size_t pm_size; /* size of unmapped region */ 78157144Sjkoshy}; 79157144Sjkoshy 80236238Sfabientstruct pmckern_soft { 81236238Sfabient enum pmc_event pm_ev; 82236238Sfabient int pm_cpu; 83236238Sfabient struct trapframe *pm_tf; 84236238Sfabient}; 85236238Sfabient 86236238Sfabient/* 87236238Sfabient * Soft PMC. 88236238Sfabient */ 89236238Sfabient 90250600Sfabient#define PMC_SOFT_DEFINE_EX(prov, mod, func, name, alloc, release) \ 91236238Sfabient struct pmc_soft pmc_##prov##_##mod##_##func##_##name = \ 92250600Sfabient { 0, alloc, release, { #prov "_" #mod "_" #func "." #name, 0 } }; \ 93236238Sfabient SYSINIT(pmc_##prov##_##mod##_##func##_##name##_init, SI_SUB_KDTRACE, \ 94236238Sfabient SI_ORDER_SECOND + 1, pmc_soft_ev_register, \ 95236238Sfabient &pmc_##prov##_##mod##_##func##_##name ); \ 96236238Sfabient SYSUNINIT(pmc_##prov##_##mod##_##func##_##name##_uninit, \ 97236238Sfabient SI_SUB_KDTRACE, SI_ORDER_SECOND + 1, pmc_soft_ev_deregister, \ 98236238Sfabient &pmc_##prov##_##mod##_##func##_##name ) 99236238Sfabient 100250600Sfabient#define PMC_SOFT_DEFINE(prov, mod, func, name) \ 101250600Sfabient PMC_SOFT_DEFINE_EX(prov, mod, func, name, NULL, NULL) 102250600Sfabient 103236238Sfabient#define PMC_SOFT_DECLARE(prov, mod, func, name) \ 104236238Sfabient extern struct pmc_soft pmc_##prov##_##mod##_##func##_##name 105236238Sfabient 106236238Sfabient/* 107236238Sfabient * PMC_SOFT_CALL can be used anywhere in the kernel. 108236238Sfabient * Require md defined PMC_FAKE_TRAPFRAME. 109236238Sfabient */ 110236238Sfabient#ifdef PMC_FAKE_TRAPFRAME 111236238Sfabient#define PMC_SOFT_CALL(pr, mo, fu, na) \ 112236238Sfabientdo { \ 113236238Sfabient if (pmc_##pr##_##mo##_##fu##_##na.ps_running) { \ 114236238Sfabient struct pmckern_soft ks; \ 115236238Sfabient register_t intr; \ 116236238Sfabient intr = intr_disable(); \ 117236238Sfabient PMC_FAKE_TRAPFRAME(&pmc_tf[curcpu]); \ 118236238Sfabient ks.pm_ev = pmc_##pr##_##mo##_##fu##_##na.ps_ev.pm_ev_code; \ 119236238Sfabient ks.pm_cpu = PCPU_GET(cpuid); \ 120236238Sfabient ks.pm_tf = &pmc_tf[curcpu]; \ 121236238Sfabient PMC_CALL_HOOK_UNLOCKED(curthread, \ 122236238Sfabient PMC_FN_SOFT_SAMPLING, (void *) &ks); \ 123236238Sfabient intr_restore(intr); \ 124236238Sfabient } \ 125236238Sfabient} while (0) 126236238Sfabient#else 127236238Sfabient#define PMC_SOFT_CALL(pr, mo, fu, na) \ 128236238Sfabientdo { \ 129236238Sfabient} while (0) 130236238Sfabient#endif 131236238Sfabient 132236238Sfabient/* 133236238Sfabient * PMC_SOFT_CALL_TF need to be used carefully. 134236238Sfabient * Userland capture will be done during AST processing. 135236238Sfabient */ 136236238Sfabient#define PMC_SOFT_CALL_TF(pr, mo, fu, na, tf) \ 137236238Sfabientdo { \ 138236238Sfabient if (pmc_##pr##_##mo##_##fu##_##na.ps_running) { \ 139236238Sfabient struct pmckern_soft ks; \ 140236238Sfabient register_t intr; \ 141236238Sfabient intr = intr_disable(); \ 142236238Sfabient ks.pm_ev = pmc_##pr##_##mo##_##fu##_##na.ps_ev.pm_ev_code; \ 143236238Sfabient ks.pm_cpu = PCPU_GET(cpuid); \ 144236238Sfabient ks.pm_tf = tf; \ 145236238Sfabient PMC_CALL_HOOK_UNLOCKED(curthread, \ 146236238Sfabient PMC_FN_SOFT_SAMPLING, (void *) &ks); \ 147236238Sfabient intr_restore(intr); \ 148236238Sfabient } \ 149236238Sfabient} while (0) 150236238Sfabient 151236238Sfabientstruct pmc_soft { 152236238Sfabient int ps_running; 153250600Sfabient void (*ps_alloc)(void); 154250600Sfabient void (*ps_release)(void); 155236238Sfabient struct pmc_dyn_event_descr ps_ev; 156236238Sfabient}; 157236238Sfabient 158145256Sjkoshy/* hook */ 159145256Sjkoshyextern int (*pmc_hook)(struct thread *_td, int _function, void *_arg); 160174395Sjkoshyextern int (*pmc_intr)(int _cpu, struct trapframe *_frame); 161145256Sjkoshy 162145256Sjkoshy/* SX lock protecting the hook */ 163145256Sjkoshyextern struct sx pmc_sx; 164145256Sjkoshy 165146799Sjkoshy/* Per-cpu flags indicating availability of sampling data */ 166222813Sattilioextern volatile cpuset_t pmc_cpumask; 167146799Sjkoshy 168147191Sjkoshy/* Count of system-wide sampling PMCs in existence */ 169147191Sjkoshyextern volatile int pmc_ss_count; 170147191Sjkoshy 171148562Sjkoshy/* kernel version number */ 172148562Sjkoshyextern const int pmc_kernel_version; 173148562Sjkoshy 174236238Sfabient/* PMC soft per cpu trapframe */ 175236238Sfabientextern struct trapframe pmc_tf[MAXCPU]; 176236238Sfabient 177146799Sjkoshy/* Hook invocation; for use within the kernel */ 178145256Sjkoshy#define PMC_CALL_HOOK(t, cmd, arg) \ 179145256Sjkoshydo { \ 180145256Sjkoshy sx_slock(&pmc_sx); \ 181145256Sjkoshy if (pmc_hook != NULL) \ 182145256Sjkoshy (pmc_hook)((t), (cmd), (arg)); \ 183145256Sjkoshy sx_sunlock(&pmc_sx); \ 184145256Sjkoshy} while (0) 185145256Sjkoshy 186146799Sjkoshy/* Hook invocation that needs an exclusive lock */ 187145256Sjkoshy#define PMC_CALL_HOOK_X(t, cmd, arg) \ 188145256Sjkoshydo { \ 189145256Sjkoshy sx_xlock(&pmc_sx); \ 190145256Sjkoshy if (pmc_hook != NULL) \ 191145256Sjkoshy (pmc_hook)((t), (cmd), (arg)); \ 192145256Sjkoshy sx_xunlock(&pmc_sx); \ 193145256Sjkoshy} while (0) 194145256Sjkoshy 195146799Sjkoshy/* 196146799Sjkoshy * Some hook invocations (e.g., from context switch and clock handling 197146799Sjkoshy * code) need to be lock-free. 198146799Sjkoshy */ 199146799Sjkoshy#define PMC_CALL_HOOK_UNLOCKED(t, cmd, arg) \ 200145256Sjkoshydo { \ 201145256Sjkoshy if (pmc_hook != NULL) \ 202146799Sjkoshy (pmc_hook)((t), (cmd), (arg)); \ 203145256Sjkoshy} while (0) 204145256Sjkoshy 205146799Sjkoshy#define PMC_SWITCH_CONTEXT(t,cmd) PMC_CALL_HOOK_UNLOCKED(t,cmd,NULL) 206145256Sjkoshy 207146799Sjkoshy/* Check if a process is using HWPMCs.*/ 208145256Sjkoshy#define PMC_PROC_IS_USING_PMCS(p) \ 209239798Sjimharris (__predict_false(p->p_flag & P_HWPMC)) 210145256Sjkoshy 211236238Sfabient/* Check if a thread have pending user capture. */ 212236238Sfabient#define PMC_IS_PENDING_CALLCHAIN(p) \ 213236238Sfabient (__predict_false((p)->td_pflags & TDP_CALLCHAIN)) 214236238Sfabient 215147191Sjkoshy#define PMC_SYSTEM_SAMPLING_ACTIVE() (pmc_ss_count > 0) 216147191Sjkoshy 217146799Sjkoshy/* Check if a CPU has recorded samples. */ 218222813Sattilio#define PMC_CPU_HAS_SAMPLES(C) (__predict_false(CPU_ISSET(C, &pmc_cpumask))) 219146799Sjkoshy 220183266Sjkoshy/* 221183266Sjkoshy * Helper functions. 222183266Sjkoshy */ 223183266Sjkoshyint pmc_cpu_is_disabled(int _cpu); /* deprecated */ 224183266Sjkoshyint pmc_cpu_is_active(int _cpu); 225183266Sjkoshyint pmc_cpu_is_present(int _cpu); 226183266Sjkoshyint pmc_cpu_is_primary(int _cpu); 227183266Sjkoshyunsigned int pmc_cpu_max(void); 228145256Sjkoshy 229183266Sjkoshy#ifdef INVARIANTS 230183266Sjkoshyint pmc_cpu_max_active(void); 231183266Sjkoshy#endif 232183266Sjkoshy 233236238Sfabient/* 234236238Sfabient * Soft events functions. 235236238Sfabient */ 236236238Sfabientvoid pmc_soft_ev_register(struct pmc_soft *ps); 237236238Sfabientvoid pmc_soft_ev_deregister(struct pmc_soft *ps); 238236238Sfabientstruct pmc_soft *pmc_soft_ev_acquire(enum pmc_event ev); 239236238Sfabientvoid pmc_soft_ev_release(struct pmc_soft *ps); 240236238Sfabient 241145256Sjkoshy#endif /* _SYS_PMCKERN_H_ */ 242