1/*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2003-2007, Joseph Koshy 5 * Copyright (c) 2007 The FreeBSD Foundation 6 * All rights reserved. 7 * 8 * Portions of this software were developed by A. Joseph Koshy under 9 * sponsorship from the FreeBSD Foundation and Google, Inc. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33/* 34 * PMC interface used by the base kernel. 35 */ 36 37#ifndef _SYS_PMCKERN_H_ 38#define _SYS_PMCKERN_H_ 39 40#include <sys/param.h> 41#include <sys/systm.h> 42#include <sys/kernel.h> 43#include <sys/lock.h> 44#include <sys/proc.h> 45#include <sys/sx.h> 46#include <sys/pmc.h> 47 48#include <machine/cpufunc.h> 49 50#define PMC_FN_PROCESS_EXEC 1 51#define PMC_FN_CSW_IN 2 52#define PMC_FN_CSW_OUT 3 53#define PMC_FN_DO_SAMPLES 4 54#define PMC_FN_UNUSED1 5 55#define PMC_FN_UNUSED2 6 56#define PMC_FN_MMAP 7 57#define PMC_FN_MUNMAP 8 58#define PMC_FN_USER_CALLCHAIN 9 59#define PMC_FN_USER_CALLCHAIN_SOFT 10 60#define PMC_FN_SOFT_SAMPLING 11 61#define PMC_FN_THR_CREATE 12 62#define PMC_FN_THR_EXIT 13 63#define PMC_FN_THR_USERRET 14 64#define PMC_FN_THR_CREATE_LOG 15 65#define PMC_FN_THR_EXIT_LOG 16 66#define PMC_FN_PROC_CREATE_LOG 17 67 68typedef enum ring_type { 69 PMC_HR = 0, /* Hardware ring buffer */ 70 PMC_SR = 1, /* Software ring buffer */ 71 PMC_UR = 2, /* userret ring buffer */ 72 PMC_NUM_SR = PMC_UR+1 73} ring_type_t; 74 75struct pmckern_procexec { 76 int pm_credentialschanged; 77 uintptr_t pm_baseaddr; 78 uintptr_t pm_dynaddr; 79}; 80 81struct pmckern_map_in { 82 void *pm_file; /* filename or vnode pointer */ 83 uintfptr_t pm_address; /* address object is loaded at */ 84}; 85 86struct pmckern_map_out { 87 uintfptr_t pm_address; /* start address of region */ 88 size_t pm_size; /* size of unmapped region */ 89}; 90 91struct pmckern_soft { 92 enum pmc_event pm_ev; 93 int pm_cpu; 94 struct trapframe *pm_tf; 95}; 96 97/* 98 * Soft PMC. 99 */ 100 101#define PMC_SOFT_DEFINE_EX(prov, mod, func, name, alloc, release) \ 102 struct pmc_soft pmc_##prov##_##mod##_##func##_##name = \ 103 { 0, alloc, release, { #prov "_" #mod "_" #func "." #name, 0 } }; \ 104 SYSINIT(pmc_##prov##_##mod##_##func##_##name##_init, SI_SUB_KDTRACE, \ 105 SI_ORDER_SECOND + 1, pmc_soft_ev_register, \ 106 &pmc_##prov##_##mod##_##func##_##name ); \ 107 SYSUNINIT(pmc_##prov##_##mod##_##func##_##name##_uninit, \ 108 SI_SUB_KDTRACE, SI_ORDER_SECOND + 1, pmc_soft_ev_deregister, \ 109 &pmc_##prov##_##mod##_##func##_##name ) 110 111#define PMC_SOFT_DEFINE(prov, mod, func, name) \ 112 PMC_SOFT_DEFINE_EX(prov, mod, func, name, NULL, NULL) 113 114#define PMC_SOFT_DECLARE(prov, mod, func, name) \ 115 extern struct pmc_soft pmc_##prov##_##mod##_##func##_##name 116 117/* 118 * PMC_SOFT_CALL can be used anywhere in the kernel. 119 * Require md defined PMC_FAKE_TRAPFRAME. 120 */ 121#ifdef PMC_FAKE_TRAPFRAME 122#define PMC_SOFT_CALL(pr, mo, fu, na) \ 123do { \ 124 if (__predict_false(pmc_##pr##_##mo##_##fu##_##na.ps_running)) { \ 125 struct pmckern_soft ks; \ 126 register_t intr; \ 127 intr = intr_disable(); \ 128 PMC_FAKE_TRAPFRAME(&pmc_tf[curcpu]); \ 129 ks.pm_ev = pmc_##pr##_##mo##_##fu##_##na.ps_ev.pm_ev_code; \ 130 ks.pm_cpu = PCPU_GET(cpuid); \ 131 ks.pm_tf = &pmc_tf[curcpu]; \ 132 PMC_CALL_HOOK_UNLOCKED(curthread, \ 133 PMC_FN_SOFT_SAMPLING, (void *) &ks); \ 134 intr_restore(intr); \ 135 } \ 136} while (0) 137#else 138#define PMC_SOFT_CALL(pr, mo, fu, na) \ 139do { \ 140} while (0) 141#endif 142 143/* 144 * PMC_SOFT_CALL_TF need to be used carefully. 145 * Userland capture will be done during AST processing. 146 */ 147#define PMC_SOFT_CALL_TF(pr, mo, fu, na, tf) \ 148do { \ 149 if (__predict_false(pmc_##pr##_##mo##_##fu##_##na.ps_running)) { \ 150 struct pmckern_soft ks; \ 151 register_t intr; \ 152 intr = intr_disable(); \ 153 ks.pm_ev = pmc_##pr##_##mo##_##fu##_##na.ps_ev.pm_ev_code; \ 154 ks.pm_cpu = PCPU_GET(cpuid); \ 155 ks.pm_tf = tf; \ 156 PMC_CALL_HOOK_UNLOCKED(curthread, \ 157 PMC_FN_SOFT_SAMPLING, (void *) &ks); \ 158 intr_restore(intr); \ 159 } \ 160} while (0) 161 162struct pmc_soft { 163 int ps_running; 164 void (*ps_alloc)(void); 165 void (*ps_release)(void); 166 struct pmc_dyn_event_descr ps_ev; 167}; 168 169struct pmclog_buffer; 170 171struct pmc_domain_buffer_header { 172 struct mtx pdbh_mtx; 173 TAILQ_HEAD(, pmclog_buffer) pdbh_head; 174 struct pmclog_buffer *pdbh_plbs; 175 int pdbh_ncpus; 176} __aligned(CACHE_LINE_SIZE); 177 178/* hook */ 179extern int (*pmc_hook)(struct thread *_td, int _function, void *_arg); 180extern int (*pmc_intr)(struct trapframe *_frame); 181 182/* SX lock protecting the hook */ 183extern struct sx pmc_sx; 184 185/* Per-cpu flags indicating availability of sampling data */ 186DPCPU_DECLARE(uint8_t, pmc_sampled); 187 188/* Count of system-wide sampling PMCs in existence */ 189extern volatile int pmc_ss_count; 190 191/* kernel version number */ 192extern const int pmc_kernel_version; 193 194/* PMC soft per cpu trapframe */ 195extern struct trapframe pmc_tf[MAXCPU]; 196 197/* per domain buffer header list */ 198extern struct pmc_domain_buffer_header *pmc_dom_hdrs[MAXMEMDOM]; 199 200/* Quick check if preparatory work is necessary */ 201#define PMC_HOOK_INSTALLED(cmd) __predict_false(pmc_hook != NULL) 202 203/* Hook invocation; for use within the kernel */ 204#define PMC_CALL_HOOK(t, cmd, arg) \ 205do { \ 206 struct epoch_tracker et; \ 207 epoch_enter_preempt(global_epoch_preempt, &et); \ 208 if (pmc_hook != NULL) \ 209 (pmc_hook)((t), (cmd), (arg)); \ 210 epoch_exit_preempt(global_epoch_preempt, &et); \ 211} while (0) 212 213/* Hook invocation that needs an exclusive lock */ 214#define PMC_CALL_HOOK_X(t, cmd, arg) \ 215do { \ 216 sx_xlock(&pmc_sx); \ 217 if (pmc_hook != NULL) \ 218 (pmc_hook)((t), (cmd), (arg)); \ 219 sx_xunlock(&pmc_sx); \ 220} while (0) 221 222/* 223 * Some hook invocations (e.g., from context switch and clock handling 224 * code) need to be lock-free. 225 */ 226#define PMC_CALL_HOOK_UNLOCKED(t, cmd, arg) \ 227do { \ 228 if (pmc_hook != NULL) \ 229 (pmc_hook)((t), (cmd), (arg)); \ 230} while (0) 231 232#define PMC_SWITCH_CONTEXT(t,cmd) PMC_CALL_HOOK_UNLOCKED(t,cmd,NULL) 233 234/* Check if a process is using HWPMCs.*/ 235#define PMC_PROC_IS_USING_PMCS(p) \ 236 (__predict_false(p->p_flag & P_HWPMC)) 237 238#define PMC_THREAD_HAS_SAMPLES(td) \ 239 (__predict_false((td)->td_pmcpend)) 240 241/* Check if a thread have pending user capture. */ 242#define PMC_IS_PENDING_CALLCHAIN(p) \ 243 (__predict_false((p)->td_pflags & TDP_CALLCHAIN)) 244 245#define PMC_SYSTEM_SAMPLING_ACTIVE() (pmc_ss_count > 0) 246 247/* Check if a CPU has recorded samples. */ 248#define PMC_CPU_HAS_SAMPLES(C) (__predict_false(DPCPU_ID_GET((C), pmc_sampled))) 249 250/* 251 * Helper functions. 252 */ 253int pmc_cpu_is_disabled(int _cpu); /* deprecated */ 254int pmc_cpu_is_active(int _cpu); 255int pmc_cpu_is_present(int _cpu); 256int pmc_cpu_is_primary(int _cpu); 257unsigned int pmc_cpu_max(void); 258 259#ifdef INVARIANTS 260int pmc_cpu_max_active(void); 261#endif 262 263/* 264 * Soft events functions. 265 */ 266void pmc_soft_ev_register(struct pmc_soft *ps); 267void pmc_soft_ev_deregister(struct pmc_soft *ps); 268struct pmc_soft *pmc_soft_ev_acquire(enum pmc_event ev); 269void pmc_soft_ev_release(struct pmc_soft *ps); 270 271#endif /* _SYS_PMCKERN_H_ */ 272