1/*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2003-2008 Joseph Koshy 5 * Copyright (c) 2007 The FreeBSD Foundation 6 * All rights reserved. 7 * 8 * Portions of this software were developed by A. Joseph Koshy under 9 * sponsorship from the FreeBSD Foundation and Google, Inc. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33#include <sys/cdefs.h> 34__FBSDID("$FreeBSD$"); 35 36#include "opt_hwpmc_hooks.h" 37 38#include <sys/param.h> 39#include <sys/ctype.h> 40#include <sys/domainset.h> 41#include <sys/param.h> 42#include <sys/malloc.h> 43#include <sys/kernel.h> 44#include <sys/lock.h> 45#include <sys/mutex.h> 46#include <sys/pmc.h> 47#include <sys/pmckern.h> 48#include <sys/smp.h> 49#include <sys/sysctl.h> 50#include <sys/systm.h> 51 52#include <vm/vm.h> 53#include <vm/vm_extern.h> 54#include <vm/vm_kern.h> 55 56#ifdef HWPMC_HOOKS 57FEATURE(hwpmc_hooks, "Kernel support for HW PMC"); 58#define PMC_KERNEL_VERSION PMC_VERSION 59#else 60#define PMC_KERNEL_VERSION 0 61#endif 62 63MALLOC_DECLARE(M_PMCHOOKS); 64MALLOC_DEFINE(M_PMCHOOKS, "pmchooks", "Memory space for PMC hooks"); 65 66/* memory pool */ 67MALLOC_DEFINE(M_PMC, "pmc", "Memory space for the PMC module"); 68 69const int pmc_kernel_version = PMC_KERNEL_VERSION; 70 71/* Hook variable. */ 72int __read_mostly (*pmc_hook)(struct thread *td, int function, void *arg) = NULL; 73 74/* Interrupt handler */ 75int __read_mostly (*pmc_intr)(struct trapframe *tf) = NULL; 76 77DPCPU_DEFINE(uint8_t, pmc_sampled); 78 79/* 80 * A global count of SS mode PMCs. When non-zero, this means that 81 * we have processes that are sampling the system as a whole. 82 */ 83volatile int pmc_ss_count; 84 85/* 86 * Since PMC(4) may not be loaded in the current kernel, the 87 * convention followed is that a non-NULL value of 'pmc_hook' implies 88 * the presence of this kernel module. 89 * 90 * This requires us to protect 'pmc_hook' with a 91 * shared (sx) lock -- thus making the process of calling into PMC(4) 92 * somewhat more expensive than a simple 'if' check and indirect call. 93 */ 94struct sx pmc_sx; 95SX_SYSINIT(pmcsx, &pmc_sx, "pmc-sx"); 96 97/* 98 * PMC Soft per cpu trapframe. 99 */ 100struct trapframe pmc_tf[MAXCPU]; 101 102/* 103 * Per domain list of buffer headers 104 */ 105__read_mostly struct pmc_domain_buffer_header *pmc_dom_hdrs[MAXMEMDOM]; 106 107/* 108 * PMC Soft use a global table to store registered events. 109 */ 110 111SYSCTL_NODE(_kern, OID_AUTO, hwpmc, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 112 "HWPMC parameters"); 113 114static int pmc_softevents = 16; 115SYSCTL_INT(_kern_hwpmc, OID_AUTO, softevents, CTLFLAG_RDTUN, 116 &pmc_softevents, 0, "maximum number of soft events"); 117 118int pmc_softs_count; 119struct pmc_soft **pmc_softs; 120 121struct mtx pmc_softs_mtx; 122MTX_SYSINIT(pmc_soft_mtx, &pmc_softs_mtx, "pmc-softs", MTX_SPIN); 123 124/* 125 * Helper functions. 126 */ 127 128/* 129 * A note on the CPU numbering scheme used by the hwpmc(4) driver. 130 * 131 * CPUs are denoted using numbers in the range 0..[pmc_cpu_max()-1]. 132 * CPUs could be numbered "sparsely" in this range; the predicate 133 * `pmc_cpu_is_present()' is used to test whether a given CPU is 134 * physically present. 135 * 136 * Further, a CPU that is physically present may be administratively 137 * disabled or otherwise unavailable for use by hwpmc(4). The 138 * `pmc_cpu_is_active()' predicate tests for CPU usability. An 139 * "active" CPU participates in thread scheduling and can field 140 * interrupts raised by PMC hardware. 141 * 142 * On systems with hyperthreaded CPUs, multiple logical CPUs may share 143 * PMC hardware resources. For such processors one logical CPU is 144 * denoted as the primary owner of the in-CPU PMC resources. The 145 * pmc_cpu_is_primary() predicate is used to distinguish this primary 146 * CPU from the others. 147 */ 148 149int 150pmc_cpu_is_active(int cpu) 151{ 152#ifdef SMP 153 return (pmc_cpu_is_present(cpu) && 154 !CPU_ISSET(cpu, &hlt_cpus_mask)); 155#else 156 return (1); 157#endif 158} 159 160/* Deprecated. */ 161int 162pmc_cpu_is_disabled(int cpu) 163{ 164 return (!pmc_cpu_is_active(cpu)); 165} 166 167int 168pmc_cpu_is_present(int cpu) 169{ 170#ifdef SMP 171 return (!CPU_ABSENT(cpu)); 172#else 173 return (1); 174#endif 175} 176 177int 178pmc_cpu_is_primary(int cpu) 179{ 180#ifdef SMP 181 return (!CPU_ISSET(cpu, &logical_cpus_mask)); 182#else 183 return (1); 184#endif 185} 186 187/* 188 * Return the maximum CPU number supported by the system. The return 189 * value is used for scaling internal data structures and for runtime 190 * checks. 191 */ 192unsigned int 193pmc_cpu_max(void) 194{ 195#ifdef SMP 196 return (mp_maxid+1); 197#else 198 return (1); 199#endif 200} 201 202#ifdef INVARIANTS 203 204/* 205 * Return the count of CPUs in the `active' state in the system. 206 */ 207int 208pmc_cpu_max_active(void) 209{ 210#ifdef SMP 211 /* 212 * When support for CPU hot-plugging is added to the kernel, 213 * this function would change to return the current number 214 * of "active" CPUs. 215 */ 216 return (mp_ncpus); 217#else 218 return (1); 219#endif 220} 221 222#endif 223 224/* 225 * Cleanup event name: 226 * - remove duplicate '_' 227 * - all uppercase 228 */ 229static void 230pmc_soft_namecleanup(char *name) 231{ 232 char *p, *q; 233 234 p = q = name; 235 236 for ( ; *p == '_' ; p++) 237 ; 238 for ( ; *p ; p++) { 239 if (*p == '_' && (*(p + 1) == '_' || *(p + 1) == '\0')) 240 continue; 241 else 242 *q++ = toupper(*p); 243 } 244 *q = '\0'; 245} 246 247void 248pmc_soft_ev_register(struct pmc_soft *ps) 249{ 250 static int warned = 0; 251 int n; 252 253 ps->ps_running = 0; 254 ps->ps_ev.pm_ev_code = 0; /* invalid */ 255 pmc_soft_namecleanup(ps->ps_ev.pm_ev_name); 256 257 mtx_lock_spin(&pmc_softs_mtx); 258 259 if (pmc_softs_count >= pmc_softevents) { 260 /* 261 * XXX Reusing events can enter a race condition where 262 * new allocated event will be used as an old one. 263 */ 264 for (n = 0; n < pmc_softevents; n++) 265 if (pmc_softs[n] == NULL) 266 break; 267 if (n == pmc_softevents) { 268 mtx_unlock_spin(&pmc_softs_mtx); 269 if (!warned) { 270 printf("hwpmc: too many soft events, " 271 "increase kern.hwpmc.softevents tunable\n"); 272 warned = 1; 273 } 274 return; 275 } 276 277 ps->ps_ev.pm_ev_code = PMC_EV_SOFT_FIRST + n; 278 pmc_softs[n] = ps; 279 } else { 280 ps->ps_ev.pm_ev_code = PMC_EV_SOFT_FIRST + pmc_softs_count; 281 pmc_softs[pmc_softs_count++] = ps; 282 } 283 284 mtx_unlock_spin(&pmc_softs_mtx); 285} 286 287void 288pmc_soft_ev_deregister(struct pmc_soft *ps) 289{ 290 291 KASSERT(ps != NULL, ("pmc_soft_deregister: called with NULL")); 292 293 mtx_lock_spin(&pmc_softs_mtx); 294 295 if (ps->ps_ev.pm_ev_code != 0 && 296 (ps->ps_ev.pm_ev_code - PMC_EV_SOFT_FIRST) < pmc_softevents) { 297 KASSERT((int)ps->ps_ev.pm_ev_code >= PMC_EV_SOFT_FIRST && 298 (int)ps->ps_ev.pm_ev_code <= PMC_EV_SOFT_LAST, 299 ("pmc_soft_deregister: invalid event value")); 300 pmc_softs[ps->ps_ev.pm_ev_code - PMC_EV_SOFT_FIRST] = NULL; 301 } 302 303 mtx_unlock_spin(&pmc_softs_mtx); 304} 305 306struct pmc_soft * 307pmc_soft_ev_acquire(enum pmc_event ev) 308{ 309 struct pmc_soft *ps; 310 311 if (ev == 0 || (ev - PMC_EV_SOFT_FIRST) >= pmc_softevents) 312 return NULL; 313 314 KASSERT((int)ev >= PMC_EV_SOFT_FIRST && 315 (int)ev <= PMC_EV_SOFT_LAST, 316 ("event out of range")); 317 318 mtx_lock_spin(&pmc_softs_mtx); 319 320 ps = pmc_softs[ev - PMC_EV_SOFT_FIRST]; 321 if (ps == NULL) 322 mtx_unlock_spin(&pmc_softs_mtx); 323 324 return ps; 325} 326 327void 328pmc_soft_ev_release(struct pmc_soft *ps) 329{ 330 331 mtx_unlock_spin(&pmc_softs_mtx); 332} 333 334/* 335 * Initialise hwpmc. 336 */ 337static void 338init_hwpmc(void *dummy __unused) 339{ 340 int domain, cpu; 341 342 if (pmc_softevents <= 0 || 343 pmc_softevents > PMC_EV_DYN_COUNT) { 344 (void) printf("hwpmc: tunable \"softevents\"=%d out of " 345 "range.\n", pmc_softevents); 346 pmc_softevents = PMC_EV_DYN_COUNT; 347 } 348 pmc_softs = malloc(pmc_softevents * sizeof(*pmc_softs), M_PMCHOOKS, 349 M_WAITOK | M_ZERO); 350 351 for (domain = 0; domain < vm_ndomains; domain++) { 352 pmc_dom_hdrs[domain] = malloc_domainset( 353 sizeof(struct pmc_domain_buffer_header), M_PMC, 354 DOMAINSET_PREF(domain), M_WAITOK | M_ZERO); 355 mtx_init(&pmc_dom_hdrs[domain]->pdbh_mtx, "pmc_bufferlist_mtx", "pmc-leaf", MTX_SPIN); 356 TAILQ_INIT(&pmc_dom_hdrs[domain]->pdbh_head); 357 } 358 CPU_FOREACH(cpu) { 359 domain = pcpu_find(cpu)->pc_domain; 360 KASSERT(pmc_dom_hdrs[domain] != NULL, ("no mem allocated for domain: %d", domain)); 361 pmc_dom_hdrs[domain]->pdbh_ncpus++; 362 } 363 364} 365 366SYSINIT(hwpmc, SI_SUB_KDTRACE, SI_ORDER_FIRST, init_hwpmc, NULL); 367