hwpmc_ppro.c revision 330897
1/*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2003-2005,2008 Joseph Koshy 5 * Copyright (c) 2007 The FreeBSD Foundation 6 * All rights reserved. 7 * 8 * Portions of this software were developed by A. Joseph Koshy under 9 * sponsorship from the FreeBSD Foundation and Google, Inc. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33#include <sys/cdefs.h> 34__FBSDID("$FreeBSD: stable/11/sys/dev/hwpmc/hwpmc_ppro.c 330897 2018-03-14 03:19:51Z eadler $"); 35 36#include <sys/param.h> 37#include <sys/bus.h> 38#include <sys/lock.h> 39#include <sys/mutex.h> 40#include <sys/pmc.h> 41#include <sys/pmckern.h> 42#include <sys/smp.h> 43#include <sys/systm.h> 44 45#include <machine/intr_machdep.h> 46#include <x86/apicvar.h> 47#include <machine/cpu.h> 48#include <machine/cpufunc.h> 49#include <machine/cputypes.h> 50#include <machine/md_var.h> 51#include <machine/pmc_mdep.h> 52#include <machine/specialreg.h> 53 54/* 55 * PENTIUM PRO SUPPORT 56 * 57 * Quirks: 58 * 59 * - Both PMCs are enabled by a single bit P6_EVSEL_EN in performance 60 * counter '0'. This bit needs to be '1' if any of the two 61 * performance counters are in use. Perf counters can also be 62 * switched off by writing zeros to their EVSEL register. 63 * 64 * - While the width of these counters is 40 bits, we do not appear to 65 * have a way of writing 40 bits to the counter MSRs. A WRMSR 66 * instruction will sign extend bit 31 of the value being written to 67 * the perf counter -- a value of 0x80000000 written to an perf 68 * counter register will be sign extended to 0xFF80000000. 69 * 70 * This quirk primarily affects thread-mode PMCs in counting mode, as 71 * these PMCs read and write PMC registers at every context switch. 72 */ 73 74struct p6pmc_descr { 75 struct pmc_descr pm_descr; /* common information */ 76 uint32_t pm_pmc_msr; 77 uint32_t pm_evsel_msr; 78}; 79 80static struct p6pmc_descr p6_pmcdesc[P6_NPMCS] = { 81 82#define P6_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | PMC_CAP_SYSTEM | \ 83 PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE | \ 84 PMC_CAP_INVERT | PMC_CAP_QUALIFIER) 85 86 /* PMC 0 */ 87 { 88 .pm_descr = 89 { 90 .pd_name ="P6-0", 91 .pd_class = PMC_CLASS_P6, 92 .pd_caps = P6_PMC_CAPS, 93 .pd_width = 40 94 }, 95 .pm_pmc_msr = P6_MSR_PERFCTR0, 96 .pm_evsel_msr = P6_MSR_EVSEL0 97 }, 98 99 /* PMC 1 */ 100 { 101 .pm_descr = 102 { 103 .pd_name ="P6-1", 104 .pd_class = PMC_CLASS_P6, 105 .pd_caps = P6_PMC_CAPS, 106 .pd_width = 40 107 }, 108 .pm_pmc_msr = P6_MSR_PERFCTR1, 109 .pm_evsel_msr = P6_MSR_EVSEL1 110 } 111}; 112 113static enum pmc_cputype p6_cputype; 114 115/* 116 * P6 Event descriptor 117 * 118 * The 'pm_flags' field has the following structure: 119 * - The upper 4 bits are used to track which counter an event is valid on. 120 * - The lower bits form a bitmask of flags indicating support for the event 121 * on a given CPU. 122 */ 123 124struct p6_event_descr { 125 const enum pmc_event pm_event; 126 uint32_t pm_evsel; 127 uint32_t pm_flags; 128 uint32_t pm_unitmask; 129}; 130 131#define P6F_CTR(C) (1 << (28 + (C))) 132#define P6F_CTR0 P6F_CTR(0) 133#define P6F_CTR1 P6F_CTR(1) 134#define P6F(CPU) (1 << ((CPU) - PMC_CPU_INTEL_P6)) 135#define _P6F(C) P6F(PMC_CPU_INTEL_##C) 136#define P6F_P6 _P6F(P6) 137#define P6F_CL _P6F(CL) 138#define P6F_PII _P6F(PII) 139#define P6F_PIII _P6F(PIII) 140#define P6F_PM _P6F(PM) 141#define P6F_ALL_CPUS (P6F_P6 | P6F_PII | P6F_CL | P6F_PIII | P6F_PM) 142#define P6F_ALL_CTRS (P6F_CTR0 | P6F_CTR1) 143#define P6F_ALL (P6F_ALL_CPUS | P6F_ALL_CTRS) 144 145#define P6_EVENT_VALID_FOR_CPU(P,CPU) ((P)->pm_flags & P6F(CPU)) 146#define P6_EVENT_VALID_FOR_CTR(P,CTR) ((P)->pm_flags & P6F_CTR(CTR)) 147 148static const struct p6_event_descr p6_events[] = { 149 150#define P6_EVDESCR(NAME, EVSEL, FLAGS, UMASK) \ 151 { \ 152 .pm_event = PMC_EV_P6_##NAME, \ 153 .pm_evsel = (EVSEL), \ 154 .pm_flags = (FLAGS), \ 155 .pm_unitmask = (UMASK) \ 156 } 157 158P6_EVDESCR(DATA_MEM_REFS, 0x43, P6F_ALL, 0x00), 159P6_EVDESCR(DCU_LINES_IN, 0x45, P6F_ALL, 0x00), 160P6_EVDESCR(DCU_M_LINES_IN, 0x46, P6F_ALL, 0x00), 161P6_EVDESCR(DCU_M_LINES_OUT, 0x47, P6F_ALL, 0x00), 162P6_EVDESCR(DCU_MISS_OUTSTANDING, 0x47, P6F_ALL, 0x00), 163P6_EVDESCR(IFU_FETCH, 0x80, P6F_ALL, 0x00), 164P6_EVDESCR(IFU_FETCH_MISS, 0x81, P6F_ALL, 0x00), 165P6_EVDESCR(ITLB_MISS, 0x85, P6F_ALL, 0x00), 166P6_EVDESCR(IFU_MEM_STALL, 0x86, P6F_ALL, 0x00), 167P6_EVDESCR(ILD_STALL, 0x87, P6F_ALL, 0x00), 168P6_EVDESCR(L2_IFETCH, 0x28, P6F_ALL, 0x0F), 169P6_EVDESCR(L2_LD, 0x29, P6F_ALL, 0x0F), 170P6_EVDESCR(L2_ST, 0x2A, P6F_ALL, 0x0F), 171P6_EVDESCR(L2_LINES_IN, 0x24, P6F_ALL, 0x0F), 172P6_EVDESCR(L2_LINES_OUT, 0x26, P6F_ALL, 0x0F), 173P6_EVDESCR(L2_M_LINES_INM, 0x25, P6F_ALL, 0x00), 174P6_EVDESCR(L2_M_LINES_OUTM, 0x27, P6F_ALL, 0x0F), 175P6_EVDESCR(L2_RQSTS, 0x2E, P6F_ALL, 0x0F), 176P6_EVDESCR(L2_ADS, 0x21, P6F_ALL, 0x00), 177P6_EVDESCR(L2_DBUS_BUSY, 0x22, P6F_ALL, 0x00), 178P6_EVDESCR(L2_DBUS_BUSY_RD, 0x23, P6F_ALL, 0x00), 179P6_EVDESCR(BUS_DRDY_CLOCKS, 0x62, P6F_ALL, 0x20), 180P6_EVDESCR(BUS_LOCK_CLOCKS, 0x63, P6F_ALL, 0x20), 181P6_EVDESCR(BUS_REQ_OUTSTANDING, 0x60, P6F_ALL, 0x00), 182P6_EVDESCR(BUS_TRAN_BRD, 0x65, P6F_ALL, 0x20), 183P6_EVDESCR(BUS_TRAN_RFO, 0x66, P6F_ALL, 0x20), 184P6_EVDESCR(BUS_TRANS_WB, 0x67, P6F_ALL, 0x20), 185P6_EVDESCR(BUS_TRAN_IFETCH, 0x68, P6F_ALL, 0x20), 186P6_EVDESCR(BUS_TRAN_INVAL, 0x69, P6F_ALL, 0x20), 187P6_EVDESCR(BUS_TRAN_PWR, 0x6A, P6F_ALL, 0x20), 188P6_EVDESCR(BUS_TRANS_P, 0x6B, P6F_ALL, 0x20), 189P6_EVDESCR(BUS_TRANS_IO, 0x6C, P6F_ALL, 0x20), 190P6_EVDESCR(BUS_TRAN_DEF, 0x6D, P6F_ALL, 0x20), 191P6_EVDESCR(BUS_TRAN_BURST, 0x6E, P6F_ALL, 0x20), 192P6_EVDESCR(BUS_TRAN_ANY, 0x70, P6F_ALL, 0x20), 193P6_EVDESCR(BUS_TRAN_MEM, 0x6F, P6F_ALL, 0x20), 194P6_EVDESCR(BUS_DATA_RCV, 0x64, P6F_ALL, 0x00), 195P6_EVDESCR(BUS_BNR_DRV, 0x61, P6F_ALL, 0x00), 196P6_EVDESCR(BUS_HIT_DRV, 0x7A, P6F_ALL, 0x00), 197P6_EVDESCR(BUS_HITM_DRV, 0x7B, P6F_ALL, 0x00), 198P6_EVDESCR(BUS_SNOOP_STALL, 0x7E, P6F_ALL, 0x00), 199P6_EVDESCR(FLOPS, 0xC1, P6F_ALL_CPUS | P6F_CTR0, 0x00), 200P6_EVDESCR(FP_COMPS_OPS_EXE, 0x10, P6F_ALL_CPUS | P6F_CTR0, 0x00), 201P6_EVDESCR(FP_ASSIST, 0x11, P6F_ALL_CPUS | P6F_CTR1, 0x00), 202P6_EVDESCR(MUL, 0x12, P6F_ALL_CPUS | P6F_CTR1, 0x00), 203P6_EVDESCR(DIV, 0x13, P6F_ALL_CPUS | P6F_CTR1, 0x00), 204P6_EVDESCR(CYCLES_DIV_BUSY, 0x14, P6F_ALL_CPUS | P6F_CTR0, 0x00), 205P6_EVDESCR(LD_BLOCKS, 0x03, P6F_ALL, 0x00), 206P6_EVDESCR(SB_DRAINS, 0x04, P6F_ALL, 0x00), 207P6_EVDESCR(MISALIGN_MEM_REF, 0x05, P6F_ALL, 0x00), 208P6_EVDESCR(EMON_KNI_PREF_DISPATCHED, 0x07, P6F_PIII | P6F_ALL_CTRS, 0x03), 209P6_EVDESCR(EMON_KNI_PREF_MISS, 0x4B, P6F_PIII | P6F_ALL_CTRS, 0x03), 210P6_EVDESCR(INST_RETIRED, 0xC0, P6F_ALL, 0x00), 211P6_EVDESCR(UOPS_RETIRED, 0xC2, P6F_ALL, 0x00), 212P6_EVDESCR(INST_DECODED, 0xD0, P6F_ALL, 0x00), 213P6_EVDESCR(EMON_KNI_INST_RETIRED, 0xD8, P6F_PIII | P6F_ALL_CTRS, 0x01), 214P6_EVDESCR(EMON_KNI_COMP_INST_RET, 0xD9, P6F_PIII | P6F_ALL_CTRS, 0x01), 215P6_EVDESCR(HW_INT_RX, 0xC8, P6F_ALL, 0x00), 216P6_EVDESCR(CYCLES_INT_MASKED, 0xC6, P6F_ALL, 0x00), 217P6_EVDESCR(CYCLES_INT_PENDING_AND_MASKED, 0xC7, P6F_ALL, 0x00), 218P6_EVDESCR(BR_INST_RETIRED, 0xC4, P6F_ALL, 0x00), 219P6_EVDESCR(BR_MISS_PRED_RETIRED, 0xC5, P6F_ALL, 0x00), 220P6_EVDESCR(BR_TAKEN_RETIRED, 0xC9, P6F_ALL, 0x00), 221P6_EVDESCR(BR_MISS_PRED_TAKEN_RET, 0xCA, P6F_ALL, 0x00), 222P6_EVDESCR(BR_INST_DECODED, 0xE0, P6F_ALL, 0x00), 223P6_EVDESCR(BTB_MISSES, 0xE2, P6F_ALL, 0x00), 224P6_EVDESCR(BR_BOGUS, 0xE4, P6F_ALL, 0x00), 225P6_EVDESCR(BACLEARS, 0xE6, P6F_ALL, 0x00), 226P6_EVDESCR(RESOURCE_STALLS, 0xA2, P6F_ALL, 0x00), 227P6_EVDESCR(PARTIAL_RAT_STALLS, 0xD2, P6F_ALL, 0x00), 228P6_EVDESCR(SEGMENT_REG_LOADS, 0x06, P6F_ALL, 0x00), 229P6_EVDESCR(CPU_CLK_UNHALTED, 0x79, P6F_ALL, 0x00), 230P6_EVDESCR(MMX_INSTR_EXEC, 0xB0, 231 P6F_ALL_CTRS | P6F_CL | P6F_PII, 0x00), 232P6_EVDESCR(MMX_SAT_INSTR_EXEC, 0xB1, 233 P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x00), 234P6_EVDESCR(MMX_UOPS_EXEC, 0xB2, 235 P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x0F), 236P6_EVDESCR(MMX_INSTR_TYPE_EXEC, 0xB3, 237 P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x3F), 238P6_EVDESCR(FP_MMX_TRANS, 0xCC, 239 P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x01), 240P6_EVDESCR(MMX_ASSIST, 0xCD, 241 P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x00), 242P6_EVDESCR(MMX_INSTR_RET, 0xCE, P6F_ALL_CTRS | P6F_PII, 0x00), 243P6_EVDESCR(SEG_RENAME_STALLS, 0xD4, 244 P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x0F), 245P6_EVDESCR(SEG_REG_RENAMES, 0xD5, 246 P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x0F), 247P6_EVDESCR(RET_SEG_RENAMES, 0xD6, 248 P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x00), 249P6_EVDESCR(EMON_EST_TRANS, 0x58, P6F_ALL_CTRS | P6F_PM, 0x02), 250P6_EVDESCR(EMON_THERMAL_TRIP, 0x59, P6F_ALL_CTRS | P6F_PM, 0x00), 251P6_EVDESCR(BR_INST_EXEC, 0x88, P6F_ALL_CTRS | P6F_PM, 0x00), 252P6_EVDESCR(BR_MISSP_EXEC, 0x89, P6F_ALL_CTRS | P6F_PM, 0x00), 253P6_EVDESCR(BR_BAC_MISSP_EXEC, 0x8A, P6F_ALL_CTRS | P6F_PM, 0x00), 254P6_EVDESCR(BR_CND_EXEC, 0x8B, P6F_ALL_CTRS | P6F_PM, 0x00), 255P6_EVDESCR(BR_CND_MISSP_EXEC, 0x8C, P6F_ALL_CTRS | P6F_PM, 0x00), 256P6_EVDESCR(BR_IND_EXEC, 0x8D, P6F_ALL_CTRS | P6F_PM, 0x00), 257P6_EVDESCR(BR_IND_MISSP_EXEC, 0x8E, P6F_ALL_CTRS | P6F_PM, 0x00), 258P6_EVDESCR(BR_RET_EXEC, 0x8F, P6F_ALL_CTRS | P6F_PM, 0x00), 259P6_EVDESCR(BR_RET_MISSP_EXEC, 0x90, P6F_ALL_CTRS | P6F_PM, 0x00), 260P6_EVDESCR(BR_RET_BAC_MISSP_EXEC, 0x91, P6F_ALL_CTRS | P6F_PM, 0x00), 261P6_EVDESCR(BR_CALL_EXEC, 0x92, P6F_ALL_CTRS | P6F_PM, 0x00), 262P6_EVDESCR(BR_CALL_MISSP_EXEC, 0x93, P6F_ALL_CTRS | P6F_PM, 0x00), 263P6_EVDESCR(BR_IND_CALL_EXEC, 0x94, P6F_ALL_CTRS | P6F_PM, 0x00), 264P6_EVDESCR(EMON_SIMD_INSTR_RETIRED, 0xCE, P6F_ALL_CTRS | P6F_PM, 0x00), 265P6_EVDESCR(EMON_SYNCH_UOPS, 0xD3, P6F_ALL_CTRS | P6F_PM, 0x00), 266P6_EVDESCR(EMON_ESP_UOPS, 0xD7, P6F_ALL_CTRS | P6F_PM, 0x00), 267P6_EVDESCR(EMON_FUSED_UOPS_RET, 0xDA, P6F_ALL_CTRS | P6F_PM, 0x03), 268P6_EVDESCR(EMON_UNFUSION, 0xDB, P6F_ALL_CTRS | P6F_PM, 0x00), 269P6_EVDESCR(EMON_PREF_RQSTS_UP, 0xF0, P6F_ALL_CTRS | P6F_PM, 0x00), 270P6_EVDESCR(EMON_PREF_RQSTS_DN, 0xD8, P6F_ALL_CTRS | P6F_PM, 0x00), 271P6_EVDESCR(EMON_SSE_SSE2_INST_RETIRED, 0xD8, P6F_ALL_CTRS | P6F_PM, 0x03), 272P6_EVDESCR(EMON_SSE_SSE2_COMP_INST_RETIRED, 0xD9, P6F_ALL_CTRS | P6F_PM, 0x03) 273 274#undef P6_EVDESCR 275}; 276 277#define P6_NEVENTS (PMC_EV_P6_LAST - PMC_EV_P6_FIRST + 1) 278 279static const struct p6_event_descr * 280p6_find_event(enum pmc_event ev) 281{ 282 int n; 283 284 for (n = 0; n < P6_NEVENTS; n++) 285 if (p6_events[n].pm_event == ev) 286 break; 287 if (n == P6_NEVENTS) 288 return NULL; 289 return &p6_events[n]; 290} 291 292/* 293 * Per-CPU data structure for P6 class CPUs 294 * 295 * [common stuff] 296 * [flags for maintaining PMC start/stop state] 297 * [3 struct pmc_hw pointers] 298 * [3 struct pmc_hw structures] 299 */ 300 301struct p6_cpu { 302 struct pmc_hw pc_p6pmcs[P6_NPMCS]; 303 uint32_t pc_state; 304}; 305 306static struct p6_cpu **p6_pcpu; 307 308/* 309 * If CTR1 is active, we need to keep the 'EN' bit if CTR0 set, 310 * with the rest of CTR0 being zero'ed out. 311 */ 312#define P6_SYNC_CTR_STATE(PC) do { \ 313 uint32_t _config, _enable; \ 314 _enable = 0; \ 315 if ((PC)->pc_state & 0x02) \ 316 _enable |= P6_EVSEL_EN; \ 317 if ((PC)->pc_state & 0x01) \ 318 _config = rdmsr(P6_MSR_EVSEL0) | \ 319 P6_EVSEL_EN; \ 320 else \ 321 _config = 0; \ 322 wrmsr(P6_MSR_EVSEL0, _config | _enable); \ 323 } while (0) 324 325#define P6_MARK_STARTED(PC,RI) do { \ 326 (PC)->pc_state |= (1 << ((RI)-1)); \ 327 } while (0) 328 329#define P6_MARK_STOPPED(PC,RI) do { \ 330 (PC)->pc_state &= ~(1<< ((RI)-1)); \ 331 } while (0) 332 333static int 334p6_pcpu_init(struct pmc_mdep *md, int cpu) 335{ 336 int first_ri, n; 337 struct p6_cpu *p6c; 338 struct pmc_cpu *pc; 339 struct pmc_hw *phw; 340 341 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 342 ("[p6,%d] bad cpu %d", __LINE__, cpu)); 343 344 PMCDBG1(MDP,INI,0,"p6-init cpu=%d", cpu); 345 346 p6c = malloc(sizeof (struct p6_cpu), M_PMC, M_WAITOK|M_ZERO); 347 pc = pmc_pcpu[cpu]; 348 349 KASSERT(pc != NULL, ("[p6,%d] cpu %d null per-cpu", __LINE__, cpu)); 350 351 phw = p6c->pc_p6pmcs; 352 p6_pcpu[cpu] = p6c; 353 354 first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P6].pcd_ri; 355 356 for (n = 0; n < P6_NPMCS; n++, phw++) { 357 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 358 PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); 359 phw->phw_pmc = NULL; 360 pc->pc_hwpmcs[n + first_ri] = phw; 361 } 362 363 return (0); 364} 365 366static int 367p6_pcpu_fini(struct pmc_mdep *md, int cpu) 368{ 369 int first_ri, n; 370 struct p6_cpu *p6c; 371 struct pmc_cpu *pc; 372 373 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 374 ("[p6,%d] bad cpu %d", __LINE__, cpu)); 375 376 PMCDBG1(MDP,INI,0,"p6-cleanup cpu=%d", cpu); 377 378 p6c = p6_pcpu[cpu]; 379 p6_pcpu[cpu] = NULL; 380 381 KASSERT(p6c != NULL, ("[p6,%d] null pcpu", __LINE__)); 382 383 free(p6c, M_PMC); 384 385 first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P6].pcd_ri; 386 pc = pmc_pcpu[cpu]; 387 for (n = 0; n < P6_NPMCS; n++) 388 pc->pc_hwpmcs[n + first_ri] = NULL; 389 390 return (0); 391} 392 393static int 394p6_read_pmc(int cpu, int ri, pmc_value_t *v) 395{ 396 struct pmc *pm; 397 struct p6pmc_descr *pd; 398 pmc_value_t tmp; 399 400 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 401 ("[p6,%d] illegal cpu value %d", __LINE__, cpu)); 402 KASSERT(ri >= 0 && ri < P6_NPMCS, 403 ("[p6,%d] illegal row-index %d", __LINE__, ri)); 404 405 pm = p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc; 406 pd = &p6_pmcdesc[ri]; 407 408 KASSERT(pm, 409 ("[p6,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); 410 411 tmp = rdmsr(pd->pm_pmc_msr) & P6_PERFCTR_READ_MASK; 412 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 413 *v = P6_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp); 414 else 415 *v = tmp; 416 417 PMCDBG4(MDP,REA,1, "p6-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri, 418 pd->pm_pmc_msr, *v); 419 420 return (0); 421} 422 423static int 424p6_write_pmc(int cpu, int ri, pmc_value_t v) 425{ 426 struct pmc *pm; 427 struct p6pmc_descr *pd; 428 429 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 430 ("[p6,%d] illegal cpu value %d", __LINE__, cpu)); 431 KASSERT(ri >= 0 && ri < P6_NPMCS, 432 ("[p6,%d] illegal row-index %d", __LINE__, ri)); 433 434 pm = p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc; 435 pd = &p6_pmcdesc[ri]; 436 437 KASSERT(pm, 438 ("[p6,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); 439 440 PMCDBG4(MDP,WRI,1, "p6-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri, 441 pd->pm_pmc_msr, v); 442 443 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 444 v = P6_RELOAD_COUNT_TO_PERFCTR_VALUE(v); 445 446 wrmsr(pd->pm_pmc_msr, v & P6_PERFCTR_WRITE_MASK); 447 448 return (0); 449} 450 451static int 452p6_config_pmc(int cpu, int ri, struct pmc *pm) 453{ 454 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 455 ("[p6,%d] illegal CPU %d", __LINE__, cpu)); 456 457 KASSERT(ri >= 0 && ri < P6_NPMCS, 458 ("[p6,%d] illegal row-index %d", __LINE__, ri)); 459 460 PMCDBG3(MDP,CFG,1, "p6-config cpu=%d ri=%d pm=%p", cpu, ri, pm); 461 462 KASSERT(p6_pcpu[cpu] != NULL, ("[p6,%d] null per-cpu %d", __LINE__, 463 cpu)); 464 465 p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc = pm; 466 467 return (0); 468} 469 470/* 471 * Retrieve a configured PMC pointer from hardware state. 472 */ 473 474static int 475p6_get_config(int cpu, int ri, struct pmc **ppm) 476{ 477 478 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 479 ("[p6,%d] illegal CPU %d", __LINE__, cpu)); 480 KASSERT(ri >= 0 && ri < P6_NPMCS, 481 ("[p6,%d] illegal row-index %d", __LINE__, ri)); 482 483 *ppm = p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc; 484 485 return (0); 486} 487 488 489/* 490 * A pmc may be allocated to a given row index if: 491 * - the event is valid for this CPU 492 * - the event is valid for this counter index 493 */ 494 495static int 496p6_allocate_pmc(int cpu, int ri, struct pmc *pm, 497 const struct pmc_op_pmcallocate *a) 498{ 499 uint32_t allowed_unitmask, caps, config, unitmask; 500 const struct p6pmc_descr *pd; 501 const struct p6_event_descr *pevent; 502 enum pmc_event ev; 503 504 (void) cpu; 505 506 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 507 ("[p6,%d] illegal CPU %d", __LINE__, cpu)); 508 KASSERT(ri >= 0 && ri < P6_NPMCS, 509 ("[p6,%d] illegal row-index value %d", __LINE__, ri)); 510 511 pd = &p6_pmcdesc[ri]; 512 513 PMCDBG4(MDP,ALL,1, "p6-allocate ri=%d class=%d pmccaps=0x%x " 514 "reqcaps=0x%x", ri, pd->pm_descr.pd_class, pd->pm_descr.pd_caps, 515 pm->pm_caps); 516 517 /* check class */ 518 if (pd->pm_descr.pd_class != a->pm_class) 519 return (EINVAL); 520 521 /* check requested capabilities */ 522 caps = a->pm_caps; 523 if ((pd->pm_descr.pd_caps & caps) != caps) 524 return (EPERM); 525 526 ev = pm->pm_event; 527 528 if (ev < PMC_EV_P6_FIRST || ev > PMC_EV_P6_LAST) 529 return (EINVAL); 530 531 if ((pevent = p6_find_event(ev)) == NULL) 532 return (ESRCH); 533 534 if (!P6_EVENT_VALID_FOR_CPU(pevent, p6_cputype) || 535 !P6_EVENT_VALID_FOR_CTR(pevent, (ri-1))) 536 return (EINVAL); 537 538 /* For certain events, Pentium M differs from the stock P6 */ 539 allowed_unitmask = 0; 540 if (p6_cputype == PMC_CPU_INTEL_PM) { 541 if (ev == PMC_EV_P6_L2_LD || ev == PMC_EV_P6_L2_LINES_IN || 542 ev == PMC_EV_P6_L2_LINES_OUT) 543 allowed_unitmask = P6_EVSEL_TO_UMASK(0x3F); 544 else if (ev == PMC_EV_P6_L2_M_LINES_OUTM) 545 allowed_unitmask = P6_EVSEL_TO_UMASK(0x30); 546 } else 547 allowed_unitmask = P6_EVSEL_TO_UMASK(pevent->pm_unitmask); 548 549 unitmask = a->pm_md.pm_ppro.pm_ppro_config & P6_EVSEL_UMASK_MASK; 550 if (unitmask & ~allowed_unitmask) /* disallow reserved bits */ 551 return (EINVAL); 552 553 if (ev == PMC_EV_P6_MMX_UOPS_EXEC) /* hardcoded mask */ 554 unitmask = P6_EVSEL_TO_UMASK(0x0F); 555 556 config = 0; 557 558 config |= P6_EVSEL_EVENT_SELECT(pevent->pm_evsel); 559 560 if (unitmask & (caps & PMC_CAP_QUALIFIER)) 561 config |= unitmask; 562 563 if (caps & PMC_CAP_THRESHOLD) 564 config |= a->pm_md.pm_ppro.pm_ppro_config & 565 P6_EVSEL_CMASK_MASK; 566 567 /* set at least one of the 'usr' or 'os' caps */ 568 if (caps & PMC_CAP_USER) 569 config |= P6_EVSEL_USR; 570 if (caps & PMC_CAP_SYSTEM) 571 config |= P6_EVSEL_OS; 572 if ((caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0) 573 config |= (P6_EVSEL_USR|P6_EVSEL_OS); 574 575 if (caps & PMC_CAP_EDGE) 576 config |= P6_EVSEL_E; 577 if (caps & PMC_CAP_INVERT) 578 config |= P6_EVSEL_INV; 579 if (caps & PMC_CAP_INTERRUPT) 580 config |= P6_EVSEL_INT; 581 582 pm->pm_md.pm_ppro.pm_ppro_evsel = config; 583 584 PMCDBG1(MDP,ALL,2, "p6-allocate config=0x%x", config); 585 586 return (0); 587} 588 589static int 590p6_release_pmc(int cpu, int ri, struct pmc *pm) 591{ 592 (void) pm; 593 594 PMCDBG3(MDP,REL,1, "p6-release cpu=%d ri=%d pm=%p", cpu, ri, pm); 595 596 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 597 ("[p6,%d] illegal CPU value %d", __LINE__, cpu)); 598 KASSERT(ri >= 0 && ri < P6_NPMCS, 599 ("[p6,%d] illegal row-index %d", __LINE__, ri)); 600 601 KASSERT(p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc == NULL, 602 ("[p6,%d] PHW pmc non-NULL", __LINE__)); 603 604 return (0); 605} 606 607static int 608p6_start_pmc(int cpu, int ri) 609{ 610 uint32_t config; 611 struct pmc *pm; 612 struct p6_cpu *pc; 613 const struct p6pmc_descr *pd; 614 615 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 616 ("[p6,%d] illegal CPU value %d", __LINE__, cpu)); 617 KASSERT(ri >= 0 && ri < P6_NPMCS, 618 ("[p6,%d] illegal row-index %d", __LINE__, ri)); 619 620 pc = p6_pcpu[cpu]; 621 pm = pc->pc_p6pmcs[ri].phw_pmc; 622 pd = &p6_pmcdesc[ri]; 623 624 KASSERT(pm, 625 ("[p6,%d] starting cpu%d,ri%d with no pmc configured", 626 __LINE__, cpu, ri)); 627 628 PMCDBG2(MDP,STA,1, "p6-start cpu=%d ri=%d", cpu, ri); 629 630 config = pm->pm_md.pm_ppro.pm_ppro_evsel; 631 632 PMCDBG4(MDP,STA,2, "p6-start/2 cpu=%d ri=%d evselmsr=0x%x config=0x%x", 633 cpu, ri, pd->pm_evsel_msr, config); 634 635 P6_MARK_STARTED(pc, ri); 636 wrmsr(pd->pm_evsel_msr, config); 637 638 P6_SYNC_CTR_STATE(pc); 639 640 return (0); 641} 642 643static int 644p6_stop_pmc(int cpu, int ri) 645{ 646 struct pmc *pm; 647 struct p6_cpu *pc; 648 struct p6pmc_descr *pd; 649 650 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 651 ("[p6,%d] illegal cpu value %d", __LINE__, cpu)); 652 KASSERT(ri >= 0 && ri < P6_NPMCS, 653 ("[p6,%d] illegal row index %d", __LINE__, ri)); 654 655 pc = p6_pcpu[cpu]; 656 pm = pc->pc_p6pmcs[ri].phw_pmc; 657 pd = &p6_pmcdesc[ri]; 658 659 KASSERT(pm, 660 ("[p6,%d] cpu%d ri%d no configured PMC to stop", __LINE__, 661 cpu, ri)); 662 663 PMCDBG2(MDP,STO,1, "p6-stop cpu=%d ri=%d", cpu, ri); 664 665 wrmsr(pd->pm_evsel_msr, 0); /* stop hw */ 666 P6_MARK_STOPPED(pc, ri); /* update software state */ 667 668 P6_SYNC_CTR_STATE(pc); /* restart CTR1 if need be */ 669 670 PMCDBG2(MDP,STO,2, "p6-stop/2 cpu=%d ri=%d", cpu, ri); 671 672 return (0); 673} 674 675static int 676p6_intr(int cpu, struct trapframe *tf) 677{ 678 int error, retval, ri; 679 uint32_t perf0cfg; 680 struct pmc *pm; 681 struct p6_cpu *pc; 682 pmc_value_t v; 683 684 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 685 ("[p6,%d] CPU %d out of range", __LINE__, cpu)); 686 687 retval = 0; 688 pc = p6_pcpu[cpu]; 689 690 /* stop both PMCs */ 691 perf0cfg = rdmsr(P6_MSR_EVSEL0); 692 wrmsr(P6_MSR_EVSEL0, perf0cfg & ~P6_EVSEL_EN); 693 694 for (ri = 0; ri < P6_NPMCS; ri++) { 695 696 if ((pm = pc->pc_p6pmcs[ri].phw_pmc) == NULL || 697 !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { 698 continue; 699 } 700 701 if (!P6_PMC_HAS_OVERFLOWED(ri)) 702 continue; 703 704 retval = 1; 705 706 if (pm->pm_state != PMC_STATE_RUNNING) 707 continue; 708 709 error = pmc_process_interrupt(cpu, PMC_HR, pm, tf, 710 TRAPF_USERMODE(tf)); 711 if (error) 712 P6_MARK_STOPPED(pc,ri); 713 714 /* reload sampling count */ 715 v = pm->pm_sc.pm_reloadcount; 716 wrmsr(P6_MSR_PERFCTR0 + ri, 717 P6_RELOAD_COUNT_TO_PERFCTR_VALUE(v)); 718 719 } 720 721 /* 722 * On P6 processors, the LAPIC needs to have its PMC interrupt 723 * unmasked after a PMC interrupt. 724 */ 725 if (retval) 726 lapic_reenable_pmc(); 727 728 atomic_add_int(retval ? &pmc_stats.pm_intr_processed : 729 &pmc_stats.pm_intr_ignored, 1); 730 731 /* restart counters that can be restarted */ 732 P6_SYNC_CTR_STATE(pc); 733 734 return (retval); 735} 736 737static int 738p6_describe(int cpu, int ri, struct pmc_info *pi, 739 struct pmc **ppmc) 740{ 741 int error; 742 size_t copied; 743 struct pmc_hw *phw; 744 struct p6pmc_descr *pd; 745 746 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 747 ("[p6,%d] illegal CPU %d", __LINE__, cpu)); 748 KASSERT(ri >= 0 && ri < P6_NPMCS, 749 ("[p6,%d] row-index %d out of range", __LINE__, ri)); 750 751 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 752 pd = &p6_pmcdesc[ri]; 753 754 KASSERT(phw == &p6_pcpu[cpu]->pc_p6pmcs[ri], 755 ("[p6,%d] phw mismatch", __LINE__)); 756 757 if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, 758 PMC_NAME_MAX, &copied)) != 0) 759 return (error); 760 761 pi->pm_class = pd->pm_descr.pd_class; 762 763 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 764 pi->pm_enabled = TRUE; 765 *ppmc = phw->phw_pmc; 766 } else { 767 pi->pm_enabled = FALSE; 768 *ppmc = NULL; 769 } 770 771 return (0); 772} 773 774static int 775p6_get_msr(int ri, uint32_t *msr) 776{ 777 KASSERT(ri >= 0 && ri < P6_NPMCS, 778 ("[p6,%d ri %d out of range", __LINE__, ri)); 779 780 *msr = p6_pmcdesc[ri].pm_pmc_msr - P6_MSR_PERFCTR0; 781 782 return (0); 783} 784 785int 786pmc_p6_initialize(struct pmc_mdep *md, int ncpus) 787{ 788 struct pmc_classdep *pcd; 789 790 KASSERT(cpu_vendor_id == CPU_VENDOR_INTEL, 791 ("[p6,%d] Initializing non-intel processor", __LINE__)); 792 793 PMCDBG0(MDP,INI,1, "p6-initialize"); 794 795 /* Allocate space for pointers to per-cpu descriptors. */ 796 p6_pcpu = malloc(sizeof(struct p6_cpu **) * ncpus, M_PMC, 797 M_ZERO|M_WAITOK); 798 799 /* Fill in the class dependent descriptor. */ 800 pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P6]; 801 802 switch (md->pmd_cputype) { 803 804 /* 805 * P6 Family Processors 806 */ 807 case PMC_CPU_INTEL_P6: 808 case PMC_CPU_INTEL_CL: 809 case PMC_CPU_INTEL_PII: 810 case PMC_CPU_INTEL_PIII: 811 case PMC_CPU_INTEL_PM: 812 813 p6_cputype = md->pmd_cputype; 814 815 pcd->pcd_caps = P6_PMC_CAPS; 816 pcd->pcd_class = PMC_CLASS_P6; 817 pcd->pcd_num = P6_NPMCS; 818 pcd->pcd_ri = md->pmd_npmc; 819 pcd->pcd_width = 40; 820 821 pcd->pcd_allocate_pmc = p6_allocate_pmc; 822 pcd->pcd_config_pmc = p6_config_pmc; 823 pcd->pcd_describe = p6_describe; 824 pcd->pcd_get_config = p6_get_config; 825 pcd->pcd_get_msr = p6_get_msr; 826 pcd->pcd_pcpu_fini = p6_pcpu_fini; 827 pcd->pcd_pcpu_init = p6_pcpu_init; 828 pcd->pcd_read_pmc = p6_read_pmc; 829 pcd->pcd_release_pmc = p6_release_pmc; 830 pcd->pcd_start_pmc = p6_start_pmc; 831 pcd->pcd_stop_pmc = p6_stop_pmc; 832 pcd->pcd_write_pmc = p6_write_pmc; 833 834 md->pmd_pcpu_fini = NULL; 835 md->pmd_pcpu_init = NULL; 836 md->pmd_intr = p6_intr; 837 838 md->pmd_npmc += P6_NPMCS; 839 840 break; 841 842 default: 843 KASSERT(0,("[p6,%d] Unknown CPU type", __LINE__)); 844 return ENOSYS; 845 } 846 847 return (0); 848} 849 850void 851pmc_p6_finalize(struct pmc_mdep *md) 852{ 853#if defined(INVARIANTS) 854 int i, ncpus; 855#endif 856 857 KASSERT(p6_pcpu != NULL, ("[p6,%d] NULL p6_pcpu", __LINE__)); 858 859#if defined(INVARIANTS) 860 ncpus = pmc_cpu_max(); 861 for (i = 0; i < ncpus; i++) 862 KASSERT(p6_pcpu[i] == NULL, ("[p6,%d] non-null pcpu %d", 863 __LINE__, i)); 864#endif 865 866 free(p6_pcpu, M_PMC); 867 p6_pcpu = NULL; 868} 869