1/*- 2 * Copyright (c) 2014 Andrew Turner 3 * Copyright (c) 2014 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * Portions of this software were developed by Semihalf 7 * under sponsorship of the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 */ 31 32#include <sys/param.h> 33#include <sys/kernel.h> 34#include <sys/malloc.h> 35#include <sys/proc.h> 36#include <sys/pcpu.h> 37#include <sys/sbuf.h> 38#include <sys/smp.h> 39#include <sys/sysctl.h> 40#include <sys/sysent.h> 41#include <sys/systm.h> 42 43#include <machine/atomic.h> 44#include <machine/cpu.h> 45#include <machine/cpufunc.h> 46#include <machine/elf.h> 47#include <machine/md_var.h> 48#include <machine/undefined.h> 49 50static MALLOC_DEFINE(M_IDENTCPU, "CPU ID", "arm64 CPU identification memory"); 51 52struct cpu_desc; 53 54static void print_cpu_midr(struct sbuf *sb, u_int cpu); 55static void print_cpu_features(u_int cpu, struct cpu_desc *desc, 56 struct cpu_desc *prev_desc); 57static void print_cpu_caches(struct sbuf *sb, struct cpu_desc *desc); 58#ifdef COMPAT_FREEBSD32 59static u_long parse_cpu_features_hwcap32(void); 60#endif 61 62char machine[] = "arm64"; 63 64#ifdef SCTL_MASK32 65extern int adaptive_machine_arch; 66#endif 67 68static SYSCTL_NODE(_machdep, OID_AUTO, cache, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 69 "Cache management tuning"); 70 71static int allow_dic = 1; 72SYSCTL_INT(_machdep_cache, OID_AUTO, allow_dic, CTLFLAG_RDTUN, &allow_dic, 0, 73 "Allow optimizations based on the DIC cache bit"); 74 75static int allow_idc = 1; 76SYSCTL_INT(_machdep_cache, OID_AUTO, allow_idc, CTLFLAG_RDTUN, &allow_idc, 0, 77 "Allow optimizations based on the IDC cache bit"); 78 79static void check_cpu_regs(u_int cpu, struct cpu_desc *desc, 80 struct cpu_desc *prev_desc); 81 82/* 83 * The default implementation of I-cache sync assumes we have an 84 * aliasing cache until we know otherwise. 85 */ 86void (*arm64_icache_sync_range)(void *, vm_size_t) = 87 &arm64_aliasing_icache_sync_range; 88 89static int 90sysctl_hw_machine(SYSCTL_HANDLER_ARGS) 91{ 92#ifdef SCTL_MASK32 93 static const char machine32[] = "arm"; 94#endif 95 int error; 96 97#ifdef SCTL_MASK32 98 if ((req->flags & SCTL_MASK32) != 0 && adaptive_machine_arch) 99 error = SYSCTL_OUT(req, machine32, sizeof(machine32)); 100 else 101#endif 102 error = SYSCTL_OUT(req, machine, sizeof(machine)); 103 return (error); 104} 105 106SYSCTL_PROC(_hw, HW_MACHINE, machine, CTLTYPE_STRING | CTLFLAG_RD | 107 CTLFLAG_CAPRD | CTLFLAG_MPSAFE, NULL, 0, sysctl_hw_machine, "A", 108 "Machine class"); 109 110static char cpu_model[64]; 111SYSCTL_STRING(_hw, HW_MODEL, model, CTLFLAG_RD | CTLFLAG_CAPRD, 112 cpu_model, sizeof(cpu_model), "Machine model"); 113 114#define MAX_CACHES 8 /* Maximum number of caches supported 115 architecturally. */ 116/* 117 * Per-CPU affinity as provided in MPIDR_EL1 118 * Indexed by CPU number in logical order selected by the system. 119 * Relevant fields can be extracted using CPU_AFFn macros, 120 * Aff3.Aff2.Aff1.Aff0 construct a unique CPU address in the system. 121 * 122 * Fields used by us: 123 * Aff1 - Cluster number 124 * Aff0 - CPU number in Aff1 cluster 125 */ 126uint64_t __cpu_affinity[MAXCPU]; 127static u_int cpu_aff_levels; 128 129struct cpu_desc { 130 uint64_t mpidr; 131 uint64_t id_aa64afr0; 132 uint64_t id_aa64afr1; 133 uint64_t id_aa64dfr0; 134 uint64_t id_aa64dfr1; 135 uint64_t id_aa64isar0; 136 uint64_t id_aa64isar1; 137 uint64_t id_aa64isar2; 138 uint64_t id_aa64mmfr0; 139 uint64_t id_aa64mmfr1; 140 uint64_t id_aa64mmfr2; 141 uint64_t id_aa64mmfr3; 142 uint64_t id_aa64mmfr4; 143 uint64_t id_aa64pfr0; 144 uint64_t id_aa64pfr1; 145 uint64_t id_aa64pfr2; 146 uint64_t id_aa64zfr0; 147 uint64_t ctr; 148#ifdef COMPAT_FREEBSD32 149 uint64_t id_isar5; 150 uint64_t mvfr0; 151 uint64_t mvfr1; 152#endif 153 uint64_t clidr; 154 uint32_t ccsidr[MAX_CACHES][2]; /* 2 possible types. */ 155 bool have_sve; 156}; 157 158static struct cpu_desc cpu_desc0; 159static struct cpu_desc *cpu_desc; 160static struct cpu_desc kern_cpu_desc; 161static struct cpu_desc user_cpu_desc; 162static struct cpu_desc l_user_cpu_desc; 163 164static struct cpu_desc * 165get_cpu_desc(u_int cpu) 166{ 167 /* The cpu_desc for CPU 0 is used before the allocator is ready. */ 168 if (cpu == 0) 169 return (&cpu_desc0); 170 171 MPASS(cpu_desc != NULL); 172 return (&cpu_desc[cpu - 1]); 173} 174 175struct cpu_parts { 176 u_int part_id; 177 const char *part_name; 178}; 179#define CPU_PART_NONE { 0, NULL } 180 181struct cpu_implementers { 182 u_int impl_id; 183 const char *impl_name; 184 /* 185 * Part number is implementation defined 186 * so each vendor will have its own set of values and names. 187 */ 188 const struct cpu_parts *cpu_parts; 189}; 190#define CPU_IMPLEMENTER_NONE { 0, NULL, NULL } 191 192/* 193 * Per-implementer table of (PartNum, CPU Name) pairs. 194 */ 195/* ARM Ltd. */ 196static const struct cpu_parts cpu_parts_arm[] = { 197 { CPU_PART_AEM_V8, "AEMv8" }, 198 { CPU_PART_FOUNDATION, "Foundation-Model" }, 199 { CPU_PART_CORTEX_A34, "Cortex-A34" }, 200 { CPU_PART_CORTEX_A35, "Cortex-A35" }, 201 { CPU_PART_CORTEX_A53, "Cortex-A53" }, 202 { CPU_PART_CORTEX_A55, "Cortex-A55" }, 203 { CPU_PART_CORTEX_A57, "Cortex-A57" }, 204 { CPU_PART_CORTEX_A65, "Cortex-A65" }, 205 { CPU_PART_CORTEX_A65AE, "Cortex-A65AE" }, 206 { CPU_PART_CORTEX_A72, "Cortex-A72" }, 207 { CPU_PART_CORTEX_A73, "Cortex-A73" }, 208 { CPU_PART_CORTEX_A75, "Cortex-A75" }, 209 { CPU_PART_CORTEX_A76, "Cortex-A76" }, 210 { CPU_PART_CORTEX_A76AE, "Cortex-A76AE" }, 211 { CPU_PART_CORTEX_A77, "Cortex-A77" }, 212 { CPU_PART_CORTEX_A78, "Cortex-A78" }, 213 { CPU_PART_CORTEX_A78C, "Cortex-A78C" }, 214 { CPU_PART_CORTEX_A510, "Cortex-A510" }, 215 { CPU_PART_CORTEX_A710, "Cortex-A710" }, 216 { CPU_PART_CORTEX_A715, "Cortex-A715" }, 217 { CPU_PART_CORTEX_X1, "Cortex-X1" }, 218 { CPU_PART_CORTEX_X1C, "Cortex-X1C" }, 219 { CPU_PART_CORTEX_X2, "Cortex-X2" }, 220 { CPU_PART_CORTEX_X3, "Cortex-X3" }, 221 { CPU_PART_NEOVERSE_E1, "Neoverse-E1" }, 222 { CPU_PART_NEOVERSE_N1, "Neoverse-N1" }, 223 { CPU_PART_NEOVERSE_N2, "Neoverse-N2" }, 224 { CPU_PART_NEOVERSE_V1, "Neoverse-V1" }, 225 { CPU_PART_NEOVERSE_V2, "Neoverse-V2" }, 226 CPU_PART_NONE, 227}; 228 229/* Cavium */ 230static const struct cpu_parts cpu_parts_cavium[] = { 231 { CPU_PART_THUNDERX, "ThunderX" }, 232 { CPU_PART_THUNDERX2, "ThunderX2" }, 233 CPU_PART_NONE, 234}; 235 236/* APM / Ampere */ 237static const struct cpu_parts cpu_parts_apm[] = { 238 { CPU_PART_EMAG8180, "eMAG 8180" }, 239 CPU_PART_NONE, 240}; 241 242/* Qualcomm */ 243static const struct cpu_parts cpu_parts_qcom[] = { 244 { CPU_PART_KRYO400_GOLD, "Kryo 400 Gold" }, 245 { CPU_PART_KRYO400_SILVER, "Kryo 400 Silver" }, 246 CPU_PART_NONE, 247}; 248 249/* Apple */ 250static const struct cpu_parts cpu_parts_apple[] = { 251 { CPU_PART_M1_ICESTORM, "M1 Icestorm" }, 252 { CPU_PART_M1_FIRESTORM, "M1 Firestorm" }, 253 { CPU_PART_M1_ICESTORM_PRO, "M1 Pro Icestorm" }, 254 { CPU_PART_M1_FIRESTORM_PRO, "M1 Pro Firestorm" }, 255 { CPU_PART_M1_ICESTORM_MAX, "M1 Max Icestorm" }, 256 { CPU_PART_M1_FIRESTORM_MAX, "M1 Max Firestorm" }, 257 { CPU_PART_M2_BLIZZARD, "M2 Blizzard" }, 258 { CPU_PART_M2_AVALANCHE, "M2 Avalanche" }, 259 { CPU_PART_M2_BLIZZARD_PRO, "M2 Pro Blizzard" }, 260 { CPU_PART_M2_AVALANCHE_PRO, "M2 Pro Avalanche" }, 261 { CPU_PART_M2_BLIZZARD_MAX, "M2 Max Blizzard" }, 262 { CPU_PART_M2_AVALANCHE_MAX, "M2 Max Avalanche" }, 263 CPU_PART_NONE, 264}; 265 266/* Unknown */ 267static const struct cpu_parts cpu_parts_none[] = { 268 CPU_PART_NONE, 269}; 270 271/* 272 * Implementers table. 273 */ 274const struct cpu_implementers cpu_implementers[] = { 275 { CPU_IMPL_AMPERE, "Ampere", cpu_parts_none }, 276 { CPU_IMPL_APPLE, "Apple", cpu_parts_apple }, 277 { CPU_IMPL_APM, "APM", cpu_parts_apm }, 278 { CPU_IMPL_ARM, "ARM", cpu_parts_arm }, 279 { CPU_IMPL_BROADCOM, "Broadcom", cpu_parts_none }, 280 { CPU_IMPL_CAVIUM, "Cavium", cpu_parts_cavium }, 281 { CPU_IMPL_DEC, "DEC", cpu_parts_none }, 282 { CPU_IMPL_FREESCALE, "Freescale", cpu_parts_none }, 283 { CPU_IMPL_FUJITSU, "Fujitsu", cpu_parts_none }, 284 { CPU_IMPL_INFINEON, "IFX", cpu_parts_none }, 285 { CPU_IMPL_INTEL, "Intel", cpu_parts_none }, 286 { CPU_IMPL_MARVELL, "Marvell", cpu_parts_none }, 287 { CPU_IMPL_NVIDIA, "NVIDIA", cpu_parts_none }, 288 { CPU_IMPL_QUALCOMM, "Qualcomm", cpu_parts_qcom }, 289 CPU_IMPLEMENTER_NONE, 290}; 291 292#define MRS_TYPE_MASK 0xf 293#define MRS_TYPE_FBSD_SHIFT 0 294#define MRS_TYPE_LNX_SHIFT 8 295#define MRS_INVALID 0 296#define MRS_EXACT 1 297#define MRS_EXACT_VAL(x) (MRS_EXACT | ((x) << 4)) 298#define MRS_EXACT_FIELD(x) (((x) >> 4) & 0xf) 299#define MRS_LOWER 2 300 301struct mrs_field_value { 302 uint64_t value; 303 const char *desc; 304}; 305 306#define MRS_FIELD_VALUE(_value, _desc) \ 307 { \ 308 .value = (_value), \ 309 .desc = (_desc), \ 310 } 311 312#define MRS_FIELD_VALUE_NONE_IMPL(_reg, _field, _none, _impl) \ 313 MRS_FIELD_VALUE(_reg ## _ ## _field ## _ ## _none, ""), \ 314 MRS_FIELD_VALUE(_reg ## _ ## _field ## _ ## _impl, #_field) 315 316#define MRS_FIELD_VALUE_COUNT(_reg, _field, _desc) \ 317 MRS_FIELD_VALUE(0ul << _reg ## _ ## _field ## _SHIFT, "1 " _desc), \ 318 MRS_FIELD_VALUE(1ul << _reg ## _ ## _field ## _SHIFT, "2 " _desc "s"), \ 319 MRS_FIELD_VALUE(2ul << _reg ## _ ## _field ## _SHIFT, "3 " _desc "s"), \ 320 MRS_FIELD_VALUE(3ul << _reg ## _ ## _field ## _SHIFT, "4 " _desc "s"), \ 321 MRS_FIELD_VALUE(4ul << _reg ## _ ## _field ## _SHIFT, "5 " _desc "s"), \ 322 MRS_FIELD_VALUE(5ul << _reg ## _ ## _field ## _SHIFT, "6 " _desc "s"), \ 323 MRS_FIELD_VALUE(6ul << _reg ## _ ## _field ## _SHIFT, "7 " _desc "s"), \ 324 MRS_FIELD_VALUE(7ul << _reg ## _ ## _field ## _SHIFT, "8 " _desc "s"), \ 325 MRS_FIELD_VALUE(8ul << _reg ## _ ## _field ## _SHIFT, "9 " _desc "s"), \ 326 MRS_FIELD_VALUE(9ul << _reg ## _ ## _field ## _SHIFT, "10 "_desc "s"), \ 327 MRS_FIELD_VALUE(10ul<< _reg ## _ ## _field ## _SHIFT, "11 "_desc "s"), \ 328 MRS_FIELD_VALUE(11ul<< _reg ## _ ## _field ## _SHIFT, "12 "_desc "s"), \ 329 MRS_FIELD_VALUE(12ul<< _reg ## _ ## _field ## _SHIFT, "13 "_desc "s"), \ 330 MRS_FIELD_VALUE(13ul<< _reg ## _ ## _field ## _SHIFT, "14 "_desc "s"), \ 331 MRS_FIELD_VALUE(14ul<< _reg ## _ ## _field ## _SHIFT, "15 "_desc "s"), \ 332 MRS_FIELD_VALUE(15ul<< _reg ## _ ## _field ## _SHIFT, "16 "_desc "s") 333 334#define MRS_FIELD_VALUE_END { .desc = NULL } 335 336struct mrs_field_hwcap { 337 uint64_t min; 338 u_long hwcap_val; 339 u_int hwcap_id; 340}; 341 342#define MRS_HWCAP(_hwcap_id, _val, _min) \ 343{ \ 344 .hwcap_id = (_hwcap_id), \ 345 .hwcap_val = (_val), \ 346 .min = (_min), \ 347} 348 349#define MRS_HWCAP_END { .hwcap_id = 0 } 350 351struct mrs_field { 352 const char *name; 353 const struct mrs_field_value *values; 354 const struct mrs_field_hwcap *hwcaps; 355 uint64_t mask; 356 bool sign; 357 u_int type; 358 u_int shift; 359}; 360 361#define MRS_FIELD_HWCAP_SPLIT(_register, _name, _sign, _fbsd_type, \ 362 _lnx_type, _values, _hwcap) \ 363 { \ 364 .name = #_name, \ 365 .sign = (_sign), \ 366 .type = ((_fbsd_type) << MRS_TYPE_FBSD_SHIFT) | \ 367 ((_lnx_type) << MRS_TYPE_LNX_SHIFT), \ 368 .shift = _register ## _ ## _name ## _SHIFT, \ 369 .mask = _register ## _ ## _name ## _MASK, \ 370 .values = (_values), \ 371 .hwcaps = (_hwcap), \ 372 } 373 374#define MRS_FIELD_HWCAP(_register, _name, _sign, _type, _values, _hwcap) \ 375 MRS_FIELD_HWCAP_SPLIT(_register, _name, _sign, _type, _type, \ 376 _values, _hwcap) 377 378#define MRS_FIELD(_register, _name, _sign, _type, _values) \ 379 MRS_FIELD_HWCAP(_register, _name, _sign, _type, _values, NULL) 380 381#define MRS_FIELD_END { .type = MRS_INVALID, } 382 383/* ID_AA64AFR0_EL1 */ 384static const struct mrs_field id_aa64afr0_fields[] = { 385 MRS_FIELD_END, 386}; 387 388 389/* ID_AA64AFR1_EL1 */ 390static const struct mrs_field id_aa64afr1_fields[] = { 391 MRS_FIELD_END, 392}; 393 394 395/* ID_AA64DFR0_EL1 */ 396static const struct mrs_field_value id_aa64dfr0_hpmn0[] = { 397 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, HPMN0, CONSTR, DEFINED), 398 MRS_FIELD_VALUE_END, 399}; 400 401static const struct mrs_field_value id_aa64dfr0_brbe[] = { 402 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, BRBE, NONE, IMPL), 403 MRS_FIELD_VALUE(ID_AA64DFR0_BRBE_EL3, "BRBE EL3"), 404 MRS_FIELD_VALUE_END, 405}; 406 407static const struct mrs_field_value id_aa64dfr0_mtpmu[] = { 408 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, MTPMU, NONE, IMPL), 409 MRS_FIELD_VALUE(ID_AA64DFR0_MTPMU_NONE_MT_RES0, "MTPMU res0"), 410 MRS_FIELD_VALUE_END, 411}; 412 413static const struct mrs_field_value id_aa64dfr0_tracebuffer[] = { 414 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, TraceBuffer, NONE, IMPL), 415 MRS_FIELD_VALUE_END, 416}; 417 418static const struct mrs_field_value id_aa64dfr0_tracefilt[] = { 419 MRS_FIELD_VALUE(ID_AA64DFR0_TraceFilt_NONE, ""), 420 MRS_FIELD_VALUE(ID_AA64DFR0_TraceFilt_8_4, "Trace v8.4"), 421 MRS_FIELD_VALUE_END, 422}; 423 424static const struct mrs_field_value id_aa64dfr0_doublelock[] = { 425 MRS_FIELD_VALUE(ID_AA64DFR0_DoubleLock_IMPL, "DoubleLock"), 426 MRS_FIELD_VALUE(ID_AA64DFR0_DoubleLock_NONE, ""), 427 MRS_FIELD_VALUE_END, 428}; 429 430static const struct mrs_field_value id_aa64dfr0_pmsver[] = { 431 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_NONE, ""), 432 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE, "SPE"), 433 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE_1_1, "SPEv1p1"), 434 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE_1_2, "SPEv1p2"), 435 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE_1_3, "SPEv1p3"), 436 MRS_FIELD_VALUE_END, 437}; 438 439static const struct mrs_field_value id_aa64dfr0_ctx_cmps[] = { 440 MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, CTX_CMPs, "CTX BKPT"), 441 MRS_FIELD_VALUE_END, 442}; 443 444static const struct mrs_field_value id_aa64dfr0_wrps[] = { 445 MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, WRPs, "Watchpoint"), 446 MRS_FIELD_VALUE_END, 447}; 448 449static const struct mrs_field_value id_aa64dfr0_brps[] = { 450 MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, BRPs, "Breakpoint"), 451 MRS_FIELD_VALUE_END, 452}; 453 454static const struct mrs_field_value id_aa64dfr0_pmuver[] = { 455 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_NONE, ""), 456 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3, "PMUv3"), 457 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_1, "PMUv3p1"), 458 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_4, "PMUv3p4"), 459 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_5, "PMUv3p5"), 460 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_7, "PMUv3p7"), 461 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_8, "PMUv3p8"), 462 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_IMPL, "IMPL PMU"), 463 MRS_FIELD_VALUE_END, 464}; 465 466static const struct mrs_field_value id_aa64dfr0_tracever[] = { 467 MRS_FIELD_VALUE(ID_AA64DFR0_TraceVer_NONE, ""), 468 MRS_FIELD_VALUE(ID_AA64DFR0_TraceVer_IMPL, "Trace"), 469 MRS_FIELD_VALUE_END, 470}; 471 472static const struct mrs_field_value id_aa64dfr0_debugver[] = { 473 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8, "Debugv8"), 474 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_VHE, "Debugv8_VHE"), 475 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_2, "Debugv8p2"), 476 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_4, "Debugv8p4"), 477 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_8, "Debugv8p8"), 478 MRS_FIELD_VALUE_END, 479}; 480 481static const struct mrs_field id_aa64dfr0_fields[] = { 482 MRS_FIELD(ID_AA64DFR0, HPMN0, false, MRS_EXACT, id_aa64dfr0_hpmn0), 483 MRS_FIELD(ID_AA64DFR0, BRBE, false, MRS_EXACT, id_aa64dfr0_brbe), 484 MRS_FIELD(ID_AA64DFR0, MTPMU, true, MRS_EXACT, id_aa64dfr0_mtpmu), 485 MRS_FIELD(ID_AA64DFR0, TraceBuffer, false, MRS_EXACT, 486 id_aa64dfr0_tracebuffer), 487 MRS_FIELD(ID_AA64DFR0, TraceFilt, false, MRS_EXACT, 488 id_aa64dfr0_tracefilt), 489 MRS_FIELD(ID_AA64DFR0, DoubleLock, false, MRS_EXACT, 490 id_aa64dfr0_doublelock), 491 MRS_FIELD(ID_AA64DFR0, PMSVer, false, MRS_EXACT, id_aa64dfr0_pmsver), 492 MRS_FIELD(ID_AA64DFR0, CTX_CMPs, false, MRS_EXACT, 493 id_aa64dfr0_ctx_cmps), 494 MRS_FIELD(ID_AA64DFR0, WRPs, false, MRS_LOWER, id_aa64dfr0_wrps), 495 MRS_FIELD(ID_AA64DFR0, BRPs, false, MRS_LOWER, id_aa64dfr0_brps), 496 MRS_FIELD(ID_AA64DFR0, PMUVer, false, MRS_EXACT, id_aa64dfr0_pmuver), 497 MRS_FIELD(ID_AA64DFR0, TraceVer, false, MRS_EXACT, 498 id_aa64dfr0_tracever), 499 MRS_FIELD(ID_AA64DFR0, DebugVer, false, MRS_EXACT_VAL(0x6), 500 id_aa64dfr0_debugver), 501 MRS_FIELD_END, 502}; 503 504 505/* ID_AA64DFR1_EL1 */ 506static const struct mrs_field id_aa64dfr1_fields[] = { 507 MRS_FIELD_END, 508}; 509 510 511/* ID_AA64ISAR0_EL1 */ 512static const struct mrs_field_value id_aa64isar0_rndr[] = { 513 MRS_FIELD_VALUE(ID_AA64ISAR0_RNDR_NONE, ""), 514 MRS_FIELD_VALUE(ID_AA64ISAR0_RNDR_IMPL, "RNG"), 515 MRS_FIELD_VALUE_END, 516}; 517 518static const struct mrs_field_hwcap id_aa64isar0_rndr_caps[] = { 519 MRS_HWCAP(2, HWCAP2_RNG, ID_AA64ISAR0_RNDR_IMPL), 520 MRS_HWCAP_END 521}; 522 523static const struct mrs_field_value id_aa64isar0_tlb[] = { 524 MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_NONE, ""), 525 MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_TLBIOS, "TLBI-OS"), 526 MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_TLBIOSR, "TLBI-OSR"), 527 MRS_FIELD_VALUE_END, 528}; 529 530static const struct mrs_field_value id_aa64isar0_ts[] = { 531 MRS_FIELD_VALUE(ID_AA64ISAR0_TS_NONE, ""), 532 MRS_FIELD_VALUE(ID_AA64ISAR0_TS_CondM_8_4, "CondM-8.4"), 533 MRS_FIELD_VALUE(ID_AA64ISAR0_TS_CondM_8_5, "CondM-8.5"), 534 MRS_FIELD_VALUE_END, 535}; 536 537static const struct mrs_field_hwcap id_aa64isar0_ts_caps[] = { 538 MRS_HWCAP(1, HWCAP_FLAGM, ID_AA64ISAR0_TS_CondM_8_4), 539 MRS_HWCAP(2, HWCAP2_FLAGM2, ID_AA64ISAR0_TS_CondM_8_5), 540 MRS_HWCAP_END 541}; 542 543static const struct mrs_field_value id_aa64isar0_fhm[] = { 544 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, FHM, NONE, IMPL), 545 MRS_FIELD_VALUE_END, 546}; 547 548static const struct mrs_field_hwcap id_aa64isar0_fhm_caps[] = { 549 MRS_HWCAP(1, HWCAP_ASIMDFHM, ID_AA64ISAR0_FHM_IMPL), 550 MRS_HWCAP_END 551}; 552 553static const struct mrs_field_value id_aa64isar0_dp[] = { 554 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, DP, NONE, IMPL), 555 MRS_FIELD_VALUE_END, 556}; 557 558static const struct mrs_field_hwcap id_aa64isar0_dp_caps[] = { 559 MRS_HWCAP(1, HWCAP_ASIMDDP, ID_AA64ISAR0_DP_IMPL), 560 MRS_HWCAP_END 561}; 562 563static const struct mrs_field_value id_aa64isar0_sm4[] = { 564 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SM4, NONE, IMPL), 565 MRS_FIELD_VALUE_END, 566}; 567 568static const struct mrs_field_hwcap id_aa64isar0_sm4_caps[] = { 569 MRS_HWCAP(1, HWCAP_SM4, ID_AA64ISAR0_SM4_IMPL), 570 MRS_HWCAP_END 571}; 572 573static const struct mrs_field_value id_aa64isar0_sm3[] = { 574 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SM3, NONE, IMPL), 575 MRS_FIELD_VALUE_END, 576}; 577 578static const struct mrs_field_hwcap id_aa64isar0_sm3_caps[] = { 579 MRS_HWCAP(1, HWCAP_SM3, ID_AA64ISAR0_SM3_IMPL), 580 MRS_HWCAP_END 581}; 582 583static const struct mrs_field_value id_aa64isar0_sha3[] = { 584 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA3, NONE, IMPL), 585 MRS_FIELD_VALUE_END, 586}; 587 588static const struct mrs_field_hwcap id_aa64isar0_sha3_caps[] = { 589 MRS_HWCAP(1, HWCAP_SHA3, ID_AA64ISAR0_SHA3_IMPL), 590 MRS_HWCAP_END 591}; 592 593static const struct mrs_field_value id_aa64isar0_rdm[] = { 594 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, RDM, NONE, IMPL), 595 MRS_FIELD_VALUE_END, 596}; 597 598static const struct mrs_field_hwcap id_aa64isar0_rdm_caps[] = { 599 MRS_HWCAP(1, HWCAP_ASIMDRDM, ID_AA64ISAR0_RDM_IMPL), 600 MRS_HWCAP_END 601}; 602 603static const struct mrs_field_value id_aa64isar0_tme[] = { 604 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, TME, NONE, IMPL), 605 MRS_FIELD_VALUE_END, 606}; 607 608static const struct mrs_field_value id_aa64isar0_atomic[] = { 609 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, Atomic, NONE, IMPL), 610 MRS_FIELD_VALUE_END, 611}; 612 613static const struct mrs_field_hwcap id_aa64isar0_atomic_caps[] = { 614 MRS_HWCAP(1, HWCAP_ATOMICS, ID_AA64ISAR0_Atomic_IMPL), 615 MRS_HWCAP_END 616}; 617 618static const struct mrs_field_value id_aa64isar0_crc32[] = { 619 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, CRC32, NONE, BASE), 620 MRS_FIELD_VALUE_END, 621}; 622 623static const struct mrs_field_hwcap id_aa64isar0_crc32_caps[] = { 624 MRS_HWCAP(1, HWCAP_CRC32, ID_AA64ISAR0_CRC32_BASE), 625 MRS_HWCAP_END 626}; 627 628static const struct mrs_field_value id_aa64isar0_sha2[] = { 629 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA2, NONE, BASE), 630 MRS_FIELD_VALUE(ID_AA64ISAR0_SHA2_512, "SHA2+SHA512"), 631 MRS_FIELD_VALUE_END, 632}; 633 634static const struct mrs_field_hwcap id_aa64isar0_sha2_caps[] = { 635 MRS_HWCAP(1, HWCAP_SHA2, ID_AA64ISAR0_SHA2_BASE), 636 MRS_HWCAP(1, HWCAP_SHA512, ID_AA64ISAR0_SHA2_512), 637 MRS_HWCAP_END 638}; 639 640static const struct mrs_field_value id_aa64isar0_sha1[] = { 641 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA1, NONE, BASE), 642 MRS_FIELD_VALUE_END, 643}; 644 645static const struct mrs_field_hwcap id_aa64isar0_sha1_caps[] = { 646 MRS_HWCAP(1, HWCAP_SHA1, ID_AA64ISAR0_SHA1_BASE), 647 MRS_HWCAP_END 648}; 649 650static const struct mrs_field_value id_aa64isar0_aes[] = { 651 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, AES, NONE, BASE), 652 MRS_FIELD_VALUE(ID_AA64ISAR0_AES_PMULL, "AES+PMULL"), 653 MRS_FIELD_VALUE_END, 654}; 655 656static const struct mrs_field_hwcap id_aa64isar0_aes_caps[] = { 657 MRS_HWCAP(1, HWCAP_AES, ID_AA64ISAR0_AES_BASE), 658 MRS_HWCAP(1, HWCAP_PMULL, ID_AA64ISAR0_AES_PMULL), 659 MRS_HWCAP_END 660}; 661 662static const struct mrs_field id_aa64isar0_fields[] = { 663 MRS_FIELD_HWCAP(ID_AA64ISAR0, RNDR, false, MRS_LOWER, 664 id_aa64isar0_rndr, id_aa64isar0_rndr_caps), 665 MRS_FIELD(ID_AA64ISAR0, TLB, false, MRS_EXACT, id_aa64isar0_tlb), 666 MRS_FIELD_HWCAP(ID_AA64ISAR0, TS, false, MRS_LOWER, id_aa64isar0_ts, 667 id_aa64isar0_ts_caps), 668 MRS_FIELD_HWCAP(ID_AA64ISAR0, FHM, false, MRS_LOWER, id_aa64isar0_fhm, 669 id_aa64isar0_fhm_caps), 670 MRS_FIELD_HWCAP(ID_AA64ISAR0, DP, false, MRS_LOWER, id_aa64isar0_dp, 671 id_aa64isar0_dp_caps), 672 MRS_FIELD_HWCAP(ID_AA64ISAR0, SM4, false, MRS_LOWER, id_aa64isar0_sm4, 673 id_aa64isar0_sm4_caps), 674 MRS_FIELD_HWCAP(ID_AA64ISAR0, SM3, false, MRS_LOWER, id_aa64isar0_sm3, 675 id_aa64isar0_sm3_caps), 676 MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA3, false, MRS_LOWER, id_aa64isar0_sha3, 677 id_aa64isar0_sha3_caps), 678 MRS_FIELD_HWCAP(ID_AA64ISAR0, RDM, false, MRS_LOWER, id_aa64isar0_rdm, 679 id_aa64isar0_rdm_caps), 680 MRS_FIELD(ID_AA64ISAR0, TME, false, MRS_EXACT, id_aa64isar0_tme), 681 MRS_FIELD_HWCAP(ID_AA64ISAR0, Atomic, false, MRS_LOWER, 682 id_aa64isar0_atomic, id_aa64isar0_atomic_caps), 683 MRS_FIELD_HWCAP(ID_AA64ISAR0, CRC32, false, MRS_LOWER, 684 id_aa64isar0_crc32, id_aa64isar0_crc32_caps), 685 MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA2, false, MRS_LOWER, id_aa64isar0_sha2, 686 id_aa64isar0_sha2_caps), 687 MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA1, false, MRS_LOWER, 688 id_aa64isar0_sha1, id_aa64isar0_sha1_caps), 689 MRS_FIELD_HWCAP(ID_AA64ISAR0, AES, false, MRS_LOWER, id_aa64isar0_aes, 690 id_aa64isar0_aes_caps), 691 MRS_FIELD_END, 692}; 693 694 695/* ID_AA64ISAR1_EL1 */ 696static const struct mrs_field_value id_aa64isar1_ls64[] = { 697 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, LS64, NONE, IMPL), 698 MRS_FIELD_VALUE(ID_AA64ISAR1_LS64_V, "LS64v"), 699 MRS_FIELD_VALUE(ID_AA64ISAR1_LS64_ACCDATA, "LS64+ACCDATA"), 700 MRS_FIELD_VALUE_END, 701}; 702 703static const struct mrs_field_value id_aa64isar1_xs[] = { 704 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, XS, NONE, IMPL), 705 MRS_FIELD_VALUE_END, 706}; 707 708static const struct mrs_field_value id_aa64isar1_i8mm[] = { 709 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, I8MM, NONE, IMPL), 710 MRS_FIELD_VALUE_END, 711}; 712 713static const struct mrs_field_hwcap id_aa64isar1_i8mm_caps[] = { 714 MRS_HWCAP(2, HWCAP2_I8MM, ID_AA64ISAR1_I8MM_IMPL), 715 MRS_HWCAP_END 716}; 717 718static const struct mrs_field_value id_aa64isar1_dgh[] = { 719 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, DGH, NONE, IMPL), 720 MRS_FIELD_VALUE_END, 721}; 722 723static const struct mrs_field_hwcap id_aa64isar1_dgh_caps[] = { 724 MRS_HWCAP(2, HWCAP2_DGH, ID_AA64ISAR1_DGH_IMPL), 725 MRS_HWCAP_END 726}; 727 728static const struct mrs_field_value id_aa64isar1_bf16[] = { 729 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, BF16, NONE, IMPL), 730 MRS_FIELD_VALUE(ID_AA64ISAR1_BF16_EBF, "EBF16"), 731 MRS_FIELD_VALUE_END, 732}; 733 734static const struct mrs_field_hwcap id_aa64isar1_bf16_caps[] = { 735 MRS_HWCAP(2, HWCAP2_BF16, ID_AA64ISAR1_BF16_IMPL), 736 MRS_HWCAP_END 737}; 738 739static const struct mrs_field_value id_aa64isar1_specres[] = { 740 MRS_FIELD_VALUE(ID_AA64ISAR1_SPECRES_NONE, ""), 741 MRS_FIELD_VALUE(ID_AA64ISAR1_SPECRES_IMPL, "PredInv"), 742 MRS_FIELD_VALUE_END, 743}; 744 745static const struct mrs_field_value id_aa64isar1_sb[] = { 746 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, SB, NONE, IMPL), 747 MRS_FIELD_VALUE_END, 748}; 749 750static const struct mrs_field_hwcap id_aa64isar1_sb_caps[] = { 751 MRS_HWCAP(1, HWCAP_SB, ID_AA64ISAR1_SB_IMPL), 752 MRS_HWCAP_END 753}; 754 755static const struct mrs_field_value id_aa64isar1_frintts[] = { 756 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, FRINTTS, NONE, IMPL), 757 MRS_FIELD_VALUE_END, 758}; 759 760static const struct mrs_field_hwcap id_aa64isar1_frintts_caps[] = { 761 MRS_HWCAP(2, HWCAP2_FRINT, ID_AA64ISAR1_FRINTTS_IMPL), 762 MRS_HWCAP_END 763}; 764 765static const struct mrs_field_value id_aa64isar1_gpi[] = { 766 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, GPI, NONE, IMPL), 767 MRS_FIELD_VALUE_END, 768}; 769 770static const struct mrs_field_hwcap id_aa64isar1_gpi_caps[] = { 771 MRS_HWCAP(1, HWCAP_PACG, ID_AA64ISAR1_GPI_IMPL), 772 MRS_HWCAP_END 773}; 774 775static const struct mrs_field_value id_aa64isar1_gpa[] = { 776 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, GPA, NONE, IMPL), 777 MRS_FIELD_VALUE_END, 778}; 779 780static const struct mrs_field_hwcap id_aa64isar1_gpa_caps[] = { 781 MRS_HWCAP(1, HWCAP_PACG, ID_AA64ISAR1_GPA_IMPL), 782 MRS_HWCAP_END 783}; 784 785static const struct mrs_field_value id_aa64isar1_lrcpc[] = { 786 MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_NONE, ""), 787 MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_RCPC_8_3, "RCPC-8.3"), 788 MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_RCPC_8_4, "RCPC-8.4"), 789 MRS_FIELD_VALUE_END, 790}; 791 792static const struct mrs_field_hwcap id_aa64isar1_lrcpc_caps[] = { 793 MRS_HWCAP(1, HWCAP_LRCPC, ID_AA64ISAR1_LRCPC_RCPC_8_3), 794 MRS_HWCAP(1, HWCAP_ILRCPC, ID_AA64ISAR1_LRCPC_RCPC_8_4), 795 MRS_HWCAP_END 796}; 797 798static const struct mrs_field_value id_aa64isar1_fcma[] = { 799 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, FCMA, NONE, IMPL), 800 MRS_FIELD_VALUE_END, 801}; 802 803static const struct mrs_field_hwcap id_aa64isar1_fcma_caps[] = { 804 MRS_HWCAP(1, HWCAP_FCMA, ID_AA64ISAR1_FCMA_IMPL), 805 MRS_HWCAP_END 806}; 807 808static const struct mrs_field_value id_aa64isar1_jscvt[] = { 809 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, JSCVT, NONE, IMPL), 810 MRS_FIELD_VALUE_END, 811}; 812 813static const struct mrs_field_hwcap id_aa64isar1_jscvt_caps[] = { 814 MRS_HWCAP(1, HWCAP_JSCVT, ID_AA64ISAR1_JSCVT_IMPL), 815 MRS_HWCAP_END 816}; 817 818static const struct mrs_field_value id_aa64isar1_api[] = { 819 MRS_FIELD_VALUE(ID_AA64ISAR1_API_NONE, ""), 820 MRS_FIELD_VALUE(ID_AA64ISAR1_API_PAC, "API PAC"), 821 MRS_FIELD_VALUE(ID_AA64ISAR1_API_EPAC, "API EPAC"), 822 MRS_FIELD_VALUE(ID_AA64ISAR1_API_EPAC2, "Impl PAuth+EPAC2"), 823 MRS_FIELD_VALUE(ID_AA64ISAR1_API_FPAC, "Impl PAuth+FPAC"), 824 MRS_FIELD_VALUE(ID_AA64ISAR1_API_FPAC_COMBINED, 825 "Impl PAuth+FPAC+Combined"), 826 MRS_FIELD_VALUE_END, 827}; 828 829static const struct mrs_field_hwcap id_aa64isar1_api_caps[] = { 830 MRS_HWCAP(1, HWCAP_PACA, ID_AA64ISAR1_API_PAC), 831 MRS_HWCAP_END 832}; 833 834static const struct mrs_field_value id_aa64isar1_apa[] = { 835 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_NONE, ""), 836 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_PAC, "APA PAC"), 837 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_EPAC, "APA EPAC"), 838 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_EPAC2, "APA EPAC2"), 839 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_FPAC, "APA FPAC"), 840 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_FPAC_COMBINED, 841 "APA FPAC+Combined"), 842 MRS_FIELD_VALUE_END, 843}; 844 845static const struct mrs_field_hwcap id_aa64isar1_apa_caps[] = { 846 MRS_HWCAP(1, HWCAP_PACA, ID_AA64ISAR1_APA_PAC), 847 MRS_HWCAP_END 848}; 849 850static const struct mrs_field_value id_aa64isar1_dpb[] = { 851 MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_NONE, ""), 852 MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_DCCVAP, "DCPoP"), 853 MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_DCCVADP, "DCCVADP"), 854 MRS_FIELD_VALUE_END, 855}; 856 857static const struct mrs_field_hwcap id_aa64isar1_dpb_caps[] = { 858 MRS_HWCAP(1, HWCAP_DCPOP, ID_AA64ISAR1_DPB_DCCVAP), 859 MRS_HWCAP(2, HWCAP2_DCPODP, ID_AA64ISAR1_DPB_DCCVADP), 860 MRS_HWCAP_END 861}; 862 863static const struct mrs_field id_aa64isar1_fields[] = { 864 MRS_FIELD(ID_AA64ISAR1, LS64, false, MRS_EXACT, id_aa64isar1_ls64), 865 MRS_FIELD(ID_AA64ISAR1, XS, false, MRS_EXACT, id_aa64isar1_xs), 866 MRS_FIELD_HWCAP(ID_AA64ISAR1, I8MM, false, MRS_LOWER, 867 id_aa64isar1_i8mm, id_aa64isar1_i8mm_caps), 868 MRS_FIELD_HWCAP(ID_AA64ISAR1, DGH, false, MRS_LOWER, id_aa64isar1_dgh, 869 id_aa64isar1_dgh_caps), 870 MRS_FIELD_HWCAP(ID_AA64ISAR1, BF16, false, MRS_LOWER, 871 id_aa64isar1_bf16, id_aa64isar1_bf16_caps), 872 MRS_FIELD(ID_AA64ISAR1, SPECRES, false, MRS_EXACT, 873 id_aa64isar1_specres), 874 MRS_FIELD_HWCAP(ID_AA64ISAR1, SB, false, MRS_LOWER, id_aa64isar1_sb, 875 id_aa64isar1_sb_caps), 876 MRS_FIELD_HWCAP(ID_AA64ISAR1, FRINTTS, false, MRS_LOWER, 877 id_aa64isar1_frintts, id_aa64isar1_frintts_caps), 878 MRS_FIELD_HWCAP(ID_AA64ISAR1, GPI, false, MRS_EXACT, id_aa64isar1_gpi, 879 id_aa64isar1_gpi_caps), 880 MRS_FIELD_HWCAP(ID_AA64ISAR1, GPA, false, MRS_EXACT, id_aa64isar1_gpa, 881 id_aa64isar1_gpa_caps), 882 MRS_FIELD_HWCAP(ID_AA64ISAR1, LRCPC, false, MRS_LOWER, 883 id_aa64isar1_lrcpc, id_aa64isar1_lrcpc_caps), 884 MRS_FIELD_HWCAP(ID_AA64ISAR1, FCMA, false, MRS_LOWER, 885 id_aa64isar1_fcma, id_aa64isar1_fcma_caps), 886 MRS_FIELD_HWCAP(ID_AA64ISAR1, JSCVT, false, MRS_LOWER, 887 id_aa64isar1_jscvt, id_aa64isar1_jscvt_caps), 888 MRS_FIELD_HWCAP(ID_AA64ISAR1, API, false, MRS_EXACT, id_aa64isar1_api, 889 id_aa64isar1_api_caps), 890 MRS_FIELD_HWCAP(ID_AA64ISAR1, APA, false, MRS_EXACT, id_aa64isar1_apa, 891 id_aa64isar1_apa_caps), 892 MRS_FIELD_HWCAP(ID_AA64ISAR1, DPB, false, MRS_LOWER, id_aa64isar1_dpb, 893 id_aa64isar1_dpb_caps), 894 MRS_FIELD_END, 895}; 896 897 898/* ID_AA64ISAR2_EL1 */ 899static const struct mrs_field_value id_aa64isar2_pac_frac[] = { 900 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, PAC_frac, NONE, IMPL), 901 MRS_FIELD_VALUE_END, 902}; 903 904static const struct mrs_field_value id_aa64isar2_bc[] = { 905 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, BC, NONE, IMPL), 906 MRS_FIELD_VALUE_END, 907}; 908 909static const struct mrs_field_value id_aa64isar2_mops[] = { 910 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, MOPS, NONE, IMPL), 911 MRS_FIELD_VALUE_END, 912}; 913 914static const struct mrs_field_value id_aa64isar2_apa3[] = { 915 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_NONE, ""), 916 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_PAC, "APA3 PAC"), 917 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_EPAC, "APA3 EPAC"), 918 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_EPAC2, "APA3 EPAC2"), 919 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_FPAC, "APA3 FPAC"), 920 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_FPAC_COMBINED, 921 "APA3 FPAC+Combined"), 922 MRS_FIELD_VALUE_END, 923}; 924 925static const struct mrs_field_hwcap id_aa64isar2_apa3_caps[] = { 926 MRS_HWCAP(1, HWCAP_PACA, ID_AA64ISAR2_APA3_PAC), 927 MRS_HWCAP_END 928}; 929 930static const struct mrs_field_value id_aa64isar2_gpa3[] = { 931 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, GPA3, NONE, IMPL), 932 MRS_FIELD_VALUE_END, 933}; 934 935static const struct mrs_field_hwcap id_aa64isar2_gpa3_caps[] = { 936 MRS_HWCAP(1, HWCAP_PACG, ID_AA64ISAR2_GPA3_IMPL), 937 MRS_HWCAP_END 938}; 939 940static const struct mrs_field_value id_aa64isar2_rpres[] = { 941 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, RPRES, NONE, IMPL), 942 MRS_FIELD_VALUE_END, 943}; 944 945static const struct mrs_field_value id_aa64isar2_wfxt[] = { 946 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, WFxT, NONE, IMPL), 947 MRS_FIELD_VALUE_END, 948}; 949 950static const struct mrs_field id_aa64isar2_fields[] = { 951 MRS_FIELD(ID_AA64ISAR2, PAC_frac, false, MRS_EXACT, 952 id_aa64isar2_pac_frac), 953 MRS_FIELD(ID_AA64ISAR2, BC, false, MRS_EXACT, id_aa64isar2_bc), 954 MRS_FIELD(ID_AA64ISAR2, MOPS, false, MRS_EXACT, id_aa64isar2_mops), 955 MRS_FIELD_HWCAP(ID_AA64ISAR2, APA3, false, MRS_EXACT, 956 id_aa64isar2_apa3, id_aa64isar2_apa3_caps), 957 MRS_FIELD_HWCAP(ID_AA64ISAR2, GPA3, false, MRS_EXACT, 958 id_aa64isar2_gpa3, id_aa64isar2_gpa3_caps), 959 MRS_FIELD(ID_AA64ISAR2, RPRES, false, MRS_EXACT, id_aa64isar2_rpres), 960 MRS_FIELD(ID_AA64ISAR2, WFxT, false, MRS_EXACT, id_aa64isar2_wfxt), 961 MRS_FIELD_END, 962}; 963 964 965/* ID_AA64MMFR0_EL1 */ 966static const struct mrs_field_value id_aa64mmfr0_ecv[] = { 967 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, ECV, NONE, IMPL), 968 MRS_FIELD_VALUE(ID_AA64MMFR0_ECV_CNTHCTL, "ECV+CNTHCTL"), 969 MRS_FIELD_VALUE_END, 970}; 971 972static const struct mrs_field_value id_aa64mmfr0_fgt[] = { 973 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, FGT, NONE, IMPL), 974 MRS_FIELD_VALUE_END, 975}; 976 977static const struct mrs_field_value id_aa64mmfr0_exs[] = { 978 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, ExS, ALL, IMPL), 979 MRS_FIELD_VALUE_END, 980}; 981 982static const struct mrs_field_value id_aa64mmfr0_tgran4_2[] = { 983 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_TGran4, ""), 984 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_NONE, "No S2 TGran4"), 985 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_IMPL, "S2 TGran4"), 986 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_LPA2, "S2 TGran4+LPA2"), 987 MRS_FIELD_VALUE_END, 988}; 989 990static const struct mrs_field_value id_aa64mmfr0_tgran64_2[] = { 991 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_TGran64, ""), 992 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_NONE, "No S2 TGran64"), 993 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_IMPL, "S2 TGran64"), 994 MRS_FIELD_VALUE_END, 995}; 996 997static const struct mrs_field_value id_aa64mmfr0_tgran16_2[] = { 998 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_TGran16, ""), 999 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_NONE, "No S2 TGran16"), 1000 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_IMPL, "S2 TGran16"), 1001 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_LPA2, "S2 TGran16+LPA2"), 1002 MRS_FIELD_VALUE_END, 1003}; 1004 1005static const struct mrs_field_value id_aa64mmfr0_tgran4[] = { 1006 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran4, NONE, IMPL), 1007 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_LPA2, "TGran4+LPA2"), 1008 MRS_FIELD_VALUE_END, 1009}; 1010 1011static const struct mrs_field_value id_aa64mmfr0_tgran64[] = { 1012 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran64, NONE, IMPL), 1013 MRS_FIELD_VALUE_END, 1014}; 1015 1016static const struct mrs_field_value id_aa64mmfr0_tgran16[] = { 1017 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran16, NONE, IMPL), 1018 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_LPA2, "TGran16+LPA2"), 1019 MRS_FIELD_VALUE_END, 1020}; 1021 1022static const struct mrs_field_value id_aa64mmfr0_bigendel0[] = { 1023 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, BigEndEL0, FIXED, MIXED), 1024 MRS_FIELD_VALUE_END, 1025}; 1026 1027static const struct mrs_field_value id_aa64mmfr0_snsmem[] = { 1028 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, SNSMem, NONE, DISTINCT), 1029 MRS_FIELD_VALUE_END, 1030}; 1031 1032static const struct mrs_field_value id_aa64mmfr0_bigend[] = { 1033 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, BigEnd, FIXED, MIXED), 1034 MRS_FIELD_VALUE_END, 1035}; 1036 1037static const struct mrs_field_value id_aa64mmfr0_asidbits[] = { 1038 MRS_FIELD_VALUE(ID_AA64MMFR0_ASIDBits_8, "8bit ASID"), 1039 MRS_FIELD_VALUE(ID_AA64MMFR0_ASIDBits_16, "16bit ASID"), 1040 MRS_FIELD_VALUE_END, 1041}; 1042 1043static const struct mrs_field_value id_aa64mmfr0_parange[] = { 1044 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4G, "4GB PA"), 1045 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_64G, "64GB PA"), 1046 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_1T, "1TB PA"), 1047 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4T, "4TB PA"), 1048 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_16T, "16TB PA"), 1049 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_256T, "256TB PA"), 1050 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4P, "4PB PA"), 1051 MRS_FIELD_VALUE_END, 1052}; 1053 1054static const struct mrs_field id_aa64mmfr0_fields[] = { 1055 MRS_FIELD(ID_AA64MMFR0, ECV, false, MRS_EXACT, id_aa64mmfr0_ecv), 1056 MRS_FIELD(ID_AA64MMFR0, FGT, false, MRS_EXACT, id_aa64mmfr0_fgt), 1057 MRS_FIELD(ID_AA64MMFR0, ExS, false, MRS_EXACT, id_aa64mmfr0_exs), 1058 MRS_FIELD(ID_AA64MMFR0, TGran4_2, false, MRS_EXACT, 1059 id_aa64mmfr0_tgran4_2), 1060 MRS_FIELD(ID_AA64MMFR0, TGran64_2, false, MRS_EXACT, 1061 id_aa64mmfr0_tgran64_2), 1062 MRS_FIELD(ID_AA64MMFR0, TGran16_2, false, MRS_EXACT, 1063 id_aa64mmfr0_tgran16_2), 1064 MRS_FIELD(ID_AA64MMFR0, TGran4, false, MRS_EXACT, id_aa64mmfr0_tgran4), 1065 MRS_FIELD(ID_AA64MMFR0, TGran64, false, MRS_EXACT, 1066 id_aa64mmfr0_tgran64), 1067 MRS_FIELD(ID_AA64MMFR0, TGran16, false, MRS_EXACT, 1068 id_aa64mmfr0_tgran16), 1069 MRS_FIELD(ID_AA64MMFR0, BigEndEL0, false, MRS_EXACT, 1070 id_aa64mmfr0_bigendel0), 1071 MRS_FIELD(ID_AA64MMFR0, SNSMem, false, MRS_EXACT, id_aa64mmfr0_snsmem), 1072 MRS_FIELD(ID_AA64MMFR0, BigEnd, false, MRS_EXACT, id_aa64mmfr0_bigend), 1073 MRS_FIELD(ID_AA64MMFR0, ASIDBits, false, MRS_EXACT, 1074 id_aa64mmfr0_asidbits), 1075 MRS_FIELD(ID_AA64MMFR0, PARange, false, MRS_EXACT, 1076 id_aa64mmfr0_parange), 1077 MRS_FIELD_END, 1078}; 1079 1080 1081/* ID_AA64MMFR1_EL1 */ 1082static const struct mrs_field_value id_aa64mmfr1_cmovw[] = { 1083 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, CMOVW, NONE, IMPL), 1084 MRS_FIELD_VALUE_END, 1085}; 1086 1087static const struct mrs_field_value id_aa64mmfr1_tidcp1[] = { 1088 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, TIDCP1, NONE, IMPL), 1089 MRS_FIELD_VALUE_END, 1090}; 1091 1092static const struct mrs_field_value id_aa64mmfr1_ntlbpa[] = { 1093 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, nTLBPA, NONE, IMPL), 1094 MRS_FIELD_VALUE_END, 1095}; 1096 1097static const struct mrs_field_value id_aa64mmfr1_afp[] = { 1098 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, AFP, NONE, IMPL), 1099 MRS_FIELD_VALUE_END, 1100}; 1101 1102static const struct mrs_field_value id_aa64mmfr1_hcx[] = { 1103 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, HCX, NONE, IMPL), 1104 MRS_FIELD_VALUE_END, 1105}; 1106 1107static const struct mrs_field_value id_aa64mmfr1_ets[] = { 1108 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, ETS, NONE, IMPL), 1109 MRS_FIELD_VALUE_END, 1110}; 1111 1112static const struct mrs_field_value id_aa64mmfr1_twed[] = { 1113 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, TWED, NONE, IMPL), 1114 MRS_FIELD_VALUE_END, 1115}; 1116 1117static const struct mrs_field_value id_aa64mmfr1_xnx[] = { 1118 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, XNX, NONE, IMPL), 1119 MRS_FIELD_VALUE_END, 1120}; 1121 1122static const struct mrs_field_value id_aa64mmfr1_specsei[] = { 1123 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, SpecSEI, NONE, IMPL), 1124 MRS_FIELD_VALUE_END, 1125}; 1126 1127static const struct mrs_field_value id_aa64mmfr1_pan[] = { 1128 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, PAN, NONE, IMPL), 1129 MRS_FIELD_VALUE(ID_AA64MMFR1_PAN_ATS1E1, "PAN+ATS1E1"), 1130 MRS_FIELD_VALUE(ID_AA64MMFR1_PAN_EPAN, "EPAN"), 1131 MRS_FIELD_VALUE_END, 1132}; 1133 1134static const struct mrs_field_value id_aa64mmfr1_lo[] = { 1135 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, LO, NONE, IMPL), 1136 MRS_FIELD_VALUE_END, 1137}; 1138 1139static const struct mrs_field_value id_aa64mmfr1_hpds[] = { 1140 MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_NONE, ""), 1141 MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_HPD, "HPD"), 1142 MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_TTPBHA, "HPD+TTPBHA"), 1143 MRS_FIELD_VALUE_END, 1144}; 1145 1146static const struct mrs_field_value id_aa64mmfr1_vh[] = { 1147 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, VH, NONE, IMPL), 1148 MRS_FIELD_VALUE_END, 1149}; 1150 1151static const struct mrs_field_value id_aa64mmfr1_vmidbits[] = { 1152 MRS_FIELD_VALUE(ID_AA64MMFR1_VMIDBits_8, "8bit VMID"), 1153 MRS_FIELD_VALUE(ID_AA64MMFR1_VMIDBits_16, "16bit VMID"), 1154 MRS_FIELD_VALUE_END, 1155}; 1156 1157static const struct mrs_field_value id_aa64mmfr1_hafdbs[] = { 1158 MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_NONE, ""), 1159 MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_AF, "HAF"), 1160 MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_AF_DBS, "HAF+DS"), 1161 MRS_FIELD_VALUE_END, 1162}; 1163 1164static const struct mrs_field id_aa64mmfr1_fields[] = { 1165 MRS_FIELD(ID_AA64MMFR1, CMOVW, false, MRS_EXACT, id_aa64mmfr1_cmovw), 1166 MRS_FIELD(ID_AA64MMFR1, TIDCP1, false, MRS_EXACT, id_aa64mmfr1_tidcp1), 1167 MRS_FIELD(ID_AA64MMFR1, nTLBPA, false, MRS_EXACT, id_aa64mmfr1_ntlbpa), 1168 MRS_FIELD(ID_AA64MMFR1, AFP, false, MRS_EXACT, id_aa64mmfr1_afp), 1169 MRS_FIELD(ID_AA64MMFR1, HCX, false, MRS_EXACT, id_aa64mmfr1_hcx), 1170 MRS_FIELD(ID_AA64MMFR1, ETS, false, MRS_EXACT, id_aa64mmfr1_ets), 1171 MRS_FIELD(ID_AA64MMFR1, TWED, false, MRS_EXACT, id_aa64mmfr1_twed), 1172 MRS_FIELD(ID_AA64MMFR1, XNX, false, MRS_EXACT, id_aa64mmfr1_xnx), 1173 MRS_FIELD(ID_AA64MMFR1, SpecSEI, false, MRS_EXACT, 1174 id_aa64mmfr1_specsei), 1175 MRS_FIELD(ID_AA64MMFR1, PAN, false, MRS_EXACT, id_aa64mmfr1_pan), 1176 MRS_FIELD(ID_AA64MMFR1, LO, false, MRS_EXACT, id_aa64mmfr1_lo), 1177 MRS_FIELD(ID_AA64MMFR1, HPDS, false, MRS_EXACT, id_aa64mmfr1_hpds), 1178 MRS_FIELD(ID_AA64MMFR1, VH, false, MRS_EXACT, id_aa64mmfr1_vh), 1179 MRS_FIELD(ID_AA64MMFR1, VMIDBits, false, MRS_EXACT, 1180 id_aa64mmfr1_vmidbits), 1181 MRS_FIELD(ID_AA64MMFR1, HAFDBS, false, MRS_EXACT, id_aa64mmfr1_hafdbs), 1182 MRS_FIELD_END, 1183}; 1184 1185 1186/* ID_AA64MMFR2_EL1 */ 1187static const struct mrs_field_value id_aa64mmfr2_e0pd[] = { 1188 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, E0PD, NONE, IMPL), 1189 MRS_FIELD_VALUE_END, 1190}; 1191 1192static const struct mrs_field_value id_aa64mmfr2_evt[] = { 1193 MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_NONE, ""), 1194 MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_8_2, "EVT-8.2"), 1195 MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_8_5, "EVT-8.5"), 1196 MRS_FIELD_VALUE_END, 1197}; 1198 1199static const struct mrs_field_value id_aa64mmfr2_bbm[] = { 1200 MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL0, ""), 1201 MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL1, "BBM level 1"), 1202 MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL2, "BBM level 2"), 1203 MRS_FIELD_VALUE_END, 1204}; 1205 1206static const struct mrs_field_value id_aa64mmfr2_ttl[] = { 1207 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, TTL, NONE, IMPL), 1208 MRS_FIELD_VALUE_END, 1209}; 1210 1211static const struct mrs_field_value id_aa64mmfr2_fwb[] = { 1212 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, FWB, NONE, IMPL), 1213 MRS_FIELD_VALUE_END, 1214}; 1215 1216static const struct mrs_field_value id_aa64mmfr2_ids[] = { 1217 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, IDS, NONE, IMPL), 1218 MRS_FIELD_VALUE_END, 1219}; 1220 1221static const struct mrs_field_value id_aa64mmfr2_at[] = { 1222 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, AT, NONE, IMPL), 1223 MRS_FIELD_VALUE_END, 1224}; 1225 1226static const struct mrs_field_hwcap id_aa64mmfr2_at_caps[] = { 1227 MRS_HWCAP(1, HWCAP_USCAT, ID_AA64MMFR2_AT_IMPL), 1228 MRS_HWCAP_END 1229}; 1230 1231static const struct mrs_field_value id_aa64mmfr2_st[] = { 1232 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, ST, NONE, IMPL), 1233 MRS_FIELD_VALUE_END, 1234}; 1235 1236static const struct mrs_field_value id_aa64mmfr2_nv[] = { 1237 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, NV, NONE, 8_3), 1238 MRS_FIELD_VALUE(ID_AA64MMFR2_NV_8_4, "NV v8.4"), 1239 MRS_FIELD_VALUE_END, 1240}; 1241 1242static const struct mrs_field_value id_aa64mmfr2_ccidx[] = { 1243 MRS_FIELD_VALUE(ID_AA64MMFR2_CCIDX_32, "32bit CCIDX"), 1244 MRS_FIELD_VALUE(ID_AA64MMFR2_CCIDX_64, "64bit CCIDX"), 1245 MRS_FIELD_VALUE_END, 1246}; 1247 1248static const struct mrs_field_value id_aa64mmfr2_varange[] = { 1249 MRS_FIELD_VALUE(ID_AA64MMFR2_VARange_48, "48bit VA"), 1250 MRS_FIELD_VALUE(ID_AA64MMFR2_VARange_52, "52bit VA"), 1251 MRS_FIELD_VALUE_END, 1252}; 1253 1254static const struct mrs_field_value id_aa64mmfr2_iesb[] = { 1255 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, IESB, NONE, IMPL), 1256 MRS_FIELD_VALUE_END, 1257}; 1258 1259static const struct mrs_field_value id_aa64mmfr2_lsm[] = { 1260 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, LSM, NONE, IMPL), 1261 MRS_FIELD_VALUE_END, 1262}; 1263 1264static const struct mrs_field_value id_aa64mmfr2_uao[] = { 1265 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, UAO, NONE, IMPL), 1266 MRS_FIELD_VALUE_END, 1267}; 1268 1269static const struct mrs_field_value id_aa64mmfr2_cnp[] = { 1270 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, CnP, NONE, IMPL), 1271 MRS_FIELD_VALUE_END, 1272}; 1273 1274static const struct mrs_field id_aa64mmfr2_fields[] = { 1275 MRS_FIELD(ID_AA64MMFR2, E0PD, false, MRS_EXACT, id_aa64mmfr2_e0pd), 1276 MRS_FIELD(ID_AA64MMFR2, EVT, false, MRS_EXACT, id_aa64mmfr2_evt), 1277 MRS_FIELD(ID_AA64MMFR2, BBM, false, MRS_EXACT, id_aa64mmfr2_bbm), 1278 MRS_FIELD(ID_AA64MMFR2, TTL, false, MRS_EXACT, id_aa64mmfr2_ttl), 1279 MRS_FIELD(ID_AA64MMFR2, FWB, false, MRS_EXACT, id_aa64mmfr2_fwb), 1280 MRS_FIELD(ID_AA64MMFR2, IDS, false, MRS_EXACT, id_aa64mmfr2_ids), 1281 MRS_FIELD_HWCAP(ID_AA64MMFR2, AT, false, MRS_LOWER, id_aa64mmfr2_at, 1282 id_aa64mmfr2_at_caps), 1283 MRS_FIELD(ID_AA64MMFR2, ST, false, MRS_EXACT, id_aa64mmfr2_st), 1284 MRS_FIELD(ID_AA64MMFR2, NV, false, MRS_EXACT, id_aa64mmfr2_nv), 1285 MRS_FIELD(ID_AA64MMFR2, CCIDX, false, MRS_EXACT, id_aa64mmfr2_ccidx), 1286 MRS_FIELD(ID_AA64MMFR2, VARange, false, MRS_EXACT, 1287 id_aa64mmfr2_varange), 1288 MRS_FIELD(ID_AA64MMFR2, IESB, false, MRS_EXACT, id_aa64mmfr2_iesb), 1289 MRS_FIELD(ID_AA64MMFR2, LSM, false, MRS_EXACT, id_aa64mmfr2_lsm), 1290 MRS_FIELD(ID_AA64MMFR2, UAO, false, MRS_EXACT, id_aa64mmfr2_uao), 1291 MRS_FIELD(ID_AA64MMFR2, CnP, false, MRS_EXACT, id_aa64mmfr2_cnp), 1292 MRS_FIELD_END, 1293}; 1294 1295 1296/* ID_AA64MMFR2_EL1 */ 1297static const struct mrs_field_value id_aa64mmfr3_spec_fpacc[] = { 1298 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, Spec_FPACC, NONE, IMPL), 1299 MRS_FIELD_VALUE_END, 1300}; 1301 1302static const struct mrs_field_value id_aa64mmfr3_mec[] = { 1303 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, MEC, NONE, IMPL), 1304 MRS_FIELD_VALUE_END, 1305}; 1306 1307static const struct mrs_field_value id_aa64mmfr3_sctlrx[] = { 1308 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, SCTLRX, NONE, IMPL), 1309 MRS_FIELD_VALUE_END, 1310}; 1311 1312static const struct mrs_field_value id_aa64mmfr3_tcrx[] = { 1313 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, TCRX, NONE, IMPL), 1314 MRS_FIELD_VALUE_END, 1315}; 1316 1317static const struct mrs_field id_aa64mmfr3_fields[] = { 1318 MRS_FIELD(ID_AA64MMFR3, Spec_FPACC, false, MRS_EXACT, 1319 id_aa64mmfr3_spec_fpacc), 1320 MRS_FIELD(ID_AA64MMFR3, MEC, false, MRS_EXACT, id_aa64mmfr3_mec), 1321 MRS_FIELD(ID_AA64MMFR3, SCTLRX, false, MRS_EXACT, id_aa64mmfr3_sctlrx), 1322 MRS_FIELD(ID_AA64MMFR3, TCRX, false, MRS_EXACT, id_aa64mmfr3_tcrx), 1323 MRS_FIELD_END, 1324}; 1325 1326 1327/* ID_AA64MMFR4_EL1 */ 1328static const struct mrs_field id_aa64mmfr4_fields[] = { 1329 MRS_FIELD_END, 1330}; 1331 1332 1333/* ID_AA64PFR0_EL1 */ 1334static const struct mrs_field_value id_aa64pfr0_csv3[] = { 1335 MRS_FIELD_VALUE(ID_AA64PFR0_CSV3_NONE, ""), 1336 MRS_FIELD_VALUE(ID_AA64PFR0_CSV3_ISOLATED, "CSV3"), 1337 MRS_FIELD_VALUE_END, 1338}; 1339 1340static const struct mrs_field_value id_aa64pfr0_csv2[] = { 1341 MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_NONE, ""), 1342 MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_ISOLATED, "CSV2"), 1343 MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_SCXTNUM, "CSV2_2"), 1344 MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_3, "CSV2_3"), 1345 MRS_FIELD_VALUE_END, 1346}; 1347 1348static const struct mrs_field_value id_aa64pfr0_rme[] = { 1349 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, RME, NONE, IMPL), 1350 MRS_FIELD_VALUE_END, 1351}; 1352 1353static const struct mrs_field_value id_aa64pfr0_dit[] = { 1354 MRS_FIELD_VALUE(ID_AA64PFR0_DIT_NONE, ""), 1355 MRS_FIELD_VALUE(ID_AA64PFR0_DIT_PSTATE, "PSTATE.DIT"), 1356 MRS_FIELD_VALUE_END, 1357}; 1358 1359static const struct mrs_field_hwcap id_aa64pfr0_dit_caps[] = { 1360 MRS_HWCAP(1, HWCAP_DIT, ID_AA64PFR0_DIT_PSTATE), 1361 MRS_HWCAP_END 1362}; 1363 1364static const struct mrs_field_value id_aa64pfr0_amu[] = { 1365 MRS_FIELD_VALUE(ID_AA64PFR0_AMU_NONE, ""), 1366 MRS_FIELD_VALUE(ID_AA64PFR0_AMU_V1, "AMUv1"), 1367 MRS_FIELD_VALUE(ID_AA64PFR0_AMU_V1_1, "AMUv1p1"), 1368 MRS_FIELD_VALUE_END, 1369}; 1370 1371static const struct mrs_field_value id_aa64pfr0_mpam[] = { 1372 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, MPAM, NONE, IMPL), 1373 MRS_FIELD_VALUE_END, 1374}; 1375 1376static const struct mrs_field_value id_aa64pfr0_sel2[] = { 1377 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, SEL2, NONE, IMPL), 1378 MRS_FIELD_VALUE_END, 1379}; 1380 1381static const struct mrs_field_value id_aa64pfr0_sve[] = { 1382 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, SVE, NONE, IMPL), 1383 MRS_FIELD_VALUE_END, 1384}; 1385 1386#if 0 1387/* Enable when we add SVE support */ 1388static const struct mrs_field_hwcap id_aa64pfr0_sve_caps[] = { 1389 MRS_HWCAP(1, HWCAP_SVE, ID_AA64PFR0_SVE_IMPL), 1390 MRS_HWCAP_END 1391}; 1392#endif 1393 1394static const struct mrs_field_value id_aa64pfr0_ras[] = { 1395 MRS_FIELD_VALUE(ID_AA64PFR0_RAS_NONE, ""), 1396 MRS_FIELD_VALUE(ID_AA64PFR0_RAS_IMPL, "RAS"), 1397 MRS_FIELD_VALUE(ID_AA64PFR0_RAS_8_4, "RAS v8.4"), 1398 MRS_FIELD_VALUE_END, 1399}; 1400 1401static const struct mrs_field_value id_aa64pfr0_gic[] = { 1402 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, GIC, CPUIF_NONE, CPUIF_EN), 1403 MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_NONE, ""), 1404 MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_EN, "GIC"), 1405 MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_4_1, "GIC 4.1"), 1406 MRS_FIELD_VALUE_END, 1407}; 1408 1409static const struct mrs_field_value id_aa64pfr0_advsimd[] = { 1410 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, AdvSIMD, NONE, IMPL), 1411 MRS_FIELD_VALUE(ID_AA64PFR0_AdvSIMD_HP, "AdvSIMD+HP"), 1412 MRS_FIELD_VALUE_END, 1413}; 1414 1415static const struct mrs_field_hwcap id_aa64pfr0_advsimd_caps[] = { 1416 MRS_HWCAP(1, HWCAP_ASIMD, ID_AA64PFR0_AdvSIMD_IMPL), 1417 MRS_HWCAP(1, HWCAP_ASIMDHP, ID_AA64PFR0_AdvSIMD_HP), 1418 MRS_HWCAP_END 1419}; 1420 1421static const struct mrs_field_value id_aa64pfr0_fp[] = { 1422 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, FP, NONE, IMPL), 1423 MRS_FIELD_VALUE(ID_AA64PFR0_FP_HP, "FP+HP"), 1424 MRS_FIELD_VALUE_END, 1425}; 1426 1427static const struct mrs_field_hwcap id_aa64pfr0_fp_caps[] = { 1428 MRS_HWCAP(1, HWCAP_FP, ID_AA64PFR0_FP_IMPL), 1429 MRS_HWCAP(1, HWCAP_FPHP, ID_AA64PFR0_FP_HP), 1430 MRS_HWCAP_END 1431}; 1432 1433static const struct mrs_field_value id_aa64pfr0_el3[] = { 1434 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, EL3, NONE, 64), 1435 MRS_FIELD_VALUE(ID_AA64PFR0_EL3_64_32, "EL3 32"), 1436 MRS_FIELD_VALUE_END, 1437}; 1438 1439static const struct mrs_field_value id_aa64pfr0_el2[] = { 1440 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, EL2, NONE, 64), 1441 MRS_FIELD_VALUE(ID_AA64PFR0_EL2_64_32, "EL2 32"), 1442 MRS_FIELD_VALUE_END, 1443}; 1444 1445static const struct mrs_field_value id_aa64pfr0_el1[] = { 1446 MRS_FIELD_VALUE(ID_AA64PFR0_EL1_64, "EL1"), 1447 MRS_FIELD_VALUE(ID_AA64PFR0_EL1_64_32, "EL1 32"), 1448 MRS_FIELD_VALUE_END, 1449}; 1450 1451static const struct mrs_field_value id_aa64pfr0_el0[] = { 1452 MRS_FIELD_VALUE(ID_AA64PFR0_EL0_64, "EL0"), 1453 MRS_FIELD_VALUE(ID_AA64PFR0_EL0_64_32, "EL0 32"), 1454 MRS_FIELD_VALUE_END, 1455}; 1456 1457static const struct mrs_field id_aa64pfr0_fields[] = { 1458 MRS_FIELD(ID_AA64PFR0, CSV3, false, MRS_EXACT, id_aa64pfr0_csv3), 1459 MRS_FIELD(ID_AA64PFR0, CSV2, false, MRS_EXACT, id_aa64pfr0_csv2), 1460 MRS_FIELD(ID_AA64PFR0, RME, false, MRS_EXACT, id_aa64pfr0_rme), 1461 MRS_FIELD_HWCAP(ID_AA64PFR0, DIT, false, MRS_LOWER, id_aa64pfr0_dit, 1462 id_aa64pfr0_dit_caps), 1463 MRS_FIELD(ID_AA64PFR0, AMU, false, MRS_EXACT, id_aa64pfr0_amu), 1464 MRS_FIELD(ID_AA64PFR0, MPAM, false, MRS_EXACT, id_aa64pfr0_mpam), 1465 MRS_FIELD(ID_AA64PFR0, SEL2, false, MRS_EXACT, id_aa64pfr0_sel2), 1466 MRS_FIELD(ID_AA64PFR0, SVE, false, MRS_EXACT, id_aa64pfr0_sve), 1467 MRS_FIELD(ID_AA64PFR0, RAS, false, MRS_EXACT, id_aa64pfr0_ras), 1468 MRS_FIELD(ID_AA64PFR0, GIC, false, MRS_EXACT, id_aa64pfr0_gic), 1469 MRS_FIELD_HWCAP(ID_AA64PFR0, AdvSIMD, true, MRS_LOWER, 1470 id_aa64pfr0_advsimd, id_aa64pfr0_advsimd_caps), 1471 MRS_FIELD_HWCAP(ID_AA64PFR0, FP, true, MRS_LOWER, id_aa64pfr0_fp, 1472 id_aa64pfr0_fp_caps), 1473 MRS_FIELD(ID_AA64PFR0, EL3, false, MRS_EXACT, id_aa64pfr0_el3), 1474 MRS_FIELD(ID_AA64PFR0, EL2, false, MRS_EXACT, id_aa64pfr0_el2), 1475 MRS_FIELD(ID_AA64PFR0, EL1, false, MRS_LOWER, id_aa64pfr0_el1), 1476 MRS_FIELD(ID_AA64PFR0, EL0, false, MRS_LOWER, id_aa64pfr0_el0), 1477 MRS_FIELD_END, 1478}; 1479 1480 1481/* ID_AA64PFR1_EL1 */ 1482static const struct mrs_field_value id_aa64pfr1_nmi[] = { 1483 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR1, NMI, NONE, IMPL), 1484 MRS_FIELD_VALUE_END, 1485}; 1486 1487static const struct mrs_field_value id_aa64pfr1_csv2_frac[] = { 1488 MRS_FIELD_VALUE(ID_AA64PFR1_CSV2_frac_p0, ""), 1489 MRS_FIELD_VALUE(ID_AA64PFR1_CSV2_frac_p1, "CSV2 p1"), 1490 MRS_FIELD_VALUE(ID_AA64PFR1_CSV2_frac_p2, "CSV2 p2"), 1491 MRS_FIELD_VALUE_END, 1492}; 1493 1494static const struct mrs_field_value id_aa64pfr1_rndr_trap[] = { 1495 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR1, RNDR_trap, NONE, IMPL), 1496 MRS_FIELD_VALUE_END, 1497}; 1498 1499static const struct mrs_field_value id_aa64pfr1_sme[] = { 1500 MRS_FIELD_VALUE(ID_AA64PFR1_SME_NONE, ""), 1501 MRS_FIELD_VALUE(ID_AA64PFR1_SME_SME, "SME"), 1502 MRS_FIELD_VALUE(ID_AA64PFR1_SME_SME2, "SME2"), 1503 MRS_FIELD_VALUE_END, 1504}; 1505 1506static const struct mrs_field_value id_aa64pfr1_mpam_frac[] = { 1507 MRS_FIELD_VALUE(ID_AA64PFR1_MPAM_frac_p0, ""), 1508 MRS_FIELD_VALUE(ID_AA64PFR1_MPAM_frac_p1, "MPAM p1"), 1509 MRS_FIELD_VALUE_END, 1510}; 1511 1512static const struct mrs_field_value id_aa64pfr1_ras_frac[] = { 1513 MRS_FIELD_VALUE(ID_AA64PFR1_RAS_frac_p0, ""), 1514 MRS_FIELD_VALUE(ID_AA64PFR1_RAS_frac_p1, "RAS p1"), 1515 MRS_FIELD_VALUE_END, 1516}; 1517 1518static const struct mrs_field_value id_aa64pfr1_mte[] = { 1519 MRS_FIELD_VALUE(ID_AA64PFR1_MTE_NONE, ""), 1520 MRS_FIELD_VALUE(ID_AA64PFR1_MTE_MTE, "MTE"), 1521 MRS_FIELD_VALUE(ID_AA64PFR1_MTE_MTE2, "MTE2"), 1522 MRS_FIELD_VALUE(ID_AA64PFR1_MTE_MTE3, "MTE3"), 1523 MRS_FIELD_VALUE_END, 1524}; 1525 1526static const struct mrs_field_value id_aa64pfr1_ssbs[] = { 1527 MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_NONE, ""), 1528 MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_PSTATE, "PSTATE.SSBS"), 1529 MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_PSTATE_MSR, "PSTATE.SSBS MSR"), 1530 MRS_FIELD_VALUE_END, 1531}; 1532 1533static const struct mrs_field_hwcap id_aa64pfr1_ssbs_caps[] = { 1534 MRS_HWCAP(1, HWCAP_SSBS, ID_AA64PFR1_SSBS_PSTATE), 1535 MRS_HWCAP_END 1536}; 1537 1538static const struct mrs_field_value id_aa64pfr1_bt[] = { 1539 MRS_FIELD_VALUE(ID_AA64PFR1_BT_NONE, ""), 1540 MRS_FIELD_VALUE(ID_AA64PFR1_BT_IMPL, "BTI"), 1541 MRS_FIELD_VALUE_END, 1542}; 1543 1544static const struct mrs_field_hwcap id_aa64pfr1_bt_caps[] = { 1545 MRS_HWCAP(2, HWCAP2_BTI, ID_AA64PFR1_BT_IMPL), 1546 MRS_HWCAP_END 1547}; 1548 1549static const struct mrs_field id_aa64pfr1_fields[] = { 1550 MRS_FIELD(ID_AA64PFR1, NMI, false, MRS_EXACT, id_aa64pfr1_nmi), 1551 MRS_FIELD(ID_AA64PFR1, CSV2_frac, false, MRS_EXACT, 1552 id_aa64pfr1_csv2_frac), 1553 MRS_FIELD(ID_AA64PFR1, RNDR_trap, false, MRS_EXACT, 1554 id_aa64pfr1_rndr_trap), 1555 MRS_FIELD(ID_AA64PFR1, SME, false, MRS_EXACT, id_aa64pfr1_sme), 1556 MRS_FIELD(ID_AA64PFR1, MPAM_frac, false, MRS_EXACT, 1557 id_aa64pfr1_mpam_frac), 1558 MRS_FIELD(ID_AA64PFR1, RAS_frac, false, MRS_EXACT, 1559 id_aa64pfr1_ras_frac), 1560 MRS_FIELD(ID_AA64PFR1, MTE, false, MRS_EXACT, id_aa64pfr1_mte), 1561 MRS_FIELD_HWCAP(ID_AA64PFR1, SSBS, false, MRS_LOWER, id_aa64pfr1_ssbs, 1562 id_aa64pfr1_ssbs_caps), 1563 MRS_FIELD_HWCAP_SPLIT(ID_AA64PFR1, BT, false, MRS_LOWER, MRS_EXACT, 1564 id_aa64pfr1_bt, id_aa64pfr1_bt_caps), 1565 MRS_FIELD_END, 1566}; 1567 1568 1569/* ID_AA64PFR2_EL1 */ 1570static const struct mrs_field id_aa64pfr2_fields[] = { 1571 MRS_FIELD_END, 1572}; 1573 1574 1575/* ID_AA64ZFR0_EL1 */ 1576static const struct mrs_field_value id_aa64zfr0_f64mm[] = { 1577 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, F64MM, NONE, IMPL), 1578 MRS_FIELD_VALUE_END, 1579}; 1580 1581static const struct mrs_field_value id_aa64zfr0_f32mm[] = { 1582 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, F32MM, NONE, IMPL), 1583 MRS_FIELD_VALUE_END, 1584}; 1585 1586static const struct mrs_field_value id_aa64zfr0_i8mm[] = { 1587 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, I8MM, NONE, IMPL), 1588 MRS_FIELD_VALUE_END, 1589}; 1590 1591static const struct mrs_field_value id_aa64zfr0_sm4[] = { 1592 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, SM4, NONE, IMPL), 1593 MRS_FIELD_VALUE_END, 1594}; 1595 1596static const struct mrs_field_value id_aa64zfr0_sha3[] = { 1597 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, SHA3, NONE, IMPL), 1598 MRS_FIELD_VALUE_END, 1599}; 1600 1601static const struct mrs_field_value id_aa64zfr0_bf16[] = { 1602 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, BF16, NONE, BASE), 1603 MRS_FIELD_VALUE(ID_AA64ZFR0_BF16_EBF, "BF16+EBF"), 1604 MRS_FIELD_VALUE_END, 1605}; 1606 1607static const struct mrs_field_value id_aa64zfr0_bitperm[] = { 1608 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, BitPerm, NONE, IMPL), 1609 MRS_FIELD_VALUE_END, 1610}; 1611 1612static const struct mrs_field_value id_aa64zfr0_aes[] = { 1613 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, AES, NONE, BASE), 1614 MRS_FIELD_VALUE(ID_AA64ZFR0_AES_PMULL, "AES+PMULL"), 1615 MRS_FIELD_VALUE_END, 1616}; 1617 1618static const struct mrs_field_value id_aa64zfr0_svever[] = { 1619 MRS_FIELD_VALUE(ID_AA64ZFR0_SVEver_SVE1, "SVE1"), 1620 MRS_FIELD_VALUE(ID_AA64ZFR0_SVEver_SVE2, "SVE2"), 1621 MRS_FIELD_VALUE_END, 1622}; 1623 1624static const struct mrs_field id_aa64zfr0_fields[] = { 1625 MRS_FIELD(ID_AA64ZFR0, F64MM, false, MRS_EXACT, id_aa64zfr0_f64mm), 1626 MRS_FIELD(ID_AA64ZFR0, F32MM, false, MRS_EXACT, id_aa64zfr0_f32mm), 1627 MRS_FIELD(ID_AA64ZFR0, I8MM, false, MRS_EXACT, id_aa64zfr0_i8mm), 1628 MRS_FIELD(ID_AA64ZFR0, SM4, false, MRS_EXACT, id_aa64zfr0_sm4), 1629 MRS_FIELD(ID_AA64ZFR0, SHA3, false, MRS_EXACT, id_aa64zfr0_sha3), 1630 MRS_FIELD(ID_AA64ZFR0, BF16, false, MRS_EXACT, id_aa64zfr0_bf16), 1631 MRS_FIELD(ID_AA64ZFR0, BitPerm, false, MRS_EXACT, id_aa64zfr0_bitperm), 1632 MRS_FIELD(ID_AA64ZFR0, AES, false, MRS_EXACT, id_aa64zfr0_aes), 1633 MRS_FIELD(ID_AA64ZFR0, SVEver, false, MRS_EXACT, id_aa64zfr0_svever), 1634 MRS_FIELD_END, 1635}; 1636 1637 1638#ifdef COMPAT_FREEBSD32 1639/* ID_ISAR5_EL1 */ 1640static const struct mrs_field_value id_isar5_vcma[] = { 1641 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, VCMA, NONE, IMPL), 1642 MRS_FIELD_VALUE_END, 1643}; 1644 1645static const struct mrs_field_value id_isar5_rdm[] = { 1646 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, RDM, NONE, IMPL), 1647 MRS_FIELD_VALUE_END, 1648}; 1649 1650static const struct mrs_field_value id_isar5_crc32[] = { 1651 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, CRC32, NONE, IMPL), 1652 MRS_FIELD_VALUE_END, 1653}; 1654 1655static const struct mrs_field_hwcap id_isar5_crc32_caps[] = { 1656 MRS_HWCAP(2, HWCAP32_2_CRC32, ID_ISAR5_CRC32_IMPL), 1657 MRS_HWCAP_END 1658}; 1659 1660static const struct mrs_field_value id_isar5_sha2[] = { 1661 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SHA2, NONE, IMPL), 1662 MRS_FIELD_VALUE_END, 1663}; 1664 1665static const struct mrs_field_hwcap id_isar5_sha2_caps[] = { 1666 MRS_HWCAP(2, HWCAP32_2_SHA2, ID_ISAR5_SHA2_IMPL), 1667 MRS_HWCAP_END 1668}; 1669 1670static const struct mrs_field_value id_isar5_sha1[] = { 1671 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SHA1, NONE, IMPL), 1672 MRS_FIELD_VALUE_END, 1673}; 1674 1675static const struct mrs_field_hwcap id_isar5_sha1_caps[] = { 1676 MRS_HWCAP(2, HWCAP32_2_SHA1, ID_ISAR5_SHA1_IMPL), 1677 MRS_HWCAP_END 1678}; 1679 1680static const struct mrs_field_value id_isar5_aes[] = { 1681 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, AES, NONE, BASE), 1682 MRS_FIELD_VALUE(ID_ISAR5_AES_VMULL, "AES+VMULL"), 1683 MRS_FIELD_VALUE_END, 1684}; 1685 1686static const struct mrs_field_hwcap id_isar5_aes_caps[] = { 1687 MRS_HWCAP(2, HWCAP32_2_AES, ID_ISAR5_AES_BASE), 1688 MRS_HWCAP(2, HWCAP32_2_PMULL, ID_ISAR5_AES_VMULL), 1689 MRS_HWCAP_END 1690}; 1691 1692static const struct mrs_field_value id_isar5_sevl[] = { 1693 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SEVL, NOP, IMPL), 1694 MRS_FIELD_VALUE_END, 1695}; 1696 1697static const struct mrs_field id_isar5_fields[] = { 1698 MRS_FIELD(ID_ISAR5, VCMA, false, MRS_LOWER, id_isar5_vcma), 1699 MRS_FIELD(ID_ISAR5, RDM, false, MRS_LOWER, id_isar5_rdm), 1700 MRS_FIELD_HWCAP(ID_ISAR5, CRC32, false, MRS_LOWER, id_isar5_crc32, 1701 id_isar5_crc32_caps), 1702 MRS_FIELD_HWCAP(ID_ISAR5, SHA2, false, MRS_LOWER, id_isar5_sha2, 1703 id_isar5_sha2_caps), 1704 MRS_FIELD_HWCAP(ID_ISAR5, SHA1, false, MRS_LOWER, id_isar5_sha1, 1705 id_isar5_sha1_caps), 1706 MRS_FIELD_HWCAP(ID_ISAR5, AES, false, MRS_LOWER, id_isar5_aes, 1707 id_isar5_aes_caps), 1708 MRS_FIELD(ID_ISAR5, SEVL, false, MRS_LOWER, id_isar5_sevl), 1709 MRS_FIELD_END, 1710}; 1711 1712/* MVFR0 */ 1713static const struct mrs_field_value mvfr0_fpround[] = { 1714 MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPRound, NONE, IMPL), 1715 MRS_FIELD_VALUE_END, 1716}; 1717 1718static const struct mrs_field_value mvfr0_fpsqrt[] = { 1719 MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPSqrt, NONE, IMPL), 1720 MRS_FIELD_VALUE_END, 1721}; 1722 1723static const struct mrs_field_value mvfr0_fpdivide[] = { 1724 MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPDivide, NONE, IMPL), 1725 MRS_FIELD_VALUE_END, 1726}; 1727 1728static const struct mrs_field_value mvfr0_fptrap[] = { 1729 MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPTrap, NONE, IMPL), 1730 MRS_FIELD_VALUE_END, 1731}; 1732 1733static const struct mrs_field_value mvfr0_fpdp[] = { 1734 MRS_FIELD_VALUE(MVFR0_FPDP_NONE, ""), 1735 MRS_FIELD_VALUE(MVFR0_FPDP_VFP_v2, "DP VFPv2"), 1736 MRS_FIELD_VALUE(MVFR0_FPDP_VFP_v3_v4, "DP VFPv3+v4"), 1737 MRS_FIELD_VALUE_END, 1738}; 1739 1740static const struct mrs_field_hwcap mvfr0_fpdp_caps[] = { 1741 MRS_HWCAP(1, HWCAP32_VFP, MVFR0_FPDP_VFP_v2), 1742 MRS_HWCAP(1, HWCAP32_VFPv3, MVFR0_FPDP_VFP_v3_v4), 1743 MRS_HWCAP_END 1744}; 1745 1746static const struct mrs_field_value mvfr0_fpsp[] = { 1747 MRS_FIELD_VALUE(MVFR0_FPSP_NONE, ""), 1748 MRS_FIELD_VALUE(MVFR0_FPSP_VFP_v2, "SP VFPv2"), 1749 MRS_FIELD_VALUE(MVFR0_FPSP_VFP_v3_v4, "SP VFPv3+v4"), 1750 MRS_FIELD_VALUE_END, 1751}; 1752 1753static const struct mrs_field_value mvfr0_simdreg[] = { 1754 MRS_FIELD_VALUE(MVFR0_SIMDReg_NONE, ""), 1755 MRS_FIELD_VALUE(MVFR0_SIMDReg_FP, "FP 16x64"), 1756 MRS_FIELD_VALUE(MVFR0_SIMDReg_AdvSIMD, "AdvSIMD"), 1757 MRS_FIELD_VALUE_END, 1758}; 1759 1760static const struct mrs_field mvfr0_fields[] = { 1761 MRS_FIELD(MVFR0, FPRound, false, MRS_LOWER, mvfr0_fpround), 1762 MRS_FIELD(MVFR0, FPSqrt, false, MRS_LOWER, mvfr0_fpsqrt), 1763 MRS_FIELD(MVFR0, FPDivide, false, MRS_LOWER, mvfr0_fpdivide), 1764 MRS_FIELD(MVFR0, FPTrap, false, MRS_LOWER, mvfr0_fptrap), 1765 MRS_FIELD_HWCAP(MVFR0, FPDP, false, MRS_LOWER, mvfr0_fpdp, 1766 mvfr0_fpdp_caps), 1767 MRS_FIELD(MVFR0, FPSP, false, MRS_LOWER, mvfr0_fpsp), 1768 MRS_FIELD(MVFR0, SIMDReg, false, MRS_LOWER, mvfr0_simdreg), 1769 MRS_FIELD_END, 1770}; 1771 1772/* MVFR1 */ 1773static const struct mrs_field_value mvfr1_simdfmac[] = { 1774 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDFMAC, NONE, IMPL), 1775 MRS_FIELD_VALUE_END, 1776}; 1777 1778static const struct mrs_field_hwcap mvfr1_simdfmac_caps[] = { 1779 MRS_HWCAP(1, HWCAP32_VFPv4, MVFR1_SIMDFMAC_IMPL), 1780 MRS_HWCAP_END 1781}; 1782 1783static const struct mrs_field_value mvfr1_fphp[] = { 1784 MRS_FIELD_VALUE(MVFR1_FPHP_NONE, ""), 1785 MRS_FIELD_VALUE(MVFR1_FPHP_CONV_SP, "FPHP SP Conv"), 1786 MRS_FIELD_VALUE(MVFR1_FPHP_CONV_DP, "FPHP DP Conv"), 1787 MRS_FIELD_VALUE(MVFR1_FPHP_ARITH, "FPHP Arith"), 1788 MRS_FIELD_VALUE_END, 1789}; 1790 1791static const struct mrs_field_value mvfr1_simdhp[] = { 1792 MRS_FIELD_VALUE(MVFR1_SIMDHP_NONE, ""), 1793 MRS_FIELD_VALUE(MVFR1_SIMDHP_CONV_SP, "SIMDHP SP Conv"), 1794 MRS_FIELD_VALUE(MVFR1_SIMDHP_ARITH, "SIMDHP Arith"), 1795 MRS_FIELD_VALUE_END, 1796}; 1797 1798static const struct mrs_field_value mvfr1_simdsp[] = { 1799 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDSP, NONE, IMPL), 1800 MRS_FIELD_VALUE_END, 1801}; 1802 1803static const struct mrs_field_value mvfr1_simdint[] = { 1804 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDInt, NONE, IMPL), 1805 MRS_FIELD_VALUE_END, 1806}; 1807 1808static const struct mrs_field_value mvfr1_simdls[] = { 1809 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDLS, NONE, IMPL), 1810 MRS_FIELD_VALUE_END, 1811}; 1812 1813static const struct mrs_field_hwcap mvfr1_simdls_caps[] = { 1814 MRS_HWCAP(1, HWCAP32_VFPv4, MVFR1_SIMDFMAC_IMPL), 1815 MRS_HWCAP_END 1816}; 1817 1818static const struct mrs_field_value mvfr1_fpdnan[] = { 1819 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, FPDNaN, NONE, IMPL), 1820 MRS_FIELD_VALUE_END, 1821}; 1822 1823static const struct mrs_field_value mvfr1_fpftz[] = { 1824 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, FPFtZ, NONE, IMPL), 1825 MRS_FIELD_VALUE_END, 1826}; 1827 1828static const struct mrs_field mvfr1_fields[] = { 1829 MRS_FIELD_HWCAP(MVFR1, SIMDFMAC, false, MRS_LOWER, mvfr1_simdfmac, 1830 mvfr1_simdfmac_caps), 1831 MRS_FIELD(MVFR1, FPHP, false, MRS_LOWER, mvfr1_fphp), 1832 MRS_FIELD(MVFR1, SIMDHP, false, MRS_LOWER, mvfr1_simdhp), 1833 MRS_FIELD(MVFR1, SIMDSP, false, MRS_LOWER, mvfr1_simdsp), 1834 MRS_FIELD(MVFR1, SIMDInt, false, MRS_LOWER, mvfr1_simdint), 1835 MRS_FIELD_HWCAP(MVFR1, SIMDLS, false, MRS_LOWER, mvfr1_simdls, 1836 mvfr1_simdls_caps), 1837 MRS_FIELD(MVFR1, FPDNaN, false, MRS_LOWER, mvfr1_fpdnan), 1838 MRS_FIELD(MVFR1, FPFtZ, false, MRS_LOWER, mvfr1_fpftz), 1839 MRS_FIELD_END, 1840}; 1841#endif /* COMPAT_FREEBSD32 */ 1842 1843struct mrs_user_reg { 1844 u_int reg; 1845 u_int CRm; 1846 u_int Op2; 1847 bool is64bit; 1848 size_t offset; 1849 const struct mrs_field *fields; 1850}; 1851 1852#define USER_REG(name, field_name, _is64bit) \ 1853 { \ 1854 .reg = name, \ 1855 .CRm = name##_CRm, \ 1856 .Op2 = name##_op2, \ 1857 .offset = __offsetof(struct cpu_desc, field_name), \ 1858 .fields = field_name##_fields, \ 1859 .is64bit = _is64bit, \ 1860 } 1861static const struct mrs_user_reg user_regs[] = { 1862 USER_REG(ID_AA64AFR0_EL1, id_aa64afr0, true), 1863 USER_REG(ID_AA64AFR1_EL1, id_aa64afr1, true), 1864 1865 USER_REG(ID_AA64DFR0_EL1, id_aa64dfr0, true), 1866 USER_REG(ID_AA64DFR1_EL1, id_aa64dfr1, true), 1867 1868 USER_REG(ID_AA64ISAR0_EL1, id_aa64isar0, true), 1869 USER_REG(ID_AA64ISAR1_EL1, id_aa64isar1, true), 1870 USER_REG(ID_AA64ISAR2_EL1, id_aa64isar2, true), 1871 1872 USER_REG(ID_AA64MMFR0_EL1, id_aa64mmfr0, true), 1873 USER_REG(ID_AA64MMFR1_EL1, id_aa64mmfr1, true), 1874 USER_REG(ID_AA64MMFR2_EL1, id_aa64mmfr2, true), 1875 USER_REG(ID_AA64MMFR3_EL1, id_aa64mmfr3, true), 1876 USER_REG(ID_AA64MMFR4_EL1, id_aa64mmfr4, true), 1877 1878 USER_REG(ID_AA64PFR0_EL1, id_aa64pfr0, true), 1879 USER_REG(ID_AA64PFR1_EL1, id_aa64pfr1, true), 1880 USER_REG(ID_AA64PFR2_EL1, id_aa64pfr2, true), 1881 1882 USER_REG(ID_AA64ZFR0_EL1, id_aa64zfr0, true), 1883 1884#ifdef COMPAT_FREEBSD32 1885 USER_REG(ID_ISAR5_EL1, id_isar5, false), 1886 1887 USER_REG(MVFR0_EL1, mvfr0, false), 1888 USER_REG(MVFR1_EL1, mvfr1, false), 1889#endif /* COMPAT_FREEBSD32 */ 1890}; 1891 1892#define CPU_DESC_FIELD(desc, idx) \ 1893 *(uint64_t *)((char *)&(desc) + user_regs[(idx)].offset) 1894 1895static int 1896user_mrs_handler(vm_offset_t va, uint32_t insn, struct trapframe *frame, 1897 uint32_t esr) 1898{ 1899 uint64_t value; 1900 int CRm, Op2, i, reg; 1901 1902 if ((insn & MRS_MASK) != MRS_VALUE) 1903 return (0); 1904 1905 /* 1906 * We only emulate Op0 == 3, Op1 == 0, CRn == 0, CRm == {0, 4-7}. 1907 * These are in the EL1 CPU identification space. 1908 * CRm == 0 holds MIDR_EL1, MPIDR_EL1, and REVID_EL1. 1909 * CRm == {4-7} holds the ID_AA64 registers. 1910 * 1911 * For full details see the ARMv8 ARM (ARM DDI 0487C.a) 1912 * Table D9-2 System instruction encodings for non-Debug System 1913 * register accesses. 1914 */ 1915 if (mrs_Op0(insn) != 3 || mrs_Op1(insn) != 0 || mrs_CRn(insn) != 0) 1916 return (0); 1917 1918 CRm = mrs_CRm(insn); 1919 if (CRm > 7 || (CRm < 4 && CRm != 0)) 1920 return (0); 1921 1922 Op2 = mrs_Op2(insn); 1923 value = 0; 1924 1925 for (i = 0; i < nitems(user_regs); i++) { 1926 if (user_regs[i].CRm == CRm && user_regs[i].Op2 == Op2) { 1927 if (SV_CURPROC_ABI() == SV_ABI_FREEBSD) 1928 value = CPU_DESC_FIELD(user_cpu_desc, i); 1929 else 1930 value = CPU_DESC_FIELD(l_user_cpu_desc, i); 1931 break; 1932 } 1933 } 1934 1935 if (CRm == 0) { 1936 switch (Op2) { 1937 case 0: 1938 value = READ_SPECIALREG(midr_el1); 1939 break; 1940 case 5: 1941 value = READ_SPECIALREG(mpidr_el1); 1942 break; 1943 case 6: 1944 value = READ_SPECIALREG(revidr_el1); 1945 break; 1946 default: 1947 return (0); 1948 } 1949 } 1950 1951 /* 1952 * We will handle this instruction, move to the next so we 1953 * don't trap here again. 1954 */ 1955 frame->tf_elr += INSN_SIZE; 1956 1957 reg = MRS_REGISTER(insn); 1958 /* If reg is 31 then write to xzr, i.e. do nothing */ 1959 if (reg == 31) 1960 return (1); 1961 1962 if (reg < nitems(frame->tf_x)) 1963 frame->tf_x[reg] = value; 1964 else if (reg == 30) 1965 frame->tf_lr = value; 1966 1967 return (1); 1968} 1969 1970/* 1971 * Compares two field values that may be signed or unsigned. 1972 * Returns: 1973 * < 0 when a is less than b 1974 * = 0 when a equals b 1975 * > 0 when a is greater than b 1976 */ 1977static int 1978mrs_field_cmp(uint64_t a, uint64_t b, u_int shift, int width, bool sign) 1979{ 1980 uint64_t mask; 1981 1982 KASSERT(width > 0 && width < 64, ("%s: Invalid width %d", __func__, 1983 width)); 1984 1985 mask = (1ul << width) - 1; 1986 /* Move the field to the lower bits */ 1987 a = (a >> shift) & mask; 1988 b = (b >> shift) & mask; 1989 1990 if (sign) { 1991 /* 1992 * The field is signed. Toggle the upper bit so the comparison 1993 * works on unsigned values as this makes positive numbers, 1994 * i.e. those with a 0 bit, larger than negative numbers, 1995 * i.e. those with a 1 bit, in an unsigned comparison. 1996 */ 1997 a ^= 1ul << (width - 1); 1998 b ^= 1ul << (width - 1); 1999 } 2000 2001 return (a - b); 2002} 2003 2004static uint64_t 2005update_lower_register(uint64_t val, uint64_t new_val, u_int shift, 2006 int width, bool sign) 2007{ 2008 uint64_t mask; 2009 2010 KASSERT(width > 0 && width < 64, ("%s: Invalid width %d", __func__, 2011 width)); 2012 2013 /* 2014 * If the new value is less than the existing value update it. 2015 */ 2016 if (mrs_field_cmp(new_val, val, shift, width, sign) < 0) { 2017 mask = (1ul << width) - 1; 2018 val &= ~(mask << shift); 2019 val |= new_val & (mask << shift); 2020 } 2021 2022 return (val); 2023} 2024 2025bool 2026extract_user_id_field(u_int reg, u_int field_shift, uint8_t *val) 2027{ 2028 uint64_t value; 2029 int i; 2030 2031 for (i = 0; i < nitems(user_regs); i++) { 2032 if (user_regs[i].reg == reg) { 2033 value = CPU_DESC_FIELD(user_cpu_desc, i); 2034 *val = value >> field_shift; 2035 return (true); 2036 } 2037 } 2038 2039 return (false); 2040} 2041 2042bool 2043get_kernel_reg(u_int reg, uint64_t *val) 2044{ 2045 int i; 2046 2047 for (i = 0; i < nitems(user_regs); i++) { 2048 if (user_regs[i].reg == reg) { 2049 *val = CPU_DESC_FIELD(kern_cpu_desc, i); 2050 return (true); 2051 } 2052 } 2053 2054 return (false); 2055} 2056 2057/* 2058 * Fetch the specified register's value, ensuring that individual field values 2059 * do not exceed those in the mask. 2060 */ 2061bool 2062get_kernel_reg_masked(u_int reg, uint64_t *valp, uint64_t mask) 2063{ 2064 const struct mrs_field *fields; 2065 uint64_t val; 2066 2067 for (int i = 0; i < nitems(user_regs); i++) { 2068 if (user_regs[i].reg == reg) { 2069 val = CPU_DESC_FIELD(kern_cpu_desc, i); 2070 fields = user_regs[i].fields; 2071 for (int j = 0; fields[j].type != 0; j++) { 2072 mask = update_lower_register(mask, val, 2073 fields[j].shift, 4, fields[j].sign); 2074 } 2075 *valp = mask; 2076 return (true); 2077 } 2078 } 2079 2080 return (false); 2081} 2082 2083static uint64_t 2084update_special_reg_field(uint64_t user_reg, u_int type, uint64_t value, 2085 u_int shift, bool sign) 2086{ 2087 switch (type & MRS_TYPE_MASK) { 2088 case MRS_EXACT: 2089 user_reg &= ~(0xful << shift); 2090 user_reg |= (uint64_t)MRS_EXACT_FIELD(type) << shift; 2091 break; 2092 case MRS_LOWER: 2093 user_reg = update_lower_register(user_reg, value, shift, 4, 2094 sign); 2095 break; 2096 default: 2097 panic("Invalid field type: %d", type); 2098 } 2099 2100 return (user_reg); 2101} 2102 2103void 2104update_special_regs(u_int cpu) 2105{ 2106 struct cpu_desc *desc; 2107 const struct mrs_field *fields; 2108 uint64_t l_user_reg, user_reg, kern_reg, value; 2109 int i, j; 2110 2111 if (cpu == 0) { 2112 /* Create a user visible cpu description with safe values */ 2113 memset(&user_cpu_desc, 0, sizeof(user_cpu_desc)); 2114 /* Safe values for these registers */ 2115 user_cpu_desc.id_aa64pfr0 = ID_AA64PFR0_AdvSIMD_NONE | 2116 ID_AA64PFR0_FP_NONE | ID_AA64PFR0_EL1_64 | 2117 ID_AA64PFR0_EL0_64; 2118 user_cpu_desc.id_aa64dfr0 = ID_AA64DFR0_DebugVer_8; 2119 /* Create the Linux user visible cpu description */ 2120 memcpy(&l_user_cpu_desc, &user_cpu_desc, sizeof(user_cpu_desc)); 2121 } 2122 2123 desc = get_cpu_desc(cpu); 2124 for (i = 0; i < nitems(user_regs); i++) { 2125 value = CPU_DESC_FIELD(*desc, i); 2126 if (cpu == 0) { 2127 kern_reg = value; 2128 user_reg = value; 2129 l_user_reg = value; 2130 } else { 2131 kern_reg = CPU_DESC_FIELD(kern_cpu_desc, i); 2132 user_reg = CPU_DESC_FIELD(user_cpu_desc, i); 2133 l_user_reg = CPU_DESC_FIELD(l_user_cpu_desc, i); 2134 } 2135 2136 fields = user_regs[i].fields; 2137 for (j = 0; fields[j].type != 0; j++) { 2138 /* Update the FreeBSD userspace ID register view */ 2139 user_reg = update_special_reg_field(user_reg, 2140 fields[j].type >> MRS_TYPE_FBSD_SHIFT, value, 2141 fields[j].shift, fields[j].sign); 2142 2143 /* Update the Linux userspace ID register view */ 2144 l_user_reg = update_special_reg_field(l_user_reg, 2145 fields[j].type >> MRS_TYPE_LNX_SHIFT, value, 2146 fields[j].shift, fields[j].sign); 2147 2148 /* Update the kernel ID register view */ 2149 kern_reg = update_lower_register(kern_reg, value, 2150 fields[j].shift, 4, fields[j].sign); 2151 } 2152 2153 CPU_DESC_FIELD(kern_cpu_desc, i) = kern_reg; 2154 CPU_DESC_FIELD(user_cpu_desc, i) = user_reg; 2155 CPU_DESC_FIELD(l_user_cpu_desc, i) = l_user_reg; 2156 } 2157} 2158 2159void 2160cpu_desc_init(void) 2161{ 2162 if (mp_ncpus == 1) 2163 return; 2164 2165 /* 2166 * Allocate memory for the non-boot CPUs to store their registers. 2167 * As this is indexed by CPU ID we need to allocate space for CPUs 2168 * 1 to mp_maxid. Because of this mp_maxid is already the correct 2169 * number of elements. 2170 */ 2171 cpu_desc = mallocarray(mp_maxid, sizeof(*cpu_desc), M_IDENTCPU, 2172 M_ZERO | M_WAITOK); 2173} 2174 2175/* HWCAP */ 2176bool __read_frequently lse_supported = false; 2177 2178bool __read_frequently icache_aliasing = false; 2179bool __read_frequently icache_vmid = false; 2180 2181int64_t dcache_line_size; /* The minimum D cache line size */ 2182int64_t icache_line_size; /* The minimum I cache line size */ 2183int64_t idcache_line_size; /* The minimum cache line size */ 2184 2185/* 2186 * Find the values to export to userspace as AT_HWCAP and AT_HWCAP2. 2187 */ 2188static void 2189parse_cpu_features(bool is64bit, struct cpu_desc *cpu_desc, u_long *hwcap, 2190 u_long *hwcap2) 2191{ 2192 const struct mrs_field_hwcap *hwcaps; 2193 const struct mrs_field *fields; 2194 uint64_t min, reg; 2195 u_long *cur_hwcap; 2196 int i, j, k; 2197 2198 for (i = 0; i < nitems(user_regs); i++) { 2199 if (user_regs[i].is64bit != is64bit) 2200 continue; 2201 2202 reg = CPU_DESC_FIELD(*cpu_desc, i); 2203 fields = user_regs[i].fields; 2204 for (j = 0; fields[j].type != 0; j++) { 2205 hwcaps = fields[j].hwcaps; 2206 if (hwcaps == NULL) 2207 continue; 2208 2209 for (k = 0; hwcaps[k].hwcap_id != 0; k++) { 2210 KASSERT(hwcaps[k].hwcap_id == 1 || 2211 hwcaps[k].hwcap_id == 2, 2212 ("%s: Invalid HWCAP ID %d", __func__, 2213 hwcaps[k].hwcap_id)); 2214 2215 cur_hwcap = hwcaps[k].hwcap_id == 1 ? 2216 hwcap : hwcap2; 2217 min = hwcaps[k].min; 2218 2219 /* 2220 * If the field is greater than the minimum 2221 * value we can set the hwcap; 2222 */ 2223 if (mrs_field_cmp(reg, min, fields[j].shift, 2224 4, fields[j].sign) >= 0) { 2225 *cur_hwcap |= hwcaps[k].hwcap_val; 2226 } 2227 } 2228 } 2229 } 2230} 2231 2232static void 2233identify_cpu_sysinit(void *dummy __unused) 2234{ 2235 struct cpu_desc *desc, *prev_desc; 2236 int cpu; 2237 bool dic, idc; 2238 2239 dic = (allow_dic != 0); 2240 idc = (allow_idc != 0); 2241 2242 prev_desc = NULL; 2243 CPU_FOREACH(cpu) { 2244 desc = get_cpu_desc(cpu); 2245 if (cpu != 0) { 2246 check_cpu_regs(cpu, desc, prev_desc); 2247 update_special_regs(cpu); 2248 } 2249 2250 if (CTR_DIC_VAL(desc->ctr) == 0) 2251 dic = false; 2252 if (CTR_IDC_VAL(desc->ctr) == 0) 2253 idc = false; 2254 prev_desc = desc; 2255 } 2256 2257 /* Find the values to export to userspace as AT_HWCAP and AT_HWCAP2 */ 2258 parse_cpu_features(true, &user_cpu_desc, &elf_hwcap, &elf_hwcap2); 2259 parse_cpu_features(true, &l_user_cpu_desc, &linux_elf_hwcap, 2260 &linux_elf_hwcap2); 2261#ifdef COMPAT_FREEBSD32 2262 parse_cpu_features(false, &user_cpu_desc, &elf32_hwcap, &elf32_hwcap2); 2263#endif 2264 2265 /* We export the CPUID registers */ 2266 elf_hwcap |= HWCAP_CPUID; 2267 linux_elf_hwcap |= HWCAP_CPUID; 2268 2269#ifdef COMPAT_FREEBSD32 2270 /* Set the default caps and any that need to check multiple fields */ 2271 elf32_hwcap |= parse_cpu_features_hwcap32(); 2272#endif 2273 2274 if (dic && idc) { 2275 arm64_icache_sync_range = &arm64_dic_idc_icache_sync_range; 2276 if (bootverbose) 2277 printf("Enabling DIC & IDC ICache sync\n"); 2278 } else if (idc) { 2279 arm64_icache_sync_range = &arm64_idc_aliasing_icache_sync_range; 2280 if (bootverbose) 2281 printf("Enabling IDC ICache sync\n"); 2282 } 2283 2284 if ((elf_hwcap & HWCAP_ATOMICS) != 0) { 2285 lse_supported = true; 2286 if (bootverbose) 2287 printf("Enabling LSE atomics in the kernel\n"); 2288 } 2289#ifdef LSE_ATOMICS 2290 if (!lse_supported) 2291 panic("CPU does not support LSE atomic instructions"); 2292#endif 2293 2294 install_undef_handler(true, user_mrs_handler); 2295} 2296SYSINIT(identify_cpu, SI_SUB_CPU, SI_ORDER_MIDDLE, identify_cpu_sysinit, NULL); 2297 2298static void 2299cpu_features_sysinit(void *dummy __unused) 2300{ 2301 struct sbuf sb; 2302 struct cpu_desc *desc, *prev_desc; 2303 u_int cpu; 2304 2305 prev_desc = NULL; 2306 CPU_FOREACH(cpu) { 2307 desc = get_cpu_desc(cpu); 2308 print_cpu_features(cpu, desc, prev_desc); 2309 prev_desc = desc; 2310 } 2311 2312 /* Fill in cpu_model for the hw.model sysctl */ 2313 sbuf_new(&sb, cpu_model, sizeof(cpu_model), SBUF_FIXEDLEN); 2314 print_cpu_midr(&sb, 0); 2315 2316 sbuf_finish(&sb); 2317 sbuf_delete(&sb); 2318 2319 free(cpu_desc, M_IDENTCPU); 2320} 2321/* Log features before APs are released and start printing to the dmesg. */ 2322SYSINIT(cpu_features, SI_SUB_SMP - 1, SI_ORDER_ANY, cpu_features_sysinit, NULL); 2323 2324static void 2325tcr_set_e0pd1(void *arg __unused) 2326{ 2327 uint64_t tcr; 2328 2329 tcr = READ_SPECIALREG(tcr_el1); 2330 tcr |= TCR_E0PD1; 2331 WRITE_SPECIALREG(tcr_el1, tcr); 2332 isb(); 2333} 2334 2335/* Enable support for more recent architecture features */ 2336static void 2337cpu_feat_support(void *arg __unused) 2338{ 2339 /* 2340 * If FEAT_E0PD is supported use it to cause faults without a page 2341 * table walk if userspace tries to access kernel memory. 2342 */ 2343 if (ID_AA64MMFR2_E0PD_VAL(kern_cpu_desc.id_aa64mmfr2) != 2344 ID_AA64MMFR2_E0PD_NONE) 2345 smp_rendezvous(NULL, tcr_set_e0pd1, NULL, NULL); 2346} 2347SYSINIT(cpu_feat_support, SI_SUB_SMP, SI_ORDER_ANY, cpu_feat_support, NULL); 2348 2349#ifdef COMPAT_FREEBSD32 2350static u_long 2351parse_cpu_features_hwcap32(void) 2352{ 2353 u_long hwcap = HWCAP32_DEFAULT; 2354 2355 if ((MVFR1_SIMDLS_VAL(user_cpu_desc.mvfr1) >= 2356 MVFR1_SIMDLS_IMPL) && 2357 (MVFR1_SIMDInt_VAL(user_cpu_desc.mvfr1) >= 2358 MVFR1_SIMDInt_IMPL) && 2359 (MVFR1_SIMDSP_VAL(user_cpu_desc.mvfr1) >= 2360 MVFR1_SIMDSP_IMPL)) 2361 hwcap |= HWCAP32_NEON; 2362 2363 return (hwcap); 2364} 2365#endif /* COMPAT_FREEBSD32 */ 2366 2367static void 2368print_ctr_fields(struct sbuf *sb, uint64_t reg, const void *arg __unused) 2369{ 2370 2371 sbuf_printf(sb, "%u byte D-cacheline,", CTR_DLINE_SIZE(reg)); 2372 sbuf_printf(sb, "%u byte I-cacheline,", CTR_ILINE_SIZE(reg)); 2373 reg &= ~(CTR_DLINE_MASK | CTR_ILINE_MASK); 2374 2375 switch(CTR_L1IP_VAL(reg)) { 2376 case CTR_L1IP_VPIPT: 2377 sbuf_printf(sb, "VPIPT"); 2378 break; 2379 case CTR_L1IP_AIVIVT: 2380 sbuf_printf(sb, "AIVIVT"); 2381 break; 2382 case CTR_L1IP_VIPT: 2383 sbuf_printf(sb, "VIPT"); 2384 break; 2385 case CTR_L1IP_PIPT: 2386 sbuf_printf(sb, "PIPT"); 2387 break; 2388 } 2389 sbuf_printf(sb, " ICache,"); 2390 reg &= ~CTR_L1IP_MASK; 2391 2392 sbuf_printf(sb, "%d byte ERG,", CTR_ERG_SIZE(reg)); 2393 sbuf_printf(sb, "%d byte CWG", CTR_CWG_SIZE(reg)); 2394 reg &= ~(CTR_ERG_MASK | CTR_CWG_MASK); 2395 2396 if (CTR_IDC_VAL(reg) != 0) 2397 sbuf_printf(sb, ",IDC"); 2398 if (CTR_DIC_VAL(reg) != 0) 2399 sbuf_printf(sb, ",DIC"); 2400 reg &= ~(CTR_IDC_MASK | CTR_DIC_MASK); 2401 reg &= ~CTR_RES1; 2402 2403 if (reg != 0) 2404 sbuf_printf(sb, ",%lx", reg); 2405} 2406 2407static void 2408print_register(struct sbuf *sb, const char *reg_name, uint64_t reg, 2409 void (*print_fields)(struct sbuf *, uint64_t, const void *), 2410 const void *arg) 2411{ 2412 2413 sbuf_printf(sb, "%29s = <", reg_name); 2414 2415 print_fields(sb, reg, arg); 2416 2417 sbuf_finish(sb); 2418 printf("%s>\n", sbuf_data(sb)); 2419 sbuf_clear(sb); 2420} 2421 2422static void 2423print_id_fields(struct sbuf *sb, uint64_t reg, const void *arg) 2424{ 2425 const struct mrs_field *fields = arg; 2426 const struct mrs_field_value *fv; 2427 int field, i, j, printed; 2428 2429#define SEP_STR ((printed++) == 0) ? "" : "," 2430 printed = 0; 2431 for (i = 0; fields[i].type != 0; i++) { 2432 fv = fields[i].values; 2433 2434 /* TODO: Handle with an unknown message */ 2435 if (fv == NULL) 2436 continue; 2437 2438 field = (reg & fields[i].mask) >> fields[i].shift; 2439 for (j = 0; fv[j].desc != NULL; j++) { 2440 if ((fv[j].value >> fields[i].shift) != field) 2441 continue; 2442 2443 if (fv[j].desc[0] != '\0') 2444 sbuf_printf(sb, "%s%s", SEP_STR, fv[j].desc); 2445 break; 2446 } 2447 if (fv[j].desc == NULL) 2448 sbuf_printf(sb, "%sUnknown %s(%x)", SEP_STR, 2449 fields[i].name, field); 2450 2451 reg &= ~(0xful << fields[i].shift); 2452 } 2453 2454 if (reg != 0) 2455 sbuf_printf(sb, "%s%#lx", SEP_STR, reg); 2456#undef SEP_STR 2457} 2458 2459static void 2460print_id_register(struct sbuf *sb, const char *reg_name, uint64_t reg, 2461 const struct mrs_field *fields) 2462{ 2463 2464 print_register(sb, reg_name, reg, print_id_fields, fields); 2465} 2466 2467static void 2468print_cpu_midr(struct sbuf *sb, u_int cpu) 2469{ 2470 const struct cpu_parts *cpu_partsp; 2471 const char *cpu_impl_name; 2472 const char *cpu_part_name; 2473 u_int midr; 2474 u_int impl_id; 2475 u_int part_id; 2476 2477 midr = pcpu_find(cpu)->pc_midr; 2478 2479 cpu_impl_name = NULL; 2480 cpu_partsp = NULL; 2481 impl_id = CPU_IMPL(midr); 2482 for (int i = 0; cpu_implementers[i].impl_name != NULL; i++) { 2483 if (impl_id == cpu_implementers[i].impl_id) { 2484 cpu_impl_name = cpu_implementers[i].impl_name; 2485 cpu_partsp = cpu_implementers[i].cpu_parts; 2486 break; 2487 } 2488 } 2489 /* Unknown implementer, so unknown part */ 2490 if (cpu_impl_name == NULL) { 2491 sbuf_printf(sb, "Unknown Implementer (midr: %08x)", midr); 2492 return; 2493 } 2494 2495 KASSERT(cpu_partsp != NULL, ("%s: No parts table for implementer %s", 2496 __func__, cpu_impl_name)); 2497 2498 cpu_part_name = NULL; 2499 part_id = CPU_PART(midr); 2500 for (int i = 0; cpu_partsp[i].part_name != NULL; i++) { 2501 if (part_id == cpu_partsp[i].part_id) { 2502 cpu_part_name = cpu_partsp[i].part_name; 2503 break; 2504 } 2505 } 2506 /* Known Implementer, Unknown part */ 2507 if (cpu_part_name == NULL) { 2508 sbuf_printf(sb, "%s Unknown CPU r%dp%d (midr: %08x)", 2509 cpu_impl_name, CPU_VAR(midr), CPU_REV(midr), midr); 2510 return; 2511 } 2512 2513 sbuf_printf(sb, "%s %s r%dp%d", cpu_impl_name, 2514 cpu_part_name, CPU_VAR(midr), CPU_REV(midr)); 2515} 2516 2517static void 2518print_cpu_cache(struct cpu_desc *desc, struct sbuf *sb, uint64_t ccs, 2519 bool icache, bool unified) 2520{ 2521 size_t cache_size; 2522 size_t line_size; 2523 2524 /* LineSize is Log2(S) - 4. */ 2525 line_size = 1 << ((ccs & CCSIDR_LineSize_MASK) + 4); 2526 /* 2527 * Calculate cache size (sets * ways * line size). There are different 2528 * formats depending on the FEAT_CCIDX bit in ID_AA64MMFR2 feature 2529 * register. 2530 */ 2531 if ((desc->id_aa64mmfr2 & ID_AA64MMFR2_CCIDX_64)) 2532 cache_size = (CCSIDR_NSETS_64(ccs) + 1) * 2533 (CCSIDR_ASSOC_64(ccs) + 1); 2534 else 2535 cache_size = (CCSIDR_NSETS(ccs) + 1) * (CCSIDR_ASSOC(ccs) + 1); 2536 2537 cache_size *= line_size; 2538 sbuf_printf(sb, "%zuKB (%s)", cache_size / 1024, 2539 icache ? "instruction" : unified ? "unified" : "data"); 2540} 2541 2542static void 2543print_cpu_caches(struct sbuf *sb, struct cpu_desc *desc) 2544{ 2545 /* Print out each cache combination */ 2546 uint64_t clidr; 2547 int i = 1; 2548 clidr = desc->clidr; 2549 2550 for (i = 0; (clidr & CLIDR_CTYPE_MASK) != 0; i++, clidr >>= 3) { 2551 int j = 0; 2552 int ctype_m = (clidr & CLIDR_CTYPE_MASK); 2553 2554 sbuf_printf(sb, " L%d cache: ", i + 1); 2555 if ((clidr & CLIDR_CTYPE_IO)) { 2556 print_cpu_cache(desc, sb, desc->ccsidr[i][j++], true, 2557 false); 2558 /* If there's more, add to the line. */ 2559 if ((ctype_m & ~CLIDR_CTYPE_IO) != 0) 2560 sbuf_printf(sb, ", "); 2561 } 2562 if ((ctype_m & ~CLIDR_CTYPE_IO) != 0) { 2563 print_cpu_cache(desc, sb, desc->ccsidr[i][j], false, 2564 (clidr & CLIDR_CTYPE_UNIFIED)); 2565 } 2566 sbuf_printf(sb, "\n"); 2567 2568 } 2569 sbuf_finish(sb); 2570 printf("%s", sbuf_data(sb)); 2571} 2572 2573static void 2574print_cpu_features(u_int cpu, struct cpu_desc *desc, 2575 struct cpu_desc *prev_desc) 2576{ 2577 struct sbuf *sb; 2578 2579 sb = sbuf_new_auto(); 2580 sbuf_printf(sb, "CPU%3u: ", cpu); 2581 print_cpu_midr(sb, cpu); 2582 2583 sbuf_cat(sb, " affinity:"); 2584 switch(cpu_aff_levels) { 2585 default: 2586 case 4: 2587 sbuf_printf(sb, " %2d", CPU_AFF3(desc->mpidr)); 2588 /* FALLTHROUGH */ 2589 case 3: 2590 sbuf_printf(sb, " %2d", CPU_AFF2(desc->mpidr)); 2591 /* FALLTHROUGH */ 2592 case 2: 2593 sbuf_printf(sb, " %2d", CPU_AFF1(desc->mpidr)); 2594 /* FALLTHROUGH */ 2595 case 1: 2596 case 0: /* On UP this will be zero */ 2597 sbuf_printf(sb, " %2d", CPU_AFF0(desc->mpidr)); 2598 break; 2599 } 2600 sbuf_finish(sb); 2601 printf("%s\n", sbuf_data(sb)); 2602 sbuf_clear(sb); 2603 2604 /* 2605 * There is a hardware errata where, if one CPU is performing a TLB 2606 * invalidation while another is performing a store-exclusive the 2607 * store-exclusive may return the wrong status. A workaround seems 2608 * to be to use an IPI to invalidate on each CPU, however given the 2609 * limited number of affected units (pass 1.1 is the evaluation 2610 * hardware revision), and the lack of information from Cavium 2611 * this has not been implemented. 2612 * 2613 * At the time of writing this the only information is from: 2614 * https://lkml.org/lkml/2016/8/4/722 2615 */ 2616 /* 2617 * XXX: CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1 on its own also 2618 * triggers on pass 2.0+. 2619 */ 2620 if (cpu == 0 && CPU_VAR(PCPU_GET(midr)) == 0 && 2621 CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1) 2622 printf("WARNING: ThunderX Pass 1.1 detected.\nThis has known " 2623 "hardware bugs that may cause the incorrect operation of " 2624 "atomic operations.\n"); 2625 2626#define SHOULD_PRINT_REG(_reg) \ 2627 (prev_desc == NULL || desc->_reg != prev_desc->_reg) 2628 2629 /* Cache Type Register */ 2630 if (SHOULD_PRINT_REG(ctr)) { 2631 print_register(sb, "Cache Type", 2632 desc->ctr, print_ctr_fields, NULL); 2633 } 2634 2635 /* AArch64 Instruction Set Attribute Register 0 */ 2636 if (SHOULD_PRINT_REG(id_aa64isar0)) 2637 print_id_register(sb, "Instruction Set Attributes 0", 2638 desc->id_aa64isar0, id_aa64isar0_fields); 2639 2640 /* AArch64 Instruction Set Attribute Register 1 */ 2641 if (SHOULD_PRINT_REG(id_aa64isar1)) 2642 print_id_register(sb, "Instruction Set Attributes 1", 2643 desc->id_aa64isar1, id_aa64isar1_fields); 2644 2645 /* AArch64 Instruction Set Attribute Register 2 */ 2646 if (SHOULD_PRINT_REG(id_aa64isar2)) 2647 print_id_register(sb, "Instruction Set Attributes 2", 2648 desc->id_aa64isar2, id_aa64isar2_fields); 2649 2650 /* AArch64 Processor Feature Register 0 */ 2651 if (SHOULD_PRINT_REG(id_aa64pfr0)) 2652 print_id_register(sb, "Processor Features 0", 2653 desc->id_aa64pfr0, id_aa64pfr0_fields); 2654 2655 /* AArch64 Processor Feature Register 1 */ 2656 if (SHOULD_PRINT_REG(id_aa64pfr1)) 2657 print_id_register(sb, "Processor Features 1", 2658 desc->id_aa64pfr1, id_aa64pfr1_fields); 2659 2660 /* AArch64 Processor Feature Register 2 */ 2661 if (SHOULD_PRINT_REG(id_aa64pfr2)) 2662 print_id_register(sb, "Processor Features 2", 2663 desc->id_aa64pfr2, id_aa64pfr2_fields); 2664 2665 /* AArch64 Memory Model Feature Register 0 */ 2666 if (SHOULD_PRINT_REG(id_aa64mmfr0)) 2667 print_id_register(sb, "Memory Model Features 0", 2668 desc->id_aa64mmfr0, id_aa64mmfr0_fields); 2669 2670 /* AArch64 Memory Model Feature Register 1 */ 2671 if (SHOULD_PRINT_REG(id_aa64mmfr1)) 2672 print_id_register(sb, "Memory Model Features 1", 2673 desc->id_aa64mmfr1, id_aa64mmfr1_fields); 2674 2675 /* AArch64 Memory Model Feature Register 2 */ 2676 if (SHOULD_PRINT_REG(id_aa64mmfr2)) 2677 print_id_register(sb, "Memory Model Features 2", 2678 desc->id_aa64mmfr2, id_aa64mmfr2_fields); 2679 2680 /* AArch64 Memory Model Feature Register 3 */ 2681 if (SHOULD_PRINT_REG(id_aa64mmfr3)) 2682 print_id_register(sb, "Memory Model Features 3", 2683 desc->id_aa64mmfr3, id_aa64mmfr3_fields); 2684 2685 /* AArch64 Memory Model Feature Register 4 */ 2686 if (SHOULD_PRINT_REG(id_aa64mmfr4)) 2687 print_id_register(sb, "Memory Model Features 4", 2688 desc->id_aa64mmfr4, id_aa64mmfr4_fields); 2689 2690 /* AArch64 Debug Feature Register 0 */ 2691 if (SHOULD_PRINT_REG(id_aa64dfr0)) 2692 print_id_register(sb, "Debug Features 0", 2693 desc->id_aa64dfr0, id_aa64dfr0_fields); 2694 2695 /* AArch64 Memory Model Feature Register 1 */ 2696 if (SHOULD_PRINT_REG(id_aa64dfr1)) 2697 print_id_register(sb, "Debug Features 1", 2698 desc->id_aa64dfr1, id_aa64dfr1_fields); 2699 2700 /* AArch64 Auxiliary Feature Register 0 */ 2701 if (SHOULD_PRINT_REG(id_aa64afr0)) 2702 print_id_register(sb, "Auxiliary Features 0", 2703 desc->id_aa64afr0, id_aa64afr0_fields); 2704 2705 /* AArch64 Auxiliary Feature Register 1 */ 2706 if (SHOULD_PRINT_REG(id_aa64afr1)) 2707 print_id_register(sb, "Auxiliary Features 1", 2708 desc->id_aa64afr1, id_aa64afr1_fields); 2709 2710 /* AArch64 SVE Feature Register 0 */ 2711 if (desc->have_sve) { 2712 if (SHOULD_PRINT_REG(id_aa64zfr0) || 2713 !prev_desc->have_sve) { 2714 print_id_register(sb, "SVE Features 0", 2715 desc->id_aa64zfr0, id_aa64zfr0_fields); 2716 } 2717 } 2718 2719#ifdef COMPAT_FREEBSD32 2720 /* AArch32 Instruction Set Attribute Register 5 */ 2721 if (SHOULD_PRINT_REG(id_isar5)) 2722 print_id_register(sb, "AArch32 Instruction Set Attributes 5", 2723 desc->id_isar5, id_isar5_fields); 2724 2725 /* AArch32 Media and VFP Feature Register 0 */ 2726 if (SHOULD_PRINT_REG(mvfr0)) 2727 print_id_register(sb, "AArch32 Media and VFP Features 0", 2728 desc->mvfr0, mvfr0_fields); 2729 2730 /* AArch32 Media and VFP Feature Register 1 */ 2731 if (SHOULD_PRINT_REG(mvfr1)) 2732 print_id_register(sb, "AArch32 Media and VFP Features 1", 2733 desc->mvfr1, mvfr1_fields); 2734#endif 2735 if (bootverbose) 2736 print_cpu_caches(sb, desc); 2737 2738 sbuf_delete(sb); 2739 sb = NULL; 2740#undef SHOULD_PRINT_REG 2741#undef SEP_STR 2742} 2743 2744void 2745identify_cache(uint64_t ctr) 2746{ 2747 2748 /* Identify the L1 cache type */ 2749 switch (CTR_L1IP_VAL(ctr)) { 2750 case CTR_L1IP_PIPT: 2751 break; 2752 case CTR_L1IP_VPIPT: 2753 icache_vmid = true; 2754 break; 2755 default: 2756 case CTR_L1IP_VIPT: 2757 icache_aliasing = true; 2758 break; 2759 } 2760 2761 if (dcache_line_size == 0) { 2762 KASSERT(icache_line_size == 0, ("%s: i-cacheline size set: %ld", 2763 __func__, icache_line_size)); 2764 2765 /* Get the D cache line size */ 2766 dcache_line_size = CTR_DLINE_SIZE(ctr); 2767 /* And the same for the I cache */ 2768 icache_line_size = CTR_ILINE_SIZE(ctr); 2769 2770 idcache_line_size = MIN(dcache_line_size, icache_line_size); 2771 } 2772 2773 if (dcache_line_size != CTR_DLINE_SIZE(ctr)) { 2774 printf("WARNING: D-cacheline size mismatch %ld != %d\n", 2775 dcache_line_size, CTR_DLINE_SIZE(ctr)); 2776 } 2777 2778 if (icache_line_size != CTR_ILINE_SIZE(ctr)) { 2779 printf("WARNING: I-cacheline size mismatch %ld != %d\n", 2780 icache_line_size, CTR_ILINE_SIZE(ctr)); 2781 } 2782} 2783 2784void 2785identify_cpu(u_int cpu) 2786{ 2787 struct cpu_desc *desc; 2788 uint64_t clidr; 2789 2790 desc = get_cpu_desc(cpu); 2791 /* Save affinity for current CPU */ 2792 desc->mpidr = get_mpidr(); 2793 CPU_AFFINITY(cpu) = desc->mpidr & CPU_AFF_MASK; 2794 2795 desc->ctr = READ_SPECIALREG(ctr_el0); 2796 desc->id_aa64dfr0 = READ_SPECIALREG(ID_AA64DFR0_EL1_REG); 2797 desc->id_aa64dfr1 = READ_SPECIALREG(ID_AA64DFR1_EL1_REG); 2798 desc->id_aa64isar0 = READ_SPECIALREG(ID_AA64ISAR0_EL1_REG); 2799 desc->id_aa64isar1 = READ_SPECIALREG(ID_AA64ISAR1_EL1_REG); 2800 desc->id_aa64isar2 = READ_SPECIALREG(ID_AA64ISAR2_EL1_REG); 2801 desc->id_aa64mmfr0 = READ_SPECIALREG(ID_AA64MMFR0_EL1_REG); 2802 desc->id_aa64mmfr1 = READ_SPECIALREG(ID_AA64MMFR1_EL1_REG); 2803 desc->id_aa64mmfr2 = READ_SPECIALREG(ID_AA64MMFR2_EL1_REG); 2804 desc->id_aa64mmfr3 = READ_SPECIALREG(ID_AA64MMFR3_EL1_REG); 2805 desc->id_aa64mmfr4 = READ_SPECIALREG(ID_AA64MMFR4_EL1_REG); 2806 desc->id_aa64pfr0 = READ_SPECIALREG(ID_AA64PFR0_EL1_REG); 2807 desc->id_aa64pfr1 = READ_SPECIALREG(ID_AA64PFR1_EL1_REG); 2808 desc->id_aa64pfr2 = READ_SPECIALREG(ID_AA64PFR2_EL1_REG); 2809 2810 /* 2811 * ID_AA64ZFR0_EL1 is only valid when at least one of: 2812 * - ID_AA64PFR0_EL1.SVE is non-zero 2813 * - ID_AA64PFR1_EL1.SME is non-zero 2814 * In other cases it is zero, but still safe to read 2815 */ 2816 desc->have_sve = 2817 (ID_AA64PFR0_SVE_VAL(desc->id_aa64pfr0) != 0); 2818 desc->id_aa64zfr0 = READ_SPECIALREG(ID_AA64ZFR0_EL1_REG); 2819 2820 desc->clidr = READ_SPECIALREG(clidr_el1); 2821 2822 clidr = desc->clidr; 2823 2824 for (int i = 0; (clidr & CLIDR_CTYPE_MASK) != 0; i++, clidr >>= 3) { 2825 int j = 0; 2826 if ((clidr & CLIDR_CTYPE_IO)) { 2827 WRITE_SPECIALREG(csselr_el1, 2828 CSSELR_Level(i) | CSSELR_InD); 2829 desc->ccsidr[i][j++] = 2830 READ_SPECIALREG(ccsidr_el1); 2831 } 2832 if ((clidr & ~CLIDR_CTYPE_IO) == 0) 2833 continue; 2834 WRITE_SPECIALREG(csselr_el1, CSSELR_Level(i)); 2835 desc->ccsidr[i][j] = READ_SPECIALREG(ccsidr_el1); 2836 } 2837 2838#ifdef COMPAT_FREEBSD32 2839 /* Only read aarch32 SRs if EL0-32 is available */ 2840 if (ID_AA64PFR0_EL0_VAL(desc->id_aa64pfr0) == ID_AA64PFR0_EL0_64_32) { 2841 desc->id_isar5 = READ_SPECIALREG(id_isar5_el1); 2842 desc->mvfr0 = READ_SPECIALREG(mvfr0_el1); 2843 desc->mvfr1 = READ_SPECIALREG(mvfr1_el1); 2844 } 2845#endif 2846} 2847 2848static void 2849check_cpu_regs(u_int cpu, struct cpu_desc *desc, struct cpu_desc *prev_desc) 2850{ 2851 switch (cpu_aff_levels) { 2852 case 0: 2853 if (CPU_AFF0(desc->mpidr) != CPU_AFF0(prev_desc->mpidr)) 2854 cpu_aff_levels = 1; 2855 /* FALLTHROUGH */ 2856 case 1: 2857 if (CPU_AFF1(desc->mpidr) != CPU_AFF1(prev_desc->mpidr)) 2858 cpu_aff_levels = 2; 2859 /* FALLTHROUGH */ 2860 case 2: 2861 if (CPU_AFF2(desc->mpidr) != CPU_AFF2(prev_desc->mpidr)) 2862 cpu_aff_levels = 3; 2863 /* FALLTHROUGH */ 2864 case 3: 2865 if (CPU_AFF3(desc->mpidr) != CPU_AFF3(prev_desc->mpidr)) 2866 cpu_aff_levels = 4; 2867 break; 2868 } 2869 2870 if (desc->ctr != prev_desc->ctr) { 2871 /* 2872 * If the cache type register is different we may 2873 * have a different l1 cache type. 2874 */ 2875 identify_cache(desc->ctr); 2876 } 2877} 2878