mp_machdep.c revision 292771
1/*- 2 * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 3. Berkeley Software Design Inc's name may not be used to endorse or 13 * promote products derived from this software without specific prior 14 * written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * from BSDI: locore.s,v 1.36.2.15 1999/08/23 22:34:41 cp Exp 29 */ 30/*- 31 * Copyright (c) 2002 Jake Burkholder. 32 * Copyright (c) 2007 - 2010 Marius Strobl <marius@FreeBSD.org> 33 * All rights reserved. 34 * 35 * Redistribution and use in source and binary forms, with or without 36 * modification, are permitted provided that the following conditions 37 * are met: 38 * 1. Redistributions of source code must retain the above copyright 39 * notice, this list of conditions and the following disclaimer. 40 * 2. Redistributions in binary form must reproduce the above copyright 41 * notice, this list of conditions and the following disclaimer in the 42 * documentation and/or other materials provided with the distribution. 43 * 44 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 45 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 46 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 47 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 48 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 49 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 50 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 51 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 52 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 53 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 54 * SUCH DAMAGE. 55 */ 56 57#include <sys/cdefs.h> 58__FBSDID("$FreeBSD: stable/10/sys/sparc64/sparc64/mp_machdep.c 292771 2015-12-27 14:39:47Z marius $"); 59 60#include <sys/param.h> 61#include <sys/systm.h> 62#include <sys/lock.h> 63#include <sys/kdb.h> 64#include <sys/kernel.h> 65#include <sys/ktr.h> 66#include <sys/mutex.h> 67#include <sys/pcpu.h> 68#include <sys/proc.h> 69#include <sys/sched.h> 70#include <sys/smp.h> 71 72#include <vm/vm.h> 73#include <vm/vm_param.h> 74#include <vm/pmap.h> 75#include <vm/vm_kern.h> 76#include <vm/vm_extern.h> 77#include <vm/vm_map.h> 78 79#include <dev/ofw/openfirm.h> 80 81#include <machine/asi.h> 82#include <machine/atomic.h> 83#include <machine/bus.h> 84#include <machine/cpu.h> 85#include <machine/cpufunc.h> 86#include <machine/md_var.h> 87#include <machine/metadata.h> 88#include <machine/ofw_machdep.h> 89#include <machine/pcb.h> 90#include <machine/smp.h> 91#include <machine/tick.h> 92#include <machine/tlb.h> 93#include <machine/tsb.h> 94#include <machine/tte.h> 95#include <machine/ver.h> 96 97#define SUNW_STARTCPU "SUNW,start-cpu" 98#define SUNW_STOPSELF "SUNW,stop-self" 99 100static ih_func_t cpu_ipi_ast; 101static ih_func_t cpu_ipi_hardclock; 102static ih_func_t cpu_ipi_preempt; 103static ih_func_t cpu_ipi_stop; 104 105/* 106 * Argument area used to pass data to non-boot processors as they start up. 107 * This must be statically initialized with a known invalid CPU module ID, 108 * since the other processors will use it before the boot CPU enters the 109 * kernel. 110 */ 111struct cpu_start_args cpu_start_args = { 0, -1, -1, 0, 0, 0 }; 112struct ipi_cache_args ipi_cache_args; 113struct ipi_rd_args ipi_rd_args; 114struct ipi_tlb_args ipi_tlb_args; 115struct pcb stoppcbs[MAXCPU]; 116 117struct mtx ipi_mtx; 118 119cpu_ipi_selected_t *cpu_ipi_selected; 120cpu_ipi_single_t *cpu_ipi_single; 121 122static u_int cpuid_to_mid[MAXCPU]; 123static u_int cpuids = 1; 124static volatile cpuset_t shutdown_cpus; 125static char ipi_pbuf[CPUSETBUFSIZ]; 126static vm_offset_t mp_tramp; 127 128static void ap_count(phandle_t node, u_int mid, u_int cpu_impl); 129static void ap_start(phandle_t node, u_int mid, u_int cpu_impl); 130static void cpu_mp_unleash(void *v); 131static void foreach_ap(phandle_t node, void (*func)(phandle_t node, 132 u_int mid, u_int cpu_impl)); 133static void sun4u_startcpu(phandle_t cpu, void *func, u_long arg); 134 135static cpu_ipi_selected_t cheetah_ipi_selected; 136static cpu_ipi_single_t cheetah_ipi_single; 137static cpu_ipi_selected_t jalapeno_ipi_selected; 138static cpu_ipi_single_t jalapeno_ipi_single; 139static cpu_ipi_selected_t spitfire_ipi_selected; 140static cpu_ipi_single_t spitfire_ipi_single; 141 142SYSINIT(cpu_mp_unleash, SI_SUB_SMP, SI_ORDER_FIRST, cpu_mp_unleash, NULL); 143 144void 145mp_init(void) 146{ 147 struct tte *tp; 148 int i; 149 150 mp_tramp = (vm_offset_t)OF_claim(NULL, PAGE_SIZE, PAGE_SIZE); 151 if (mp_tramp == (vm_offset_t)-1) 152 panic("%s", __func__); 153 bcopy(mp_tramp_code, (void *)mp_tramp, mp_tramp_code_len); 154 *(vm_offset_t *)(mp_tramp + mp_tramp_tlb_slots) = kernel_tlb_slots; 155 *(vm_offset_t *)(mp_tramp + mp_tramp_func) = (vm_offset_t)mp_startup; 156 tp = (struct tte *)(mp_tramp + mp_tramp_code_len); 157 for (i = 0; i < kernel_tlb_slots; i++) { 158 tp[i].tte_vpn = TV_VPN(kernel_tlbs[i].te_va, TS_4M); 159 tp[i].tte_data = TD_V | TD_4M | TD_PA(kernel_tlbs[i].te_pa) | 160 TD_L | TD_CP | TD_CV | TD_P | TD_W; 161 } 162 for (i = 0; i < PAGE_SIZE; i += sizeof(vm_offset_t)) 163 flush(mp_tramp + i); 164} 165 166static void 167foreach_ap(phandle_t node, void (*func)(phandle_t node, u_int mid, 168 u_int cpu_impl)) 169{ 170 static char type[sizeof("cpu")]; 171 phandle_t child; 172 uint32_t cpu_impl, portid; 173 174 /* There's no need to traverse the whole OFW tree twice. */ 175 if (mp_maxid > 0 && cpuids > mp_maxid) 176 return; 177 178 for (; node != 0; node = OF_peer(node)) { 179 child = OF_child(node); 180 if (child > 0) 181 foreach_ap(child, func); 182 else { 183 if (OF_getprop(node, "device_type", type, 184 sizeof(type)) <= 0) 185 continue; 186 if (strcmp(type, "cpu") != 0) 187 continue; 188 if (OF_getprop(node, "implementation#", &cpu_impl, 189 sizeof(cpu_impl)) <= 0) 190 panic("%s: couldn't determine CPU " 191 "implementation", __func__); 192 if (OF_getprop(node, cpu_portid_prop(cpu_impl), 193 &portid, sizeof(portid)) <= 0) 194 panic("%s: couldn't determine CPU port ID", 195 __func__); 196 if (portid == PCPU_GET(mid)) 197 continue; 198 (*func)(node, portid, cpu_impl); 199 } 200 } 201} 202 203/* 204 * Probe for other CPUs. 205 */ 206void 207cpu_mp_setmaxid(void) 208{ 209 210 CPU_SETOF(curcpu, &all_cpus); 211 mp_ncpus = 1; 212 213 foreach_ap(OF_child(OF_peer(0)), ap_count); 214 mp_ncpus = MIN(mp_ncpus, MAXCPU); 215 mp_maxid = mp_ncpus - 1; 216} 217 218static void 219ap_count(phandle_t node __unused, u_int mid __unused, u_int cpu_impl __unused) 220{ 221 222 mp_ncpus++; 223} 224 225int 226cpu_mp_probe(void) 227{ 228 229 return (mp_maxid > 0); 230} 231 232struct cpu_group * 233cpu_topo(void) 234{ 235 236 return (smp_topo_none()); 237} 238 239static void 240sun4u_startcpu(phandle_t cpu, void *func, u_long arg) 241{ 242 static struct { 243 cell_t name; 244 cell_t nargs; 245 cell_t nreturns; 246 cell_t cpu; 247 cell_t func; 248 cell_t arg; 249 } args = { 250 (cell_t)SUNW_STARTCPU, 251 3, 252 }; 253 254 args.cpu = cpu; 255 args.func = (cell_t)func; 256 args.arg = (cell_t)arg; 257 ofw_entry(&args); 258} 259 260/* 261 * Fire up any non-boot processors. 262 */ 263void 264cpu_mp_start(void) 265{ 266 u_int cpu_impl, isjbus; 267 268 mtx_init(&ipi_mtx, "ipi", NULL, MTX_SPIN); 269 270 isjbus = 0; 271 cpu_impl = PCPU_GET(impl); 272 if (cpu_impl == CPU_IMPL_ULTRASPARCIIIi || 273 cpu_impl == CPU_IMPL_ULTRASPARCIIIip) { 274 isjbus = 1; 275 cpu_ipi_selected = jalapeno_ipi_selected; 276 cpu_ipi_single = jalapeno_ipi_single; 277 } else if (cpu_impl == CPU_IMPL_SPARC64V || 278 cpu_impl >= CPU_IMPL_ULTRASPARCIII) { 279 cpu_ipi_selected = cheetah_ipi_selected; 280 cpu_ipi_single = cheetah_ipi_single; 281 } else { 282 cpu_ipi_selected = spitfire_ipi_selected; 283 cpu_ipi_single = spitfire_ipi_single; 284 } 285 286 intr_setup(PIL_AST, cpu_ipi_ast, -1, NULL, NULL); 287 intr_setup(PIL_RENDEZVOUS, (ih_func_t *)smp_rendezvous_action, 288 -1, NULL, NULL); 289 intr_setup(PIL_STOP, cpu_ipi_stop, -1, NULL, NULL); 290 intr_setup(PIL_PREEMPT, cpu_ipi_preempt, -1, NULL, NULL); 291 intr_setup(PIL_HARDCLOCK, cpu_ipi_hardclock, -1, NULL, NULL); 292 293 cpuid_to_mid[curcpu] = PCPU_GET(mid); 294 295 foreach_ap(OF_child(OF_peer(0)), ap_start); 296 KASSERT(!isjbus || mp_ncpus <= IDR_JALAPENO_MAX_BN_PAIRS, 297 ("%s: can only IPI a maximum of %d JBus-CPUs", 298 __func__, IDR_JALAPENO_MAX_BN_PAIRS)); 299} 300 301static void 302ap_start(phandle_t node, u_int mid, u_int cpu_impl) 303{ 304 volatile struct cpu_start_args *csa; 305 struct pcpu *pc; 306 register_t s; 307 vm_offset_t va; 308 u_int cpuid; 309 uint32_t clock; 310 311 if (cpuids > mp_maxid) 312 return; 313 314 if (OF_getprop(node, "clock-frequency", &clock, sizeof(clock)) <= 0) 315 panic("%s: couldn't determine CPU frequency", __func__); 316 if (clock != PCPU_GET(clock)) 317 tick_et_use_stick = 1; 318 319 csa = &cpu_start_args; 320 csa->csa_state = 0; 321 sun4u_startcpu(node, (void *)mp_tramp, 0); 322 s = intr_disable(); 323 while (csa->csa_state != CPU_TICKSYNC) 324 ; 325 membar(StoreLoad); 326 csa->csa_tick = rd(tick); 327 if (cpu_impl == CPU_IMPL_SPARC64V || 328 cpu_impl >= CPU_IMPL_ULTRASPARCIII) { 329 while (csa->csa_state != CPU_STICKSYNC) 330 ; 331 membar(StoreLoad); 332 csa->csa_stick = rdstick(); 333 } 334 while (csa->csa_state != CPU_INIT) 335 ; 336 csa->csa_tick = csa->csa_stick = 0; 337 intr_restore(s); 338 339 cpuid = cpuids++; 340 cpuid_to_mid[cpuid] = mid; 341 cpu_identify(csa->csa_ver, clock, cpuid); 342 343 va = kmem_malloc(kernel_arena, PCPU_PAGES * PAGE_SIZE, 344 M_WAITOK | M_ZERO); 345 pc = (struct pcpu *)(va + (PCPU_PAGES * PAGE_SIZE)) - 1; 346 pcpu_init(pc, cpuid, sizeof(*pc)); 347 dpcpu_init((void *)kmem_malloc(kernel_arena, DPCPU_SIZE, 348 M_WAITOK | M_ZERO), cpuid); 349 pc->pc_addr = va; 350 pc->pc_clock = clock; 351 pc->pc_impl = cpu_impl; 352 pc->pc_mid = mid; 353 pc->pc_node = node; 354 355 cache_init(pc); 356 357 CPU_SET(cpuid, &all_cpus); 358 intr_add_cpu(cpuid); 359} 360 361void 362cpu_mp_announce(void) 363{ 364 365} 366 367static void 368cpu_mp_unleash(void *v __unused) 369{ 370 volatile struct cpu_start_args *csa; 371 struct pcpu *pc; 372 register_t s; 373 vm_offset_t va; 374 vm_paddr_t pa; 375 u_int ctx_inc; 376 u_int ctx_min; 377 int i; 378 379 ctx_min = TLB_CTX_USER_MIN; 380 ctx_inc = (TLB_CTX_USER_MAX - 1) / mp_ncpus; 381 csa = &cpu_start_args; 382 csa->csa_count = mp_ncpus; 383 STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { 384 pc->pc_tlb_ctx = ctx_min; 385 pc->pc_tlb_ctx_min = ctx_min; 386 pc->pc_tlb_ctx_max = ctx_min + ctx_inc; 387 ctx_min += ctx_inc; 388 389 if (pc->pc_cpuid == curcpu) 390 continue; 391 KASSERT(pc->pc_idlethread != NULL, 392 ("%s: idlethread", __func__)); 393 pc->pc_curthread = pc->pc_idlethread; 394 pc->pc_curpcb = pc->pc_curthread->td_pcb; 395 for (i = 0; i < PCPU_PAGES; i++) { 396 va = pc->pc_addr + i * PAGE_SIZE; 397 pa = pmap_kextract(va); 398 if (pa == 0) 399 panic("%s: pmap_kextract", __func__); 400 csa->csa_ttes[i].tte_vpn = TV_VPN(va, TS_8K); 401 csa->csa_ttes[i].tte_data = TD_V | TD_8K | TD_PA(pa) | 402 TD_L | TD_CP | TD_CV | TD_P | TD_W; 403 } 404 csa->csa_state = 0; 405 csa->csa_pcpu = pc->pc_addr; 406 csa->csa_mid = pc->pc_mid; 407 s = intr_disable(); 408 while (csa->csa_state != CPU_BOOTSTRAP) 409 ; 410 intr_restore(s); 411 } 412 413 membar(StoreLoad); 414 csa->csa_count = 0; 415} 416 417void 418cpu_mp_bootstrap(struct pcpu *pc) 419{ 420 volatile struct cpu_start_args *csa; 421 422 csa = &cpu_start_args; 423 424 /* Do CPU-specific initialization. */ 425 if (pc->pc_impl >= CPU_IMPL_ULTRASPARCIII) 426 cheetah_init(pc->pc_impl); 427 else if (pc->pc_impl == CPU_IMPL_SPARC64V) 428 zeus_init(pc->pc_impl); 429 430 /* 431 * Enable the caches. Note that his may include applying workarounds. 432 */ 433 cache_enable(pc->pc_impl); 434 435 /* 436 * Clear (S)TICK timer(s) (including NPT) and ensure they are stopped. 437 */ 438 tick_clear(pc->pc_impl); 439 tick_stop(pc->pc_impl); 440 441 /* Set the kernel context. */ 442 pmap_set_kctx(); 443 444 /* Lock the kernel TSB in the TLB if necessary. */ 445 if (tsb_kernel_ldd_phys == 0) 446 pmap_map_tsb(); 447 448 /* 449 * Flush all non-locked TLB entries possibly left over by the 450 * firmware. 451 */ 452 tlb_flush_nonlocked(); 453 454 /* 455 * Enable interrupts. 456 * Note that the PIL we be lowered indirectly via sched_throw(NULL) 457 * when fake spinlock held by the idle thread eventually is released. 458 */ 459 wrpr(pstate, 0, PSTATE_KERNEL); 460 461 smp_cpus++; 462 KASSERT(curthread != NULL, ("%s: curthread", __func__)); 463 printf("SMP: AP CPU #%d Launched!\n", curcpu); 464 465 csa->csa_count--; 466 membar(StoreLoad); 467 csa->csa_state = CPU_BOOTSTRAP; 468 while (csa->csa_count != 0) 469 ; 470 471 if (smp_cpus == mp_ncpus) 472 atomic_store_rel_int(&smp_started, 1); 473 474 /* Start per-CPU event timers. */ 475 cpu_initclocks_ap(); 476 477 /* Ok, now enter the scheduler. */ 478 sched_throw(NULL); 479} 480 481void 482cpu_mp_shutdown(void) 483{ 484 cpuset_t cpus; 485 int i; 486 487 critical_enter(); 488 shutdown_cpus = all_cpus; 489 CPU_CLR(PCPU_GET(cpuid), &shutdown_cpus); 490 cpus = shutdown_cpus; 491 492 /* XXX: Stop all the CPUs which aren't already. */ 493 if (CPU_CMP(&stopped_cpus, &cpus)) { 494 495 /* cpus is just a flat "on" mask without curcpu. */ 496 CPU_NAND(&cpus, &stopped_cpus); 497 stop_cpus(cpus); 498 } 499 i = 0; 500 while (!CPU_EMPTY(&shutdown_cpus)) { 501 if (i++ > 100000) { 502 printf("timeout shutting down CPUs.\n"); 503 break; 504 } 505 } 506 critical_exit(); 507} 508 509static void 510cpu_ipi_ast(struct trapframe *tf __unused) 511{ 512 513} 514 515static void 516cpu_ipi_stop(struct trapframe *tf __unused) 517{ 518 u_int cpuid; 519 520 CTR2(KTR_SMP, "%s: stopped %d", __func__, curcpu); 521 sched_pin(); 522 savectx(&stoppcbs[curcpu]); 523 cpuid = PCPU_GET(cpuid); 524 CPU_SET_ATOMIC(cpuid, &stopped_cpus); 525 while (!CPU_ISSET(cpuid, &started_cpus)) { 526 if (CPU_ISSET(cpuid, &shutdown_cpus)) { 527 CPU_CLR_ATOMIC(cpuid, &shutdown_cpus); 528 (void)intr_disable(); 529 for (;;) 530 ; 531 } 532 } 533 CPU_CLR_ATOMIC(cpuid, &started_cpus); 534 CPU_CLR_ATOMIC(cpuid, &stopped_cpus); 535 sched_unpin(); 536 CTR2(KTR_SMP, "%s: restarted %d", __func__, curcpu); 537} 538 539static void 540cpu_ipi_preempt(struct trapframe *tf __unused) 541{ 542 543 sched_preempt(curthread); 544} 545 546static void 547cpu_ipi_hardclock(struct trapframe *tf) 548{ 549 struct trapframe *oldframe; 550 struct thread *td; 551 552 critical_enter(); 553 td = curthread; 554 td->td_intr_nesting_level++; 555 oldframe = td->td_intr_frame; 556 td->td_intr_frame = tf; 557 hardclockintr(); 558 td->td_intr_frame = oldframe; 559 td->td_intr_nesting_level--; 560 critical_exit(); 561} 562 563static void 564spitfire_ipi_selected(cpuset_t cpus, u_long d0, u_long d1, u_long d2) 565{ 566 u_int cpu; 567 568 while ((cpu = CPU_FFS(&cpus)) != 0) { 569 cpu--; 570 CPU_CLR(cpu, &cpus); 571 spitfire_ipi_single(cpu, d0, d1, d2); 572 } 573} 574 575static void 576spitfire_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2) 577{ 578 register_t s; 579 u_long ids; 580 u_int mid; 581 int i; 582 583 mtx_assert(&ipi_mtx, MA_OWNED); 584 KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__)); 585 KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & IDR_BUSY) == 0, 586 ("%s: outstanding dispatch", __func__)); 587 588 mid = cpuid_to_mid[cpu]; 589 for (i = 0; i < IPI_RETRIES; i++) { 590 s = intr_disable(); 591 stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 592 stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 593 stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 594 membar(Sync); 595 stxa(AA_INTR_SEND | (mid << IDC_ITID_SHIFT), 596 ASI_SDB_INTR_W, 0); 597 /* 598 * Workaround for SpitFire erratum #54; do a dummy read 599 * from a SDB internal register before the MEMBAR #Sync 600 * for the write to ASI_SDB_INTR_W (requiring another 601 * MEMBAR #Sync in order to make sure the write has 602 * occurred before the load). 603 */ 604 membar(Sync); 605 (void)ldxa(AA_SDB_CNTL_HIGH, ASI_SDB_CONTROL_R); 606 membar(Sync); 607 while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 608 IDR_BUSY) != 0) 609 ; 610 intr_restore(s); 611 if ((ids & (IDR_BUSY | IDR_NACK)) == 0) 612 return; 613 } 614 if (kdb_active != 0 || panicstr != NULL) 615 printf("%s: couldn't send IPI to module 0x%u\n", 616 __func__, mid); 617 else 618 panic("%s: couldn't send IPI to module 0x%u", 619 __func__, mid); 620} 621 622static void 623cheetah_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2) 624{ 625 register_t s; 626 u_long ids; 627 u_int mid; 628 int i; 629 630 mtx_assert(&ipi_mtx, MA_OWNED); 631 KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__)); 632 KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & 633 IDR_CHEETAH_ALL_BUSY) == 0, 634 ("%s: outstanding dispatch", __func__)); 635 636 mid = cpuid_to_mid[cpu]; 637 for (i = 0; i < IPI_RETRIES; i++) { 638 s = intr_disable(); 639 stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 640 stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 641 stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 642 membar(Sync); 643 stxa(AA_INTR_SEND | (mid << IDC_ITID_SHIFT), 644 ASI_SDB_INTR_W, 0); 645 membar(Sync); 646 while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 647 IDR_BUSY) != 0) 648 ; 649 intr_restore(s); 650 if ((ids & (IDR_BUSY | IDR_NACK)) == 0) 651 return; 652 } 653 if (kdb_active != 0 || panicstr != NULL) 654 printf("%s: couldn't send IPI to module 0x%u\n", 655 __func__, mid); 656 else 657 panic("%s: couldn't send IPI to module 0x%u", 658 __func__, mid); 659} 660 661static void 662cheetah_ipi_selected(cpuset_t cpus, u_long d0, u_long d1, u_long d2) 663{ 664 register_t s; 665 u_long ids; 666 u_int bnp; 667 u_int cpu; 668 int i; 669 670 mtx_assert(&ipi_mtx, MA_OWNED); 671 KASSERT(!CPU_EMPTY(&cpus), ("%s: no CPUs to IPI", __func__)); 672 KASSERT(!CPU_ISSET(curcpu, &cpus), ("%s: CPU can't IPI itself", 673 __func__)); 674 KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & 675 IDR_CHEETAH_ALL_BUSY) == 0, 676 ("%s: outstanding dispatch", __func__)); 677 678 ids = 0; 679 for (i = 0; i < IPI_RETRIES * smp_cpus; i++) { 680 s = intr_disable(); 681 stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 682 stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 683 stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 684 membar(Sync); 685 bnp = 0; 686 for (cpu = 0; cpu < smp_cpus; cpu++) { 687 if (CPU_ISSET(cpu, &cpus)) { 688 stxa(AA_INTR_SEND | (cpuid_to_mid[cpu] << 689 IDC_ITID_SHIFT) | bnp << IDC_BN_SHIFT, 690 ASI_SDB_INTR_W, 0); 691 membar(Sync); 692 bnp++; 693 if (bnp == IDR_CHEETAH_MAX_BN_PAIRS) 694 break; 695 } 696 } 697 while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 698 IDR_CHEETAH_ALL_BUSY) != 0) 699 ; 700 intr_restore(s); 701 bnp = 0; 702 for (cpu = 0; cpu < smp_cpus; cpu++) { 703 if (CPU_ISSET(cpu, &cpus)) { 704 if ((ids & (IDR_NACK << (2 * bnp))) == 0) 705 CPU_CLR(cpu, &cpus); 706 bnp++; 707 } 708 } 709 if (CPU_EMPTY(&cpus)) 710 return; 711 } 712 if (kdb_active != 0 || panicstr != NULL) 713 printf("%s: couldn't send IPI (cpus=%s ids=0x%lu)\n", 714 __func__, cpusetobj_strprint(ipi_pbuf, &cpus), ids); 715 else 716 panic("%s: couldn't send IPI (cpus=%s ids=0x%lu)", 717 __func__, cpusetobj_strprint(ipi_pbuf, &cpus), ids); 718} 719 720static void 721jalapeno_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2) 722{ 723 register_t s; 724 u_long ids; 725 u_int busy, busynack, mid; 726 int i; 727 728 mtx_assert(&ipi_mtx, MA_OWNED); 729 KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__)); 730 KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & 731 IDR_CHEETAH_ALL_BUSY) == 0, 732 ("%s: outstanding dispatch", __func__)); 733 734 mid = cpuid_to_mid[cpu]; 735 busy = IDR_BUSY << (2 * mid); 736 busynack = (IDR_BUSY | IDR_NACK) << (2 * mid); 737 for (i = 0; i < IPI_RETRIES; i++) { 738 s = intr_disable(); 739 stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 740 stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 741 stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 742 membar(Sync); 743 stxa(AA_INTR_SEND | (mid << IDC_ITID_SHIFT), 744 ASI_SDB_INTR_W, 0); 745 membar(Sync); 746 while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 747 busy) != 0) 748 ; 749 intr_restore(s); 750 if ((ids & busynack) == 0) 751 return; 752 } 753 if (kdb_active != 0 || panicstr != NULL) 754 printf("%s: couldn't send IPI to module 0x%u\n", 755 __func__, mid); 756 else 757 panic("%s: couldn't send IPI to module 0x%u", 758 __func__, mid); 759} 760 761static void 762jalapeno_ipi_selected(cpuset_t cpus, u_long d0, u_long d1, u_long d2) 763{ 764 register_t s; 765 u_long ids; 766 u_int cpu; 767 int i; 768 769 mtx_assert(&ipi_mtx, MA_OWNED); 770 KASSERT(!CPU_EMPTY(&cpus), ("%s: no CPUs to IPI", __func__)); 771 KASSERT(!CPU_ISSET(curcpu, &cpus), ("%s: CPU can't IPI itself", 772 __func__)); 773 KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & 774 IDR_CHEETAH_ALL_BUSY) == 0, 775 ("%s: outstanding dispatch", __func__)); 776 777 ids = 0; 778 for (i = 0; i < IPI_RETRIES * smp_cpus; i++) { 779 s = intr_disable(); 780 stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 781 stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 782 stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 783 membar(Sync); 784 for (cpu = 0; cpu < smp_cpus; cpu++) { 785 if (CPU_ISSET(cpu, &cpus)) { 786 stxa(AA_INTR_SEND | (cpuid_to_mid[cpu] << 787 IDC_ITID_SHIFT), ASI_SDB_INTR_W, 0); 788 membar(Sync); 789 } 790 } 791 while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 792 IDR_CHEETAH_ALL_BUSY) != 0) 793 ; 794 intr_restore(s); 795 if ((ids & 796 (IDR_CHEETAH_ALL_BUSY | IDR_CHEETAH_ALL_NACK)) == 0) 797 return; 798 for (cpu = 0; cpu < smp_cpus; cpu++) 799 if (CPU_ISSET(cpu, &cpus)) 800 if ((ids & (IDR_NACK << 801 (2 * cpuid_to_mid[cpu]))) == 0) 802 CPU_CLR(cpu, &cpus); 803 } 804 if (kdb_active != 0 || panicstr != NULL) 805 printf("%s: couldn't send IPI (cpus=%s ids=0x%lu)\n", 806 __func__, cpusetobj_strprint(ipi_pbuf, &cpus), ids); 807 else 808 panic("%s: couldn't send IPI (cpus=%s ids=0x%lu)", 809 __func__, cpusetobj_strprint(ipi_pbuf, &cpus), ids); 810} 811