mp_machdep.c revision 286055
1/*- 2 * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 3. Berkeley Software Design Inc's name may not be used to endorse or 13 * promote products derived from this software without specific prior 14 * written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * from BSDI: locore.s,v 1.36.2.15 1999/08/23 22:34:41 cp Exp 29 */ 30/*- 31 * Copyright (c) 2002 Jake Burkholder. 32 * Copyright (c) 2007 - 2010 Marius Strobl <marius@FreeBSD.org> 33 * All rights reserved. 34 * 35 * Redistribution and use in source and binary forms, with or without 36 * modification, are permitted provided that the following conditions 37 * are met: 38 * 1. Redistributions of source code must retain the above copyright 39 * notice, this list of conditions and the following disclaimer. 40 * 2. Redistributions in binary form must reproduce the above copyright 41 * notice, this list of conditions and the following disclaimer in the 42 * documentation and/or other materials provided with the distribution. 43 * 44 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 45 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 46 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 47 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 48 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 49 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 50 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 51 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 52 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 53 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 54 * SUCH DAMAGE. 55 */ 56 57#include <sys/cdefs.h> 58__FBSDID("$FreeBSD: stable/10/sys/sparc64/sparc64/mp_machdep.c 286055 2015-07-30 02:06:29Z marius $"); 59 60#include <sys/param.h> 61#include <sys/systm.h> 62#include <sys/lock.h> 63#include <sys/kdb.h> 64#include <sys/kernel.h> 65#include <sys/ktr.h> 66#include <sys/mutex.h> 67#include <sys/pcpu.h> 68#include <sys/proc.h> 69#include <sys/sched.h> 70#include <sys/smp.h> 71 72#include <vm/vm.h> 73#include <vm/vm_param.h> 74#include <vm/pmap.h> 75#include <vm/vm_kern.h> 76#include <vm/vm_extern.h> 77#include <vm/vm_map.h> 78 79#include <dev/ofw/openfirm.h> 80 81#include <machine/asi.h> 82#include <machine/atomic.h> 83#include <machine/bus.h> 84#include <machine/cpu.h> 85#include <machine/cpufunc.h> 86#include <machine/md_var.h> 87#include <machine/metadata.h> 88#include <machine/ofw_machdep.h> 89#include <machine/pcb.h> 90#include <machine/smp.h> 91#include <machine/tick.h> 92#include <machine/tlb.h> 93#include <machine/tsb.h> 94#include <machine/tte.h> 95#include <machine/ver.h> 96 97#define SUNW_STARTCPU "SUNW,start-cpu" 98#define SUNW_STOPSELF "SUNW,stop-self" 99 100static ih_func_t cpu_ipi_ast; 101static ih_func_t cpu_ipi_hardclock; 102static ih_func_t cpu_ipi_preempt; 103static ih_func_t cpu_ipi_stop; 104 105/* 106 * Argument area used to pass data to non-boot processors as they start up. 107 * This must be statically initialized with a known invalid CPU module ID, 108 * since the other processors will use it before the boot CPU enters the 109 * kernel. 110 */ 111struct cpu_start_args cpu_start_args = { 0, -1, -1, 0, 0, 0 }; 112struct ipi_cache_args ipi_cache_args; 113struct ipi_rd_args ipi_rd_args; 114struct ipi_tlb_args ipi_tlb_args; 115struct pcb stoppcbs[MAXCPU]; 116 117struct mtx ipi_mtx; 118 119cpu_ipi_selected_t *cpu_ipi_selected; 120cpu_ipi_single_t *cpu_ipi_single; 121 122static vm_offset_t mp_tramp; 123static u_int cpuid_to_mid[MAXCPU]; 124static volatile cpuset_t shutdown_cpus; 125 126static void ap_count(phandle_t node, u_int mid, u_int cpu_impl); 127static void ap_start(phandle_t node, u_int mid, u_int cpu_impl); 128static void cpu_mp_unleash(void *v); 129static void foreach_ap(phandle_t node, void (*func)(phandle_t node, 130 u_int mid, u_int cpu_impl)); 131static void sun4u_startcpu(phandle_t cpu, void *func, u_long arg); 132 133static cpu_ipi_selected_t cheetah_ipi_selected; 134static cpu_ipi_single_t cheetah_ipi_single; 135static cpu_ipi_selected_t jalapeno_ipi_selected; 136static cpu_ipi_single_t jalapeno_ipi_single; 137static cpu_ipi_selected_t spitfire_ipi_selected; 138static cpu_ipi_single_t spitfire_ipi_single; 139 140SYSINIT(cpu_mp_unleash, SI_SUB_SMP, SI_ORDER_FIRST, cpu_mp_unleash, NULL); 141 142void 143mp_init(void) 144{ 145 struct tte *tp; 146 int i; 147 148 mp_tramp = (vm_offset_t)OF_claim(NULL, PAGE_SIZE, PAGE_SIZE); 149 if (mp_tramp == (vm_offset_t)-1) 150 panic("%s", __func__); 151 bcopy(mp_tramp_code, (void *)mp_tramp, mp_tramp_code_len); 152 *(vm_offset_t *)(mp_tramp + mp_tramp_tlb_slots) = kernel_tlb_slots; 153 *(vm_offset_t *)(mp_tramp + mp_tramp_func) = (vm_offset_t)mp_startup; 154 tp = (struct tte *)(mp_tramp + mp_tramp_code_len); 155 for (i = 0; i < kernel_tlb_slots; i++) { 156 tp[i].tte_vpn = TV_VPN(kernel_tlbs[i].te_va, TS_4M); 157 tp[i].tte_data = TD_V | TD_4M | TD_PA(kernel_tlbs[i].te_pa) | 158 TD_L | TD_CP | TD_CV | TD_P | TD_W; 159 } 160 for (i = 0; i < PAGE_SIZE; i += sizeof(vm_offset_t)) 161 flush(mp_tramp + i); 162} 163 164static void 165foreach_ap(phandle_t node, void (*func)(phandle_t node, u_int mid, 166 u_int cpu_impl)) 167{ 168 char type[sizeof("cpu")]; 169 phandle_t child; 170 u_int cpuid; 171 uint32_t cpu_impl; 172 173 /* There's no need to traverse the whole OFW tree twice. */ 174 if (mp_maxid > 0 && mp_ncpus >= mp_maxid + 1) 175 return; 176 177 for (; node != 0; node = OF_peer(node)) { 178 child = OF_child(node); 179 if (child > 0) 180 foreach_ap(child, func); 181 else { 182 if (OF_getprop(node, "device_type", type, 183 sizeof(type)) <= 0) 184 continue; 185 if (strcmp(type, "cpu") != 0) 186 continue; 187 if (OF_getprop(node, "implementation#", &cpu_impl, 188 sizeof(cpu_impl)) <= 0) 189 panic("%s: couldn't determine CPU " 190 "implementation", __func__); 191 if (OF_getprop(node, cpu_cpuid_prop(cpu_impl), &cpuid, 192 sizeof(cpuid)) <= 0) 193 panic("%s: couldn't determine CPU module ID", 194 __func__); 195 if (cpuid == PCPU_GET(mid)) 196 continue; 197 (*func)(node, cpuid, cpu_impl); 198 } 199 } 200} 201 202/* 203 * Probe for other CPUs. 204 */ 205void 206cpu_mp_setmaxid(void) 207{ 208 209 CPU_SETOF(curcpu, &all_cpus); 210 mp_ncpus = 1; 211 mp_maxid = 0; 212 213 foreach_ap(OF_child(OF_peer(0)), ap_count); 214} 215 216static void 217ap_count(phandle_t node __unused, u_int mid __unused, u_int cpu_impl __unused) 218{ 219 220 mp_maxid++; 221} 222 223int 224cpu_mp_probe(void) 225{ 226 227 return (mp_maxid > 0); 228} 229 230struct cpu_group * 231cpu_topo(void) 232{ 233 234 return (smp_topo_none()); 235} 236 237static void 238sun4u_startcpu(phandle_t cpu, void *func, u_long arg) 239{ 240 static struct { 241 cell_t name; 242 cell_t nargs; 243 cell_t nreturns; 244 cell_t cpu; 245 cell_t func; 246 cell_t arg; 247 } args = { 248 (cell_t)SUNW_STARTCPU, 249 3, 250 }; 251 252 args.cpu = cpu; 253 args.func = (cell_t)func; 254 args.arg = (cell_t)arg; 255 ofw_entry(&args); 256} 257 258/* 259 * Fire up any non-boot processors. 260 */ 261void 262cpu_mp_start(void) 263{ 264 u_int cpu_impl, isjbus; 265 266 mtx_init(&ipi_mtx, "ipi", NULL, MTX_SPIN); 267 268 isjbus = 0; 269 cpu_impl = PCPU_GET(impl); 270 if (cpu_impl == CPU_IMPL_ULTRASPARCIIIi || 271 cpu_impl == CPU_IMPL_ULTRASPARCIIIip) { 272 isjbus = 1; 273 cpu_ipi_selected = jalapeno_ipi_selected; 274 cpu_ipi_single = jalapeno_ipi_single; 275 } else if (cpu_impl == CPU_IMPL_SPARC64V || 276 cpu_impl >= CPU_IMPL_ULTRASPARCIII) { 277 cpu_ipi_selected = cheetah_ipi_selected; 278 cpu_ipi_single = cheetah_ipi_single; 279 } else { 280 cpu_ipi_selected = spitfire_ipi_selected; 281 cpu_ipi_single = spitfire_ipi_single; 282 } 283 284 intr_setup(PIL_AST, cpu_ipi_ast, -1, NULL, NULL); 285 intr_setup(PIL_RENDEZVOUS, (ih_func_t *)smp_rendezvous_action, 286 -1, NULL, NULL); 287 intr_setup(PIL_STOP, cpu_ipi_stop, -1, NULL, NULL); 288 intr_setup(PIL_PREEMPT, cpu_ipi_preempt, -1, NULL, NULL); 289 intr_setup(PIL_HARDCLOCK, cpu_ipi_hardclock, -1, NULL, NULL); 290 291 cpuid_to_mid[curcpu] = PCPU_GET(mid); 292 293 foreach_ap(OF_child(OF_peer(0)), ap_start); 294 KASSERT(!isjbus || mp_ncpus <= IDR_JALAPENO_MAX_BN_PAIRS, 295 ("%s: can only IPI a maximum of %d JBus-CPUs", 296 __func__, IDR_JALAPENO_MAX_BN_PAIRS)); 297} 298 299static void 300ap_start(phandle_t node, u_int mid, u_int cpu_impl) 301{ 302 volatile struct cpu_start_args *csa; 303 struct pcpu *pc; 304 register_t s; 305 vm_offset_t va; 306 u_int cpuid; 307 uint32_t clock; 308 309 if (mp_ncpus > MAXCPU) 310 return; 311 312 if (OF_getprop(node, "clock-frequency", &clock, sizeof(clock)) <= 0) 313 panic("%s: couldn't determine CPU frequency", __func__); 314 if (clock != PCPU_GET(clock)) 315 tick_et_use_stick = 1; 316 317 csa = &cpu_start_args; 318 csa->csa_state = 0; 319 sun4u_startcpu(node, (void *)mp_tramp, 0); 320 s = intr_disable(); 321 while (csa->csa_state != CPU_TICKSYNC) 322 ; 323 membar(StoreLoad); 324 csa->csa_tick = rd(tick); 325 if (cpu_impl == CPU_IMPL_SPARC64V || 326 cpu_impl >= CPU_IMPL_ULTRASPARCIII) { 327 while (csa->csa_state != CPU_STICKSYNC) 328 ; 329 membar(StoreLoad); 330 csa->csa_stick = rdstick(); 331 } 332 while (csa->csa_state != CPU_INIT) 333 ; 334 csa->csa_tick = csa->csa_stick = 0; 335 intr_restore(s); 336 337 cpuid = mp_ncpus++; 338 cpuid_to_mid[cpuid] = mid; 339 cpu_identify(csa->csa_ver, clock, cpuid); 340 341 va = kmem_malloc(kernel_arena, PCPU_PAGES * PAGE_SIZE, 342 M_WAITOK | M_ZERO); 343 pc = (struct pcpu *)(va + (PCPU_PAGES * PAGE_SIZE)) - 1; 344 pcpu_init(pc, cpuid, sizeof(*pc)); 345 dpcpu_init((void *)kmem_malloc(kernel_arena, DPCPU_SIZE, 346 M_WAITOK | M_ZERO), cpuid); 347 pc->pc_addr = va; 348 pc->pc_clock = clock; 349 pc->pc_impl = cpu_impl; 350 pc->pc_mid = mid; 351 pc->pc_node = node; 352 353 cache_init(pc); 354 355 CPU_SET(cpuid, &all_cpus); 356 intr_add_cpu(cpuid); 357} 358 359void 360cpu_mp_announce(void) 361{ 362 363} 364 365static void 366cpu_mp_unleash(void *v __unused) 367{ 368 volatile struct cpu_start_args *csa; 369 struct pcpu *pc; 370 register_t s; 371 vm_offset_t va; 372 vm_paddr_t pa; 373 u_int ctx_inc; 374 u_int ctx_min; 375 int i; 376 377 ctx_min = TLB_CTX_USER_MIN; 378 ctx_inc = (TLB_CTX_USER_MAX - 1) / mp_ncpus; 379 csa = &cpu_start_args; 380 csa->csa_count = mp_ncpus; 381 STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { 382 pc->pc_tlb_ctx = ctx_min; 383 pc->pc_tlb_ctx_min = ctx_min; 384 pc->pc_tlb_ctx_max = ctx_min + ctx_inc; 385 ctx_min += ctx_inc; 386 387 if (pc->pc_cpuid == curcpu) 388 continue; 389 KASSERT(pc->pc_idlethread != NULL, 390 ("%s: idlethread", __func__)); 391 pc->pc_curthread = pc->pc_idlethread; 392 pc->pc_curpcb = pc->pc_curthread->td_pcb; 393 for (i = 0; i < PCPU_PAGES; i++) { 394 va = pc->pc_addr + i * PAGE_SIZE; 395 pa = pmap_kextract(va); 396 if (pa == 0) 397 panic("%s: pmap_kextract", __func__); 398 csa->csa_ttes[i].tte_vpn = TV_VPN(va, TS_8K); 399 csa->csa_ttes[i].tte_data = TD_V | TD_8K | TD_PA(pa) | 400 TD_L | TD_CP | TD_CV | TD_P | TD_W; 401 } 402 csa->csa_state = 0; 403 csa->csa_pcpu = pc->pc_addr; 404 csa->csa_mid = pc->pc_mid; 405 s = intr_disable(); 406 while (csa->csa_state != CPU_BOOTSTRAP) 407 ; 408 intr_restore(s); 409 } 410 411 membar(StoreLoad); 412 csa->csa_count = 0; 413} 414 415void 416cpu_mp_bootstrap(struct pcpu *pc) 417{ 418 volatile struct cpu_start_args *csa; 419 420 csa = &cpu_start_args; 421 422 /* Do CPU-specific initialization. */ 423 if (pc->pc_impl >= CPU_IMPL_ULTRASPARCIII) 424 cheetah_init(pc->pc_impl); 425 else if (pc->pc_impl == CPU_IMPL_SPARC64V) 426 zeus_init(pc->pc_impl); 427 428 /* 429 * Enable the caches. Note that his may include applying workarounds. 430 */ 431 cache_enable(pc->pc_impl); 432 433 /* 434 * Clear (S)TICK timer(s) (including NPT) and ensure they are stopped. 435 */ 436 tick_clear(pc->pc_impl); 437 tick_stop(pc->pc_impl); 438 439 /* Set the kernel context. */ 440 pmap_set_kctx(); 441 442 /* Lock the kernel TSB in the TLB if necessary. */ 443 if (tsb_kernel_ldd_phys == 0) 444 pmap_map_tsb(); 445 446 /* 447 * Flush all non-locked TLB entries possibly left over by the 448 * firmware. 449 */ 450 tlb_flush_nonlocked(); 451 452 /* 453 * Enable interrupts. 454 * Note that the PIL we be lowered indirectly via sched_throw(NULL) 455 * when fake spinlock held by the idle thread eventually is released. 456 */ 457 wrpr(pstate, 0, PSTATE_KERNEL); 458 459 smp_cpus++; 460 KASSERT(curthread != NULL, ("%s: curthread", __func__)); 461 printf("SMP: AP CPU #%d Launched!\n", curcpu); 462 463 csa->csa_count--; 464 membar(StoreLoad); 465 csa->csa_state = CPU_BOOTSTRAP; 466 while (csa->csa_count != 0) 467 ; 468 469 if (smp_cpus == mp_ncpus) 470 atomic_store_rel_int(&smp_started, 1); 471 472 /* Start per-CPU event timers. */ 473 cpu_initclocks_ap(); 474 475 /* Ok, now enter the scheduler. */ 476 sched_throw(NULL); 477} 478 479void 480cpu_mp_shutdown(void) 481{ 482 cpuset_t cpus; 483 int i; 484 485 critical_enter(); 486 shutdown_cpus = all_cpus; 487 CPU_CLR(PCPU_GET(cpuid), &shutdown_cpus); 488 cpus = shutdown_cpus; 489 490 /* XXX: Stop all the CPUs which aren't already. */ 491 if (CPU_CMP(&stopped_cpus, &cpus)) { 492 493 /* cpus is just a flat "on" mask without curcpu. */ 494 CPU_NAND(&cpus, &stopped_cpus); 495 stop_cpus(cpus); 496 } 497 i = 0; 498 while (!CPU_EMPTY(&shutdown_cpus)) { 499 if (i++ > 100000) { 500 printf("timeout shutting down CPUs.\n"); 501 break; 502 } 503 } 504 critical_exit(); 505} 506 507static void 508cpu_ipi_ast(struct trapframe *tf __unused) 509{ 510 511} 512 513static void 514cpu_ipi_stop(struct trapframe *tf __unused) 515{ 516 u_int cpuid; 517 518 CTR2(KTR_SMP, "%s: stopped %d", __func__, curcpu); 519 sched_pin(); 520 savectx(&stoppcbs[curcpu]); 521 cpuid = PCPU_GET(cpuid); 522 CPU_SET_ATOMIC(cpuid, &stopped_cpus); 523 while (!CPU_ISSET(cpuid, &started_cpus)) { 524 if (CPU_ISSET(cpuid, &shutdown_cpus)) { 525 CPU_CLR_ATOMIC(cpuid, &shutdown_cpus); 526 (void)intr_disable(); 527 for (;;) 528 ; 529 } 530 } 531 CPU_CLR_ATOMIC(cpuid, &started_cpus); 532 CPU_CLR_ATOMIC(cpuid, &stopped_cpus); 533 sched_unpin(); 534 CTR2(KTR_SMP, "%s: restarted %d", __func__, curcpu); 535} 536 537static void 538cpu_ipi_preempt(struct trapframe *tf __unused) 539{ 540 541 sched_preempt(curthread); 542} 543 544static void 545cpu_ipi_hardclock(struct trapframe *tf) 546{ 547 struct trapframe *oldframe; 548 struct thread *td; 549 550 critical_enter(); 551 td = curthread; 552 td->td_intr_nesting_level++; 553 oldframe = td->td_intr_frame; 554 td->td_intr_frame = tf; 555 hardclockintr(); 556 td->td_intr_frame = oldframe; 557 td->td_intr_nesting_level--; 558 critical_exit(); 559} 560 561static void 562spitfire_ipi_selected(cpuset_t cpus, u_long d0, u_long d1, u_long d2) 563{ 564 u_int cpu; 565 566 while ((cpu = CPU_FFS(&cpus)) != 0) { 567 cpu--; 568 CPU_CLR(cpu, &cpus); 569 spitfire_ipi_single(cpu, d0, d1, d2); 570 } 571} 572 573static void 574spitfire_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2) 575{ 576 register_t s; 577 u_long ids; 578 u_int mid; 579 int i; 580 581 mtx_assert(&ipi_mtx, MA_OWNED); 582 KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__)); 583 KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & IDR_BUSY) == 0, 584 ("%s: outstanding dispatch", __func__)); 585 586 mid = cpuid_to_mid[cpu]; 587 for (i = 0; i < IPI_RETRIES; i++) { 588 s = intr_disable(); 589 stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 590 stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 591 stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 592 membar(Sync); 593 stxa(AA_INTR_SEND | (mid << IDC_ITID_SHIFT), 594 ASI_SDB_INTR_W, 0); 595 /* 596 * Workaround for SpitFire erratum #54; do a dummy read 597 * from a SDB internal register before the MEMBAR #Sync 598 * for the write to ASI_SDB_INTR_W (requiring another 599 * MEMBAR #Sync in order to make sure the write has 600 * occurred before the load). 601 */ 602 membar(Sync); 603 (void)ldxa(AA_SDB_CNTL_HIGH, ASI_SDB_CONTROL_R); 604 membar(Sync); 605 while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 606 IDR_BUSY) != 0) 607 ; 608 intr_restore(s); 609 if ((ids & (IDR_BUSY | IDR_NACK)) == 0) 610 return; 611 } 612 if (kdb_active != 0 || panicstr != NULL) 613 printf("%s: couldn't send IPI to module 0x%u\n", 614 __func__, mid); 615 else 616 panic("%s: couldn't send IPI to module 0x%u", 617 __func__, mid); 618} 619 620static void 621cheetah_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2) 622{ 623 register_t s; 624 u_long ids; 625 u_int mid; 626 int i; 627 628 mtx_assert(&ipi_mtx, MA_OWNED); 629 KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__)); 630 KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & 631 IDR_CHEETAH_ALL_BUSY) == 0, 632 ("%s: outstanding dispatch", __func__)); 633 634 mid = cpuid_to_mid[cpu]; 635 for (i = 0; i < IPI_RETRIES; i++) { 636 s = intr_disable(); 637 stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 638 stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 639 stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 640 membar(Sync); 641 stxa(AA_INTR_SEND | (mid << IDC_ITID_SHIFT), 642 ASI_SDB_INTR_W, 0); 643 membar(Sync); 644 while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 645 IDR_BUSY) != 0) 646 ; 647 intr_restore(s); 648 if ((ids & (IDR_BUSY | IDR_NACK)) == 0) 649 return; 650 } 651 if (kdb_active != 0 || panicstr != NULL) 652 printf("%s: couldn't send IPI to module 0x%u\n", 653 __func__, mid); 654 else 655 panic("%s: couldn't send IPI to module 0x%u", 656 __func__, mid); 657} 658 659static void 660cheetah_ipi_selected(cpuset_t cpus, u_long d0, u_long d1, u_long d2) 661{ 662 char pbuf[CPUSETBUFSIZ]; 663 register_t s; 664 u_long ids; 665 u_int bnp; 666 u_int cpu; 667 int i; 668 669 mtx_assert(&ipi_mtx, MA_OWNED); 670 KASSERT(!CPU_EMPTY(&cpus), ("%s: no CPUs to IPI", __func__)); 671 KASSERT(!CPU_ISSET(curcpu, &cpus), ("%s: CPU can't IPI itself", 672 __func__)); 673 KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & 674 IDR_CHEETAH_ALL_BUSY) == 0, 675 ("%s: outstanding dispatch", __func__)); 676 677 ids = 0; 678 for (i = 0; i < IPI_RETRIES * mp_ncpus; i++) { 679 s = intr_disable(); 680 stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 681 stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 682 stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 683 membar(Sync); 684 bnp = 0; 685 for (cpu = 0; cpu < mp_ncpus; cpu++) { 686 if (CPU_ISSET(cpu, &cpus)) { 687 stxa(AA_INTR_SEND | (cpuid_to_mid[cpu] << 688 IDC_ITID_SHIFT) | bnp << IDC_BN_SHIFT, 689 ASI_SDB_INTR_W, 0); 690 membar(Sync); 691 bnp++; 692 if (bnp == IDR_CHEETAH_MAX_BN_PAIRS) 693 break; 694 } 695 } 696 while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 697 IDR_CHEETAH_ALL_BUSY) != 0) 698 ; 699 intr_restore(s); 700 bnp = 0; 701 for (cpu = 0; cpu < mp_ncpus; cpu++) { 702 if (CPU_ISSET(cpu, &cpus)) { 703 if ((ids & (IDR_NACK << (2 * bnp))) == 0) 704 CPU_CLR(cpu, &cpus); 705 bnp++; 706 } 707 } 708 if (CPU_EMPTY(&cpus)) 709 return; 710 } 711 if (kdb_active != 0 || panicstr != NULL) 712 printf("%s: couldn't send IPI (cpus=%s ids=0x%lu)\n", 713 __func__, cpusetobj_strprint(pbuf, &cpus), ids); 714 else 715 panic("%s: couldn't send IPI (cpus=%s ids=0x%lu)", 716 __func__, cpusetobj_strprint(pbuf, &cpus), ids); 717} 718 719static void 720jalapeno_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2) 721{ 722 register_t s; 723 u_long ids; 724 u_int busy, busynack, mid; 725 int i; 726 727 mtx_assert(&ipi_mtx, MA_OWNED); 728 KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__)); 729 KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & 730 IDR_CHEETAH_ALL_BUSY) == 0, 731 ("%s: outstanding dispatch", __func__)); 732 733 mid = cpuid_to_mid[cpu]; 734 busy = IDR_BUSY << (2 * mid); 735 busynack = (IDR_BUSY | IDR_NACK) << (2 * mid); 736 for (i = 0; i < IPI_RETRIES; i++) { 737 s = intr_disable(); 738 stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 739 stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 740 stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 741 membar(Sync); 742 stxa(AA_INTR_SEND | (mid << IDC_ITID_SHIFT), 743 ASI_SDB_INTR_W, 0); 744 membar(Sync); 745 while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 746 busy) != 0) 747 ; 748 intr_restore(s); 749 if ((ids & busynack) == 0) 750 return; 751 } 752 if (kdb_active != 0 || panicstr != NULL) 753 printf("%s: couldn't send IPI to module 0x%u\n", 754 __func__, mid); 755 else 756 panic("%s: couldn't send IPI to module 0x%u", 757 __func__, mid); 758} 759 760static void 761jalapeno_ipi_selected(cpuset_t cpus, u_long d0, u_long d1, u_long d2) 762{ 763 char pbuf[CPUSETBUFSIZ]; 764 register_t s; 765 u_long ids; 766 u_int cpu; 767 int i; 768 769 mtx_assert(&ipi_mtx, MA_OWNED); 770 KASSERT(!CPU_EMPTY(&cpus), ("%s: no CPUs to IPI", __func__)); 771 KASSERT(!CPU_ISSET(curcpu, &cpus), ("%s: CPU can't IPI itself", 772 __func__)); 773 KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & 774 IDR_CHEETAH_ALL_BUSY) == 0, 775 ("%s: outstanding dispatch", __func__)); 776 777 ids = 0; 778 for (i = 0; i < IPI_RETRIES * mp_ncpus; i++) { 779 s = intr_disable(); 780 stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 781 stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 782 stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 783 membar(Sync); 784 for (cpu = 0; cpu < mp_ncpus; cpu++) { 785 if (CPU_ISSET(cpu, &cpus)) { 786 stxa(AA_INTR_SEND | (cpuid_to_mid[cpu] << 787 IDC_ITID_SHIFT), ASI_SDB_INTR_W, 0); 788 membar(Sync); 789 } 790 } 791 while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 792 IDR_CHEETAH_ALL_BUSY) != 0) 793 ; 794 intr_restore(s); 795 if ((ids & 796 (IDR_CHEETAH_ALL_BUSY | IDR_CHEETAH_ALL_NACK)) == 0) 797 return; 798 for (cpu = 0; cpu < mp_ncpus; cpu++) 799 if (CPU_ISSET(cpu, &cpus)) 800 if ((ids & (IDR_NACK << 801 (2 * cpuid_to_mid[cpu]))) == 0) 802 CPU_CLR(cpu, &cpus); 803 } 804 if (kdb_active != 0 || panicstr != NULL) 805 printf("%s: couldn't send IPI (cpus=%s ids=0x%lu)\n", 806 __func__, cpusetobj_strprint(pbuf, &cpus), ids); 807 else 808 panic("%s: couldn't send IPI (cpus=%s ids=0x%lu)", 809 __func__, cpusetobj_strprint(pbuf, &cpus), ids); 810} 811