mp_machdep.c revision 265606
1/*- 2 * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 3. Berkeley Software Design Inc's name may not be used to endorse or 13 * promote products derived from this software without specific prior 14 * written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * from BSDI: locore.s,v 1.36.2.15 1999/08/23 22:34:41 cp Exp 29 */ 30/*- 31 * Copyright (c) 2002 Jake Burkholder. 32 * Copyright (c) 2007 - 2010 Marius Strobl <marius@FreeBSD.org> 33 * All rights reserved. 34 * 35 * Redistribution and use in source and binary forms, with or without 36 * modification, are permitted provided that the following conditions 37 * are met: 38 * 1. Redistributions of source code must retain the above copyright 39 * notice, this list of conditions and the following disclaimer. 40 * 2. Redistributions in binary form must reproduce the above copyright 41 * notice, this list of conditions and the following disclaimer in the 42 * documentation and/or other materials provided with the distribution. 43 * 44 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 45 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 46 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 47 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 48 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 49 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 50 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 51 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 52 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 53 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 54 * SUCH DAMAGE. 55 */ 56 57#include <sys/cdefs.h> 58__FBSDID("$FreeBSD: stable/10/sys/sparc64/sparc64/mp_machdep.c 265606 2014-05-07 20:28:27Z scottl $"); 59 60#include <sys/param.h> 61#include <sys/systm.h> 62#include <sys/lock.h> 63#include <sys/kdb.h> 64#include <sys/kernel.h> 65#include <sys/ktr.h> 66#include <sys/mutex.h> 67#include <sys/pcpu.h> 68#include <sys/proc.h> 69#include <sys/sched.h> 70#include <sys/smp.h> 71 72#include <vm/vm.h> 73#include <vm/vm_param.h> 74#include <vm/pmap.h> 75#include <vm/vm_kern.h> 76#include <vm/vm_extern.h> 77#include <vm/vm_map.h> 78 79#include <dev/ofw/openfirm.h> 80 81#include <machine/asi.h> 82#include <machine/atomic.h> 83#include <machine/bus.h> 84#include <machine/cpu.h> 85#include <machine/md_var.h> 86#include <machine/metadata.h> 87#include <machine/ofw_machdep.h> 88#include <machine/pcb.h> 89#include <machine/smp.h> 90#include <machine/tick.h> 91#include <machine/tlb.h> 92#include <machine/tsb.h> 93#include <machine/tte.h> 94#include <machine/ver.h> 95 96#define SUNW_STARTCPU "SUNW,start-cpu" 97#define SUNW_STOPSELF "SUNW,stop-self" 98 99static ih_func_t cpu_ipi_ast; 100static ih_func_t cpu_ipi_hardclock; 101static ih_func_t cpu_ipi_preempt; 102static ih_func_t cpu_ipi_stop; 103 104/* 105 * Argument area used to pass data to non-boot processors as they start up. 106 * This must be statically initialized with a known invalid CPU module ID, 107 * since the other processors will use it before the boot CPU enters the 108 * kernel. 109 */ 110struct cpu_start_args cpu_start_args = { 0, -1, -1, 0, 0, 0 }; 111struct ipi_cache_args ipi_cache_args; 112struct ipi_rd_args ipi_rd_args; 113struct ipi_tlb_args ipi_tlb_args; 114struct pcb stoppcbs[MAXCPU]; 115 116cpu_ipi_selected_t *cpu_ipi_selected; 117cpu_ipi_single_t *cpu_ipi_single; 118 119static vm_offset_t mp_tramp; 120static u_int cpuid_to_mid[MAXCPU]; 121static int isjbus; 122static volatile cpuset_t shutdown_cpus; 123 124static void ap_count(phandle_t node, u_int mid, u_int cpu_impl); 125static void ap_start(phandle_t node, u_int mid, u_int cpu_impl); 126static void cpu_mp_unleash(void *v); 127static void foreach_ap(phandle_t node, void (*func)(phandle_t node, 128 u_int mid, u_int cpu_impl)); 129static void sun4u_startcpu(phandle_t cpu, void *func, u_long arg); 130 131static cpu_ipi_selected_t cheetah_ipi_selected; 132static cpu_ipi_single_t cheetah_ipi_single; 133static cpu_ipi_selected_t jalapeno_ipi_selected; 134static cpu_ipi_single_t jalapeno_ipi_single; 135static cpu_ipi_selected_t spitfire_ipi_selected; 136static cpu_ipi_single_t spitfire_ipi_single; 137 138SYSINIT(cpu_mp_unleash, SI_SUB_SMP, SI_ORDER_FIRST, cpu_mp_unleash, NULL); 139 140void 141mp_init(u_int cpu_impl) 142{ 143 struct tte *tp; 144 int i; 145 146 mp_tramp = (vm_offset_t)OF_claim(NULL, PAGE_SIZE, PAGE_SIZE); 147 if (mp_tramp == (vm_offset_t)-1) 148 panic("%s", __func__); 149 bcopy(mp_tramp_code, (void *)mp_tramp, mp_tramp_code_len); 150 *(vm_offset_t *)(mp_tramp + mp_tramp_tlb_slots) = kernel_tlb_slots; 151 *(vm_offset_t *)(mp_tramp + mp_tramp_func) = (vm_offset_t)mp_startup; 152 tp = (struct tte *)(mp_tramp + mp_tramp_code_len); 153 for (i = 0; i < kernel_tlb_slots; i++) { 154 tp[i].tte_vpn = TV_VPN(kernel_tlbs[i].te_va, TS_4M); 155 tp[i].tte_data = TD_V | TD_4M | TD_PA(kernel_tlbs[i].te_pa) | 156 TD_L | TD_CP | TD_CV | TD_P | TD_W; 157 } 158 for (i = 0; i < PAGE_SIZE; i += sizeof(vm_offset_t)) 159 flush(mp_tramp + i); 160 161 /* 162 * On UP systems cpu_ipi_selected() can be called while 163 * cpu_mp_start() wasn't so initialize these here. 164 */ 165 if (cpu_impl == CPU_IMPL_ULTRASPARCIIIi || 166 cpu_impl == CPU_IMPL_ULTRASPARCIIIip) { 167 isjbus = 1; 168 cpu_ipi_selected = jalapeno_ipi_selected; 169 cpu_ipi_single = jalapeno_ipi_single; 170 } else if (cpu_impl == CPU_IMPL_SPARC64V || 171 cpu_impl >= CPU_IMPL_ULTRASPARCIII) { 172 cpu_ipi_selected = cheetah_ipi_selected; 173 cpu_ipi_single = cheetah_ipi_single; 174 } else { 175 cpu_ipi_selected = spitfire_ipi_selected; 176 cpu_ipi_single = spitfire_ipi_single; 177 } 178} 179 180static void 181foreach_ap(phandle_t node, void (*func)(phandle_t node, u_int mid, 182 u_int cpu_impl)) 183{ 184 char type[sizeof("cpu")]; 185 phandle_t child; 186 u_int cpuid; 187 uint32_t cpu_impl; 188 189 /* There's no need to traverse the whole OFW tree twice. */ 190 if (mp_maxid > 0 && mp_ncpus >= mp_maxid + 1) 191 return; 192 193 for (; node != 0; node = OF_peer(node)) { 194 child = OF_child(node); 195 if (child > 0) 196 foreach_ap(child, func); 197 else { 198 if (OF_getprop(node, "device_type", type, 199 sizeof(type)) <= 0) 200 continue; 201 if (strcmp(type, "cpu") != 0) 202 continue; 203 if (OF_getprop(node, "implementation#", &cpu_impl, 204 sizeof(cpu_impl)) <= 0) 205 panic("%s: couldn't determine CPU " 206 "implementation", __func__); 207 if (OF_getprop(node, cpu_cpuid_prop(cpu_impl), &cpuid, 208 sizeof(cpuid)) <= 0) 209 panic("%s: couldn't determine CPU module ID", 210 __func__); 211 if (cpuid == PCPU_GET(mid)) 212 continue; 213 (*func)(node, cpuid, cpu_impl); 214 } 215 } 216} 217 218/* 219 * Probe for other CPUs. 220 */ 221void 222cpu_mp_setmaxid() 223{ 224 225 CPU_SETOF(curcpu, &all_cpus); 226 mp_ncpus = 1; 227 mp_maxid = 0; 228 229 foreach_ap(OF_child(OF_peer(0)), ap_count); 230} 231 232static void 233ap_count(phandle_t node __unused, u_int mid __unused, u_int cpu_impl __unused) 234{ 235 236 mp_maxid++; 237} 238 239int 240cpu_mp_probe(void) 241{ 242 243 return (mp_maxid > 0); 244} 245 246struct cpu_group * 247cpu_topo(void) 248{ 249 250 return (smp_topo_none()); 251} 252 253static void 254sun4u_startcpu(phandle_t cpu, void *func, u_long arg) 255{ 256 static struct { 257 cell_t name; 258 cell_t nargs; 259 cell_t nreturns; 260 cell_t cpu; 261 cell_t func; 262 cell_t arg; 263 } args = { 264 (cell_t)SUNW_STARTCPU, 265 3, 266 }; 267 268 args.cpu = cpu; 269 args.func = (cell_t)func; 270 args.arg = (cell_t)arg; 271 ofw_entry(&args); 272} 273 274/* 275 * Fire up any non-boot processors. 276 */ 277void 278cpu_mp_start(void) 279{ 280 281 intr_setup(PIL_AST, cpu_ipi_ast, -1, NULL, NULL); 282 intr_setup(PIL_RENDEZVOUS, (ih_func_t *)smp_rendezvous_action, 283 -1, NULL, NULL); 284 intr_setup(PIL_STOP, cpu_ipi_stop, -1, NULL, NULL); 285 intr_setup(PIL_PREEMPT, cpu_ipi_preempt, -1, NULL, NULL); 286 intr_setup(PIL_HARDCLOCK, cpu_ipi_hardclock, -1, NULL, NULL); 287 288 cpuid_to_mid[curcpu] = PCPU_GET(mid); 289 290 foreach_ap(OF_child(OF_peer(0)), ap_start); 291 KASSERT(!isjbus || mp_ncpus <= IDR_JALAPENO_MAX_BN_PAIRS, 292 ("%s: can only IPI a maximum of %d JBus-CPUs", 293 __func__, IDR_JALAPENO_MAX_BN_PAIRS)); 294} 295 296static void 297ap_start(phandle_t node, u_int mid, u_int cpu_impl) 298{ 299 volatile struct cpu_start_args *csa; 300 struct pcpu *pc; 301 register_t s; 302 vm_offset_t va; 303 u_int cpuid; 304 uint32_t clock; 305 306 if (mp_ncpus > MAXCPU) 307 return; 308 309 if (OF_getprop(node, "clock-frequency", &clock, sizeof(clock)) <= 0) 310 panic("%s: couldn't determine CPU frequency", __func__); 311 if (clock != PCPU_GET(clock)) 312 tick_et_use_stick = 1; 313 314 csa = &cpu_start_args; 315 csa->csa_state = 0; 316 sun4u_startcpu(node, (void *)mp_tramp, 0); 317 s = intr_disable(); 318 while (csa->csa_state != CPU_TICKSYNC) 319 ; 320 membar(StoreLoad); 321 csa->csa_tick = rd(tick); 322 if (cpu_impl == CPU_IMPL_SPARC64V || 323 cpu_impl >= CPU_IMPL_ULTRASPARCIII) { 324 while (csa->csa_state != CPU_STICKSYNC) 325 ; 326 membar(StoreLoad); 327 csa->csa_stick = rdstick(); 328 } 329 while (csa->csa_state != CPU_INIT) 330 ; 331 csa->csa_tick = csa->csa_stick = 0; 332 intr_restore(s); 333 334 cpuid = mp_ncpus++; 335 cpuid_to_mid[cpuid] = mid; 336 cpu_identify(csa->csa_ver, clock, cpuid); 337 338 va = kmem_malloc(kernel_arena, PCPU_PAGES * PAGE_SIZE, 339 M_WAITOK | M_ZERO); 340 pc = (struct pcpu *)(va + (PCPU_PAGES * PAGE_SIZE)) - 1; 341 pcpu_init(pc, cpuid, sizeof(*pc)); 342 dpcpu_init((void *)kmem_malloc(kernel_arena, DPCPU_SIZE, 343 M_WAITOK | M_ZERO), cpuid); 344 pc->pc_addr = va; 345 pc->pc_clock = clock; 346 pc->pc_impl = cpu_impl; 347 pc->pc_mid = mid; 348 pc->pc_node = node; 349 350 cache_init(pc); 351 352 CPU_SET(cpuid, &all_cpus); 353 intr_add_cpu(cpuid); 354} 355 356void 357cpu_mp_announce(void) 358{ 359 360} 361 362static void 363cpu_mp_unleash(void *v) 364{ 365 volatile struct cpu_start_args *csa; 366 struct pcpu *pc; 367 register_t s; 368 vm_offset_t va; 369 vm_paddr_t pa; 370 u_int ctx_inc; 371 u_int ctx_min; 372 int i; 373 374 ctx_min = TLB_CTX_USER_MIN; 375 ctx_inc = (TLB_CTX_USER_MAX - 1) / mp_ncpus; 376 csa = &cpu_start_args; 377 csa->csa_count = mp_ncpus; 378 STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { 379 pc->pc_tlb_ctx = ctx_min; 380 pc->pc_tlb_ctx_min = ctx_min; 381 pc->pc_tlb_ctx_max = ctx_min + ctx_inc; 382 ctx_min += ctx_inc; 383 384 if (pc->pc_cpuid == curcpu) 385 continue; 386 KASSERT(pc->pc_idlethread != NULL, 387 ("%s: idlethread", __func__)); 388 pc->pc_curthread = pc->pc_idlethread; 389 pc->pc_curpcb = pc->pc_curthread->td_pcb; 390 for (i = 0; i < PCPU_PAGES; i++) { 391 va = pc->pc_addr + i * PAGE_SIZE; 392 pa = pmap_kextract(va); 393 if (pa == 0) 394 panic("%s: pmap_kextract", __func__); 395 csa->csa_ttes[i].tte_vpn = TV_VPN(va, TS_8K); 396 csa->csa_ttes[i].tte_data = TD_V | TD_8K | TD_PA(pa) | 397 TD_L | TD_CP | TD_CV | TD_P | TD_W; 398 } 399 csa->csa_state = 0; 400 csa->csa_pcpu = pc->pc_addr; 401 csa->csa_mid = pc->pc_mid; 402 s = intr_disable(); 403 while (csa->csa_state != CPU_BOOTSTRAP) 404 ; 405 intr_restore(s); 406 } 407 408 membar(StoreLoad); 409 csa->csa_count = 0; 410 smp_started = 1; 411} 412 413void 414cpu_mp_bootstrap(struct pcpu *pc) 415{ 416 volatile struct cpu_start_args *csa; 417 418 csa = &cpu_start_args; 419 420 /* Do CPU-specific initialization. */ 421 if (pc->pc_impl >= CPU_IMPL_ULTRASPARCIII) 422 cheetah_init(pc->pc_impl); 423 else if (pc->pc_impl == CPU_IMPL_SPARC64V) 424 zeus_init(pc->pc_impl); 425 426 /* 427 * Enable the caches. Note that his may include applying workarounds. 428 */ 429 cache_enable(pc->pc_impl); 430 431 /* 432 * Clear (S)TICK timer(s) (including NPT) and ensure they are stopped. 433 */ 434 tick_clear(pc->pc_impl); 435 tick_stop(pc->pc_impl); 436 437 /* Set the kernel context. */ 438 pmap_set_kctx(); 439 440 /* Lock the kernel TSB in the TLB if necessary. */ 441 if (tsb_kernel_ldd_phys == 0) 442 pmap_map_tsb(); 443 444 /* 445 * Flush all non-locked TLB entries possibly left over by the 446 * firmware. 447 */ 448 tlb_flush_nonlocked(); 449 450 /* 451 * Enable interrupts. 452 * Note that the PIL we be lowered indirectly via sched_throw(NULL) 453 * when fake spinlock held by the idle thread eventually is released. 454 */ 455 wrpr(pstate, 0, PSTATE_KERNEL); 456 457 smp_cpus++; 458 KASSERT(curthread != NULL, ("%s: curthread", __func__)); 459 printf("SMP: AP CPU #%d Launched!\n", curcpu); 460 461 csa->csa_count--; 462 membar(StoreLoad); 463 csa->csa_state = CPU_BOOTSTRAP; 464 while (csa->csa_count != 0) 465 ; 466 467 /* Start per-CPU event timers. */ 468 cpu_initclocks_ap(); 469 470 /* Ok, now enter the scheduler. */ 471 sched_throw(NULL); 472} 473 474void 475cpu_mp_shutdown(void) 476{ 477 cpuset_t cpus; 478 int i; 479 480 critical_enter(); 481 shutdown_cpus = all_cpus; 482 CPU_CLR(PCPU_GET(cpuid), &shutdown_cpus); 483 cpus = shutdown_cpus; 484 485 /* XXX: Stop all the CPUs which aren't already. */ 486 if (CPU_CMP(&stopped_cpus, &cpus)) { 487 488 /* cpus is just a flat "on" mask without curcpu. */ 489 CPU_NAND(&cpus, &stopped_cpus); 490 stop_cpus(cpus); 491 } 492 i = 0; 493 while (!CPU_EMPTY(&shutdown_cpus)) { 494 if (i++ > 100000) { 495 printf("timeout shutting down CPUs.\n"); 496 break; 497 } 498 } 499 critical_exit(); 500} 501 502static void 503cpu_ipi_ast(struct trapframe *tf __unused) 504{ 505 506} 507 508static void 509cpu_ipi_stop(struct trapframe *tf __unused) 510{ 511 u_int cpuid; 512 513 CTR2(KTR_SMP, "%s: stopped %d", __func__, curcpu); 514 sched_pin(); 515 savectx(&stoppcbs[curcpu]); 516 cpuid = PCPU_GET(cpuid); 517 CPU_SET_ATOMIC(cpuid, &stopped_cpus); 518 while (!CPU_ISSET(cpuid, &started_cpus)) { 519 if (CPU_ISSET(cpuid, &shutdown_cpus)) { 520 CPU_CLR_ATOMIC(cpuid, &shutdown_cpus); 521 (void)intr_disable(); 522 for (;;) 523 ; 524 } 525 } 526 CPU_CLR_ATOMIC(cpuid, &started_cpus); 527 CPU_CLR_ATOMIC(cpuid, &stopped_cpus); 528 sched_unpin(); 529 CTR2(KTR_SMP, "%s: restarted %d", __func__, curcpu); 530} 531 532static void 533cpu_ipi_preempt(struct trapframe *tf) 534{ 535 536 sched_preempt(curthread); 537} 538 539static void 540cpu_ipi_hardclock(struct trapframe *tf) 541{ 542 struct trapframe *oldframe; 543 struct thread *td; 544 545 critical_enter(); 546 td = curthread; 547 td->td_intr_nesting_level++; 548 oldframe = td->td_intr_frame; 549 td->td_intr_frame = tf; 550 hardclockintr(); 551 td->td_intr_frame = oldframe; 552 td->td_intr_nesting_level--; 553 critical_exit(); 554} 555 556static void 557spitfire_ipi_selected(cpuset_t cpus, u_long d0, u_long d1, u_long d2) 558{ 559 u_int cpu; 560 561 while ((cpu = CPU_FFS(&cpus)) != 0) { 562 cpu--; 563 CPU_CLR(cpu, &cpus); 564 spitfire_ipi_single(cpu, d0, d1, d2); 565 } 566} 567 568static void 569spitfire_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2) 570{ 571 register_t s; 572 u_long ids; 573 u_int mid; 574 int i; 575 576 KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__)); 577 KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & IDR_BUSY) == 0, 578 ("%s: outstanding dispatch", __func__)); 579 mid = cpuid_to_mid[cpu]; 580 for (i = 0; i < IPI_RETRIES; i++) { 581 s = intr_disable(); 582 stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 583 stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 584 stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 585 membar(Sync); 586 stxa(AA_INTR_SEND | (mid << IDC_ITID_SHIFT), 587 ASI_SDB_INTR_W, 0); 588 /* 589 * Workaround for SpitFire erratum #54; do a dummy read 590 * from a SDB internal register before the MEMBAR #Sync 591 * for the write to ASI_SDB_INTR_W (requiring another 592 * MEMBAR #Sync in order to make sure the write has 593 * occurred before the load). 594 */ 595 membar(Sync); 596 (void)ldxa(AA_SDB_CNTL_HIGH, ASI_SDB_CONTROL_R); 597 membar(Sync); 598 while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 599 IDR_BUSY) != 0) 600 ; 601 intr_restore(s); 602 if ((ids & (IDR_BUSY | IDR_NACK)) == 0) 603 return; 604 /* 605 * Leave interrupts enabled for a bit before retrying 606 * in order to avoid deadlocks if the other CPU is also 607 * trying to send an IPI. 608 */ 609 DELAY(2); 610 } 611 if (kdb_active != 0 || panicstr != NULL) 612 printf("%s: couldn't send IPI to module 0x%u\n", 613 __func__, mid); 614 else 615 panic("%s: couldn't send IPI to module 0x%u", 616 __func__, mid); 617} 618 619static void 620cheetah_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2) 621{ 622 register_t s; 623 u_long ids; 624 u_int mid; 625 int i; 626 627 KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__)); 628 KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & 629 IDR_CHEETAH_ALL_BUSY) == 0, 630 ("%s: outstanding dispatch", __func__)); 631 mid = cpuid_to_mid[cpu]; 632 for (i = 0; i < IPI_RETRIES; i++) { 633 s = intr_disable(); 634 stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 635 stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 636 stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 637 membar(Sync); 638 stxa(AA_INTR_SEND | (mid << IDC_ITID_SHIFT), 639 ASI_SDB_INTR_W, 0); 640 membar(Sync); 641 while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 642 IDR_BUSY) != 0) 643 ; 644 intr_restore(s); 645 if ((ids & (IDR_BUSY | IDR_NACK)) == 0) 646 return; 647 /* 648 * Leave interrupts enabled for a bit before retrying 649 * in order to avoid deadlocks if the other CPU is also 650 * trying to send an IPI. 651 */ 652 DELAY(2); 653 } 654 if (kdb_active != 0 || panicstr != NULL) 655 printf("%s: couldn't send IPI to module 0x%u\n", 656 __func__, mid); 657 else 658 panic("%s: couldn't send IPI to module 0x%u", 659 __func__, mid); 660} 661 662static void 663cheetah_ipi_selected(cpuset_t cpus, u_long d0, u_long d1, u_long d2) 664{ 665 char pbuf[CPUSETBUFSIZ]; 666 register_t s; 667 u_long ids; 668 u_int bnp; 669 u_int cpu; 670 int i; 671 672 KASSERT(!CPU_ISSET(curcpu, &cpus), ("%s: CPU can't IPI itself", 673 __func__)); 674 KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & 675 IDR_CHEETAH_ALL_BUSY) == 0, 676 ("%s: outstanding dispatch", __func__)); 677 if (CPU_EMPTY(&cpus)) 678 return; 679 ids = 0; 680 for (i = 0; i < IPI_RETRIES * mp_ncpus; i++) { 681 s = intr_disable(); 682 stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 683 stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 684 stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 685 membar(Sync); 686 bnp = 0; 687 for (cpu = 0; cpu < mp_ncpus; cpu++) { 688 if (CPU_ISSET(cpu, &cpus)) { 689 stxa(AA_INTR_SEND | (cpuid_to_mid[cpu] << 690 IDC_ITID_SHIFT) | bnp << IDC_BN_SHIFT, 691 ASI_SDB_INTR_W, 0); 692 membar(Sync); 693 bnp++; 694 if (bnp == IDR_CHEETAH_MAX_BN_PAIRS) 695 break; 696 } 697 } 698 while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 699 IDR_CHEETAH_ALL_BUSY) != 0) 700 ; 701 intr_restore(s); 702 bnp = 0; 703 for (cpu = 0; cpu < mp_ncpus; cpu++) { 704 if (CPU_ISSET(cpu, &cpus)) { 705 if ((ids & (IDR_NACK << (2 * bnp))) == 0) 706 CPU_CLR(cpu, &cpus); 707 bnp++; 708 } 709 } 710 if (CPU_EMPTY(&cpus)) 711 return; 712 /* 713 * Leave interrupts enabled for a bit before retrying 714 * in order to avoid deadlocks if the other CPUs are 715 * also trying to send IPIs. 716 */ 717 DELAY(2 * mp_ncpus); 718 } 719 if (kdb_active != 0 || panicstr != NULL) 720 printf("%s: couldn't send IPI (cpus=%s ids=0x%lu)\n", 721 __func__, cpusetobj_strprint(pbuf, &cpus), ids); 722 else 723 panic("%s: couldn't send IPI (cpus=%s ids=0x%lu)", 724 __func__, cpusetobj_strprint(pbuf, &cpus), ids); 725} 726 727static void 728jalapeno_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2) 729{ 730 register_t s; 731 u_long ids; 732 u_int busy, busynack, mid; 733 int i; 734 735 KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__)); 736 KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & 737 IDR_CHEETAH_ALL_BUSY) == 0, 738 ("%s: outstanding dispatch", __func__)); 739 mid = cpuid_to_mid[cpu]; 740 busy = IDR_BUSY << (2 * mid); 741 busynack = (IDR_BUSY | IDR_NACK) << (2 * mid); 742 for (i = 0; i < IPI_RETRIES; i++) { 743 s = intr_disable(); 744 stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 745 stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 746 stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 747 membar(Sync); 748 stxa(AA_INTR_SEND | (mid << IDC_ITID_SHIFT), 749 ASI_SDB_INTR_W, 0); 750 membar(Sync); 751 while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 752 busy) != 0) 753 ; 754 intr_restore(s); 755 if ((ids & busynack) == 0) 756 return; 757 /* 758 * Leave interrupts enabled for a bit before retrying 759 * in order to avoid deadlocks if the other CPU is also 760 * trying to send an IPI. 761 */ 762 DELAY(2); 763 } 764 if (kdb_active != 0 || panicstr != NULL) 765 printf("%s: couldn't send IPI to module 0x%u\n", 766 __func__, mid); 767 else 768 panic("%s: couldn't send IPI to module 0x%u", 769 __func__, mid); 770} 771 772static void 773jalapeno_ipi_selected(cpuset_t cpus, u_long d0, u_long d1, u_long d2) 774{ 775 char pbuf[CPUSETBUFSIZ]; 776 register_t s; 777 u_long ids; 778 u_int cpu; 779 int i; 780 781 KASSERT(!CPU_ISSET(curcpu, &cpus), ("%s: CPU can't IPI itself", 782 __func__)); 783 KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & 784 IDR_CHEETAH_ALL_BUSY) == 0, 785 ("%s: outstanding dispatch", __func__)); 786 if (CPU_EMPTY(&cpus)) 787 return; 788 ids = 0; 789 for (i = 0; i < IPI_RETRIES * mp_ncpus; i++) { 790 s = intr_disable(); 791 stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 792 stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 793 stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 794 membar(Sync); 795 for (cpu = 0; cpu < mp_ncpus; cpu++) { 796 if (CPU_ISSET(cpu, &cpus)) { 797 stxa(AA_INTR_SEND | (cpuid_to_mid[cpu] << 798 IDC_ITID_SHIFT), ASI_SDB_INTR_W, 0); 799 membar(Sync); 800 } 801 } 802 while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 803 IDR_CHEETAH_ALL_BUSY) != 0) 804 ; 805 intr_restore(s); 806 if ((ids & 807 (IDR_CHEETAH_ALL_BUSY | IDR_CHEETAH_ALL_NACK)) == 0) 808 return; 809 for (cpu = 0; cpu < mp_ncpus; cpu++) 810 if (CPU_ISSET(cpu, &cpus)) 811 if ((ids & (IDR_NACK << 812 (2 * cpuid_to_mid[cpu]))) == 0) 813 CPU_CLR(cpu, &cpus); 814 /* 815 * Leave interrupts enabled for a bit before retrying 816 * in order to avoid deadlocks if the other CPUs are 817 * also trying to send IPIs. 818 */ 819 DELAY(2 * mp_ncpus); 820 } 821 if (kdb_active != 0 || panicstr != NULL) 822 printf("%s: couldn't send IPI (cpus=%s ids=0x%lu)\n", 823 __func__, cpusetobj_strprint(pbuf, &cpus), ids); 824 else 825 panic("%s: couldn't send IPI (cpus=%s ids=0x%lu)", 826 __func__, cpusetobj_strprint(pbuf, &cpus), ids); 827} 828