1/*- 2 * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 3. Berkeley Software Design Inc's name may not be used to endorse or 13 * promote products derived from this software without specific prior 14 * written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * from BSDI: locore.s,v 1.36.2.15 1999/08/23 22:34:41 cp Exp 29 */ 30/*- 31 * Copyright (c) 2002 Jake Burkholder. 32 * Copyright (c) 2007 - 2010 Marius Strobl <marius@FreeBSD.org> 33 * All rights reserved. 34 * 35 * Redistribution and use in source and binary forms, with or without 36 * modification, are permitted provided that the following conditions 37 * are met: 38 * 1. Redistributions of source code must retain the above copyright 39 * notice, this list of conditions and the following disclaimer. 40 * 2. Redistributions in binary form must reproduce the above copyright 41 * notice, this list of conditions and the following disclaimer in the 42 * documentation and/or other materials provided with the distribution. 43 * 44 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 45 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 46 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 47 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 48 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 49 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 50 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 51 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 52 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 53 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 54 * SUCH DAMAGE. 55 */ 56 57#include <sys/cdefs.h> 58__FBSDID("$FreeBSD$"); 59 60#include <sys/param.h> 61#include <sys/systm.h> 62#include <sys/lock.h> 63#include <sys/kdb.h> 64#include <sys/kernel.h> 65#include <sys/ktr.h> 66#include <sys/mutex.h> 67#include <sys/pcpu.h> 68#include <sys/proc.h> 69#include <sys/sched.h> 70#include <sys/smp.h> 71 72#include <vm/vm.h> 73#include <vm/vm_param.h> 74#include <vm/pmap.h> 75#include <vm/vm_kern.h> 76#include <vm/vm_extern.h> 77#include <vm/vm_map.h> 78 79#include <dev/ofw/openfirm.h> 80 81#include <machine/asi.h> 82#include <machine/atomic.h> 83#include <machine/bus.h> 84#include <machine/cpu.h> 85#include <machine/md_var.h> 86#include <machine/metadata.h> 87#include <machine/ofw_machdep.h> 88#include <machine/pcb.h> 89#include <machine/smp.h> 90#include <machine/tick.h> 91#include <machine/tlb.h> 92#include <machine/tsb.h> 93#include <machine/tte.h> 94#include <machine/ver.h> 95 96#define SUNW_STARTCPU "SUNW,start-cpu" 97#define SUNW_STOPSELF "SUNW,stop-self" 98 99static ih_func_t cpu_ipi_ast; 100static ih_func_t cpu_ipi_hardclock; 101static ih_func_t cpu_ipi_preempt; 102static ih_func_t cpu_ipi_stop; 103 104/* 105 * Argument area used to pass data to non-boot processors as they start up. 106 * This must be statically initialized with a known invalid CPU module ID, 107 * since the other processors will use it before the boot CPU enters the 108 * kernel. 109 */ 110struct cpu_start_args cpu_start_args = { 0, -1, -1, 0, 0, 0 }; 111struct ipi_cache_args ipi_cache_args; 112struct ipi_rd_args ipi_rd_args; 113struct ipi_tlb_args ipi_tlb_args; 114struct pcb stoppcbs[MAXCPU]; 115 116cpu_ipi_selected_t *cpu_ipi_selected; 117cpu_ipi_single_t *cpu_ipi_single; 118 119static vm_offset_t mp_tramp; 120static u_int cpuid_to_mid[MAXCPU]; 121static int isjbus; 122static volatile cpuset_t shutdown_cpus; 123 124static void ap_count(phandle_t node, u_int mid, u_int cpu_impl); 125static void ap_start(phandle_t node, u_int mid, u_int cpu_impl); 126static void cpu_mp_unleash(void *v); 127static void foreach_ap(phandle_t node, void (*func)(phandle_t node, 128 u_int mid, u_int cpu_impl)); 129static void sun4u_startcpu(phandle_t cpu, void *func, u_long arg); 130 131static cpu_ipi_selected_t cheetah_ipi_selected; 132static cpu_ipi_single_t cheetah_ipi_single; 133static cpu_ipi_selected_t jalapeno_ipi_selected; 134static cpu_ipi_single_t jalapeno_ipi_single; 135static cpu_ipi_selected_t spitfire_ipi_selected; 136static cpu_ipi_single_t spitfire_ipi_single; 137 138SYSINIT(cpu_mp_unleash, SI_SUB_SMP, SI_ORDER_FIRST, cpu_mp_unleash, NULL); 139 140void 141mp_init(u_int cpu_impl) 142{ 143 struct tte *tp; 144 int i; 145 146 mp_tramp = (vm_offset_t)OF_claim(NULL, PAGE_SIZE, PAGE_SIZE); 147 if (mp_tramp == (vm_offset_t)-1) 148 panic("%s", __func__); 149 bcopy(mp_tramp_code, (void *)mp_tramp, mp_tramp_code_len); 150 *(vm_offset_t *)(mp_tramp + mp_tramp_tlb_slots) = kernel_tlb_slots; 151 *(vm_offset_t *)(mp_tramp + mp_tramp_func) = (vm_offset_t)mp_startup; 152 tp = (struct tte *)(mp_tramp + mp_tramp_code_len); 153 for (i = 0; i < kernel_tlb_slots; i++) { 154 tp[i].tte_vpn = TV_VPN(kernel_tlbs[i].te_va, TS_4M); 155 tp[i].tte_data = TD_V | TD_4M | TD_PA(kernel_tlbs[i].te_pa) | 156 TD_L | TD_CP | TD_CV | TD_P | TD_W; 157 } 158 for (i = 0; i < PAGE_SIZE; i += sizeof(vm_offset_t)) 159 flush(mp_tramp + i); 160 161 /* 162 * On UP systems cpu_ipi_selected() can be called while 163 * cpu_mp_start() wasn't so initialize these here. 164 */ 165 if (cpu_impl == CPU_IMPL_ULTRASPARCIIIi || 166 cpu_impl == CPU_IMPL_ULTRASPARCIIIip) { 167 isjbus = 1; 168 cpu_ipi_selected = jalapeno_ipi_selected; 169 cpu_ipi_single = jalapeno_ipi_single; 170 } else if (cpu_impl == CPU_IMPL_SPARC64V || 171 cpu_impl >= CPU_IMPL_ULTRASPARCIII) { 172 cpu_ipi_selected = cheetah_ipi_selected; 173 cpu_ipi_single = cheetah_ipi_single; 174 } else { 175 cpu_ipi_selected = spitfire_ipi_selected; 176 cpu_ipi_single = spitfire_ipi_single; 177 } 178} 179 180static void 181foreach_ap(phandle_t node, void (*func)(phandle_t node, u_int mid, 182 u_int cpu_impl)) 183{ 184 char type[sizeof("cpu")]; 185 phandle_t child; 186 u_int cpuid; 187 uint32_t cpu_impl; 188 189 /* There's no need to traverse the whole OFW tree twice. */ 190 if (mp_maxid > 0 && mp_ncpus >= mp_maxid + 1) 191 return; 192 193 for (; node != 0; node = OF_peer(node)) { 194 child = OF_child(node); 195 if (child > 0) 196 foreach_ap(child, func); 197 else { 198 if (OF_getprop(node, "device_type", type, 199 sizeof(type)) <= 0) 200 continue; 201 if (strcmp(type, "cpu") != 0) 202 continue; 203 if (OF_getprop(node, "implementation#", &cpu_impl, 204 sizeof(cpu_impl)) <= 0) 205 panic("%s: couldn't determine CPU " 206 "implementation", __func__); 207 if (OF_getprop(node, cpu_cpuid_prop(cpu_impl), &cpuid, 208 sizeof(cpuid)) <= 0) 209 panic("%s: couldn't determine CPU module ID", 210 __func__); 211 if (cpuid == PCPU_GET(mid)) 212 continue; 213 (*func)(node, cpuid, cpu_impl); 214 } 215 } 216} 217 218/* 219 * Probe for other CPUs. 220 */ 221void 222cpu_mp_setmaxid() 223{ 224 225 CPU_SETOF(curcpu, &all_cpus); 226 mp_ncpus = 1; 227 mp_maxid = 0; 228 229 foreach_ap(OF_child(OF_peer(0)), ap_count); 230} 231 232static void 233ap_count(phandle_t node __unused, u_int mid __unused, u_int cpu_impl __unused) 234{ 235 236 mp_maxid++; 237} 238 239int 240cpu_mp_probe(void) 241{ 242 243 return (mp_maxid > 0); 244} 245 246struct cpu_group * 247cpu_topo(void) 248{ 249 250 return (smp_topo_none()); 251} 252 253static void 254sun4u_startcpu(phandle_t cpu, void *func, u_long arg) 255{ 256 static struct { 257 cell_t name; 258 cell_t nargs; 259 cell_t nreturns; 260 cell_t cpu; 261 cell_t func; 262 cell_t arg; 263 } args = { 264 (cell_t)SUNW_STARTCPU, 265 3, 266 }; 267 268 args.cpu = cpu; 269 args.func = (cell_t)func; 270 args.arg = (cell_t)arg; 271 ofw_entry(&args); 272} 273 274/* 275 * Fire up any non-boot processors. 276 */ 277void 278cpu_mp_start(void) 279{ 280 281 intr_setup(PIL_AST, cpu_ipi_ast, -1, NULL, NULL); 282 intr_setup(PIL_RENDEZVOUS, (ih_func_t *)smp_rendezvous_action, 283 -1, NULL, NULL); 284 intr_setup(PIL_STOP, cpu_ipi_stop, -1, NULL, NULL); 285 intr_setup(PIL_PREEMPT, cpu_ipi_preempt, -1, NULL, NULL); 286 intr_setup(PIL_HARDCLOCK, cpu_ipi_hardclock, -1, NULL, NULL); 287 288 cpuid_to_mid[curcpu] = PCPU_GET(mid); 289 290 foreach_ap(OF_child(OF_peer(0)), ap_start); 291 KASSERT(!isjbus || mp_ncpus <= IDR_JALAPENO_MAX_BN_PAIRS, 292 ("%s: can only IPI a maximum of %d JBus-CPUs", 293 __func__, IDR_JALAPENO_MAX_BN_PAIRS)); 294 smp_active = 1; 295} 296 297static void 298ap_start(phandle_t node, u_int mid, u_int cpu_impl) 299{ 300 volatile struct cpu_start_args *csa; 301 struct pcpu *pc; 302 register_t s; 303 vm_offset_t va; 304 u_int cpuid; 305 uint32_t clock; 306 307 if (mp_ncpus > MAXCPU) 308 return; 309 310 if (OF_getprop(node, "clock-frequency", &clock, sizeof(clock)) <= 0) 311 panic("%s: couldn't determine CPU frequency", __func__); 312 if (clock != PCPU_GET(clock)) 313 tick_et_use_stick = 1; 314 315 csa = &cpu_start_args; 316 csa->csa_state = 0; 317 sun4u_startcpu(node, (void *)mp_tramp, 0); 318 s = intr_disable(); 319 while (csa->csa_state != CPU_TICKSYNC) 320 ; 321 membar(StoreLoad); 322 csa->csa_tick = rd(tick); 323 if (cpu_impl == CPU_IMPL_SPARC64V || 324 cpu_impl >= CPU_IMPL_ULTRASPARCIII) { 325 while (csa->csa_state != CPU_STICKSYNC) 326 ; 327 membar(StoreLoad); 328 csa->csa_stick = rdstick(); 329 } 330 while (csa->csa_state != CPU_INIT) 331 ; 332 csa->csa_tick = csa->csa_stick = 0; 333 intr_restore(s); 334 335 cpuid = mp_ncpus++; 336 cpuid_to_mid[cpuid] = mid; 337 cpu_identify(csa->csa_ver, clock, cpuid); 338 339 va = kmem_malloc(kernel_arena, PCPU_PAGES * PAGE_SIZE, 340 M_WAITOK | M_ZERO); 341 pc = (struct pcpu *)(va + (PCPU_PAGES * PAGE_SIZE)) - 1; 342 pcpu_init(pc, cpuid, sizeof(*pc)); 343 dpcpu_init((void *)kmem_malloc(kernel_arena, DPCPU_SIZE, 344 M_WAITOK | M_ZERO), cpuid); 345 pc->pc_addr = va; 346 pc->pc_clock = clock; 347 pc->pc_impl = cpu_impl; 348 pc->pc_mid = mid; 349 pc->pc_node = node; 350 351 cache_init(pc); 352 353 CPU_SET(cpuid, &all_cpus); 354 intr_add_cpu(cpuid); 355} 356 357void 358cpu_mp_announce(void) 359{ 360 361} 362 363static void 364cpu_mp_unleash(void *v) 365{ 366 volatile struct cpu_start_args *csa; 367 struct pcpu *pc; 368 register_t s; 369 vm_offset_t va; 370 vm_paddr_t pa; 371 u_int ctx_inc; 372 u_int ctx_min; 373 int i; 374 375 ctx_min = TLB_CTX_USER_MIN; 376 ctx_inc = (TLB_CTX_USER_MAX - 1) / mp_ncpus; 377 csa = &cpu_start_args; 378 csa->csa_count = mp_ncpus; 379 STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { 380 pc->pc_tlb_ctx = ctx_min; 381 pc->pc_tlb_ctx_min = ctx_min; 382 pc->pc_tlb_ctx_max = ctx_min + ctx_inc; 383 ctx_min += ctx_inc; 384 385 if (pc->pc_cpuid == curcpu) 386 continue; 387 KASSERT(pc->pc_idlethread != NULL, 388 ("%s: idlethread", __func__)); 389 pc->pc_curthread = pc->pc_idlethread; 390 pc->pc_curpcb = pc->pc_curthread->td_pcb; 391 for (i = 0; i < PCPU_PAGES; i++) { 392 va = pc->pc_addr + i * PAGE_SIZE; 393 pa = pmap_kextract(va); 394 if (pa == 0) 395 panic("%s: pmap_kextract", __func__); 396 csa->csa_ttes[i].tte_vpn = TV_VPN(va, TS_8K); 397 csa->csa_ttes[i].tte_data = TD_V | TD_8K | TD_PA(pa) | 398 TD_L | TD_CP | TD_CV | TD_P | TD_W; 399 } 400 csa->csa_state = 0; 401 csa->csa_pcpu = pc->pc_addr; 402 csa->csa_mid = pc->pc_mid; 403 s = intr_disable(); 404 while (csa->csa_state != CPU_BOOTSTRAP) 405 ; 406 intr_restore(s); 407 } 408 409 membar(StoreLoad); 410 csa->csa_count = 0; 411 smp_started = 1; 412} 413 414void 415cpu_mp_bootstrap(struct pcpu *pc) 416{ 417 volatile struct cpu_start_args *csa; 418 419 csa = &cpu_start_args; 420 421 /* Do CPU-specific initialization. */ 422 if (pc->pc_impl >= CPU_IMPL_ULTRASPARCIII) 423 cheetah_init(pc->pc_impl); 424 else if (pc->pc_impl == CPU_IMPL_SPARC64V) 425 zeus_init(pc->pc_impl); 426 427 /* 428 * Enable the caches. Note that his may include applying workarounds. 429 */ 430 cache_enable(pc->pc_impl); 431 432 /* 433 * Clear (S)TICK timer(s) (including NPT) and ensure they are stopped. 434 */ 435 tick_clear(pc->pc_impl); 436 tick_stop(pc->pc_impl); 437 438 /* Set the kernel context. */ 439 pmap_set_kctx(); 440 441 /* Lock the kernel TSB in the TLB if necessary. */ 442 if (tsb_kernel_ldd_phys == 0) 443 pmap_map_tsb(); 444 445 /* 446 * Flush all non-locked TLB entries possibly left over by the 447 * firmware. 448 */ 449 tlb_flush_nonlocked(); 450 451 /* 452 * Enable interrupts. 453 * Note that the PIL we be lowered indirectly via sched_throw(NULL) 454 * when fake spinlock held by the idle thread eventually is released. 455 */ 456 wrpr(pstate, 0, PSTATE_KERNEL); 457 458 smp_cpus++; 459 KASSERT(curthread != NULL, ("%s: curthread", __func__)); 460 printf("SMP: AP CPU #%d Launched!\n", curcpu); 461 462 csa->csa_count--; 463 membar(StoreLoad); 464 csa->csa_state = CPU_BOOTSTRAP; 465 while (csa->csa_count != 0) 466 ; 467 468 /* Start per-CPU event timers. */ 469 cpu_initclocks_ap(); 470 471 /* Ok, now enter the scheduler. */ 472 sched_throw(NULL); 473} 474 475void 476cpu_mp_shutdown(void) 477{ 478 cpuset_t cpus; 479 int i; 480 481 critical_enter(); 482 shutdown_cpus = all_cpus; 483 CPU_CLR(PCPU_GET(cpuid), &shutdown_cpus); 484 cpus = shutdown_cpus; 485 486 /* XXX: Stop all the CPUs which aren't already. */ 487 if (CPU_CMP(&stopped_cpus, &cpus)) { 488 489 /* cpus is just a flat "on" mask without curcpu. */ 490 CPU_NAND(&cpus, &stopped_cpus); 491 stop_cpus(cpus); 492 } 493 i = 0; 494 while (!CPU_EMPTY(&shutdown_cpus)) { 495 if (i++ > 100000) { 496 printf("timeout shutting down CPUs.\n"); 497 break; 498 } 499 } 500 critical_exit(); 501} 502 503static void 504cpu_ipi_ast(struct trapframe *tf __unused) 505{ 506 507} 508 509static void 510cpu_ipi_stop(struct trapframe *tf __unused) 511{ 512 u_int cpuid; 513 514 CTR2(KTR_SMP, "%s: stopped %d", __func__, curcpu); 515 sched_pin(); 516 savectx(&stoppcbs[curcpu]); 517 cpuid = PCPU_GET(cpuid); 518 CPU_SET_ATOMIC(cpuid, &stopped_cpus); 519 while (!CPU_ISSET(cpuid, &started_cpus)) { 520 if (CPU_ISSET(cpuid, &shutdown_cpus)) { 521 CPU_CLR_ATOMIC(cpuid, &shutdown_cpus); 522 (void)intr_disable(); 523 for (;;) 524 ; 525 } 526 } 527 CPU_CLR_ATOMIC(cpuid, &started_cpus); 528 CPU_CLR_ATOMIC(cpuid, &stopped_cpus); 529 sched_unpin(); 530 CTR2(KTR_SMP, "%s: restarted %d", __func__, curcpu); 531} 532 533static void 534cpu_ipi_preempt(struct trapframe *tf) 535{ 536 537 sched_preempt(curthread); 538} 539 540static void 541cpu_ipi_hardclock(struct trapframe *tf) 542{ 543 struct trapframe *oldframe; 544 struct thread *td; 545 546 critical_enter(); 547 td = curthread; 548 td->td_intr_nesting_level++; 549 oldframe = td->td_intr_frame; 550 td->td_intr_frame = tf; 551 hardclockintr(); 552 td->td_intr_frame = oldframe; 553 td->td_intr_nesting_level--; 554 critical_exit(); 555} 556 557static void 558spitfire_ipi_selected(cpuset_t cpus, u_long d0, u_long d1, u_long d2) 559{ 560 u_int cpu; 561 562 while ((cpu = CPU_FFS(&cpus)) != 0) { 563 cpu--; 564 CPU_CLR(cpu, &cpus); 565 spitfire_ipi_single(cpu, d0, d1, d2); 566 } 567} 568 569static void 570spitfire_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2) 571{ 572 register_t s; 573 u_long ids; 574 u_int mid; 575 int i; 576 577 KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__)); 578 KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & IDR_BUSY) == 0, 579 ("%s: outstanding dispatch", __func__)); 580 mid = cpuid_to_mid[cpu]; 581 for (i = 0; i < IPI_RETRIES; i++) { 582 s = intr_disable(); 583 stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 584 stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 585 stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 586 membar(Sync); 587 stxa(AA_INTR_SEND | (mid << IDC_ITID_SHIFT), 588 ASI_SDB_INTR_W, 0); 589 /* 590 * Workaround for SpitFire erratum #54; do a dummy read 591 * from a SDB internal register before the MEMBAR #Sync 592 * for the write to ASI_SDB_INTR_W (requiring another 593 * MEMBAR #Sync in order to make sure the write has 594 * occurred before the load). 595 */ 596 membar(Sync); 597 (void)ldxa(AA_SDB_CNTL_HIGH, ASI_SDB_CONTROL_R); 598 membar(Sync); 599 while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 600 IDR_BUSY) != 0) 601 ; 602 intr_restore(s); 603 if ((ids & (IDR_BUSY | IDR_NACK)) == 0) 604 return; 605 /* 606 * Leave interrupts enabled for a bit before retrying 607 * in order to avoid deadlocks if the other CPU is also 608 * trying to send an IPI. 609 */ 610 DELAY(2); 611 } 612 if (kdb_active != 0 || panicstr != NULL) 613 printf("%s: couldn't send IPI to module 0x%u\n", 614 __func__, mid); 615 else 616 panic("%s: couldn't send IPI to module 0x%u", 617 __func__, mid); 618} 619 620static void 621cheetah_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2) 622{ 623 register_t s; 624 u_long ids; 625 u_int mid; 626 int i; 627 628 KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__)); 629 KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & 630 IDR_CHEETAH_ALL_BUSY) == 0, 631 ("%s: outstanding dispatch", __func__)); 632 mid = cpuid_to_mid[cpu]; 633 for (i = 0; i < IPI_RETRIES; i++) { 634 s = intr_disable(); 635 stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 636 stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 637 stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 638 membar(Sync); 639 stxa(AA_INTR_SEND | (mid << IDC_ITID_SHIFT), 640 ASI_SDB_INTR_W, 0); 641 membar(Sync); 642 while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 643 IDR_BUSY) != 0) 644 ; 645 intr_restore(s); 646 if ((ids & (IDR_BUSY | IDR_NACK)) == 0) 647 return; 648 /* 649 * Leave interrupts enabled for a bit before retrying 650 * in order to avoid deadlocks if the other CPU is also 651 * trying to send an IPI. 652 */ 653 DELAY(2); 654 } 655 if (kdb_active != 0 || panicstr != NULL) 656 printf("%s: couldn't send IPI to module 0x%u\n", 657 __func__, mid); 658 else 659 panic("%s: couldn't send IPI to module 0x%u", 660 __func__, mid); 661} 662 663static void 664cheetah_ipi_selected(cpuset_t cpus, u_long d0, u_long d1, u_long d2) 665{ 666 char pbuf[CPUSETBUFSIZ]; 667 register_t s; 668 u_long ids; 669 u_int bnp; 670 u_int cpu; 671 int i; 672 673 KASSERT(!CPU_ISSET(curcpu, &cpus), ("%s: CPU can't IPI itself", 674 __func__)); 675 KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & 676 IDR_CHEETAH_ALL_BUSY) == 0, 677 ("%s: outstanding dispatch", __func__)); 678 if (CPU_EMPTY(&cpus)) 679 return; 680 ids = 0; 681 for (i = 0; i < IPI_RETRIES * mp_ncpus; i++) { 682 s = intr_disable(); 683 stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 684 stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 685 stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 686 membar(Sync); 687 bnp = 0; 688 for (cpu = 0; cpu < mp_ncpus; cpu++) { 689 if (CPU_ISSET(cpu, &cpus)) { 690 stxa(AA_INTR_SEND | (cpuid_to_mid[cpu] << 691 IDC_ITID_SHIFT) | bnp << IDC_BN_SHIFT, 692 ASI_SDB_INTR_W, 0); 693 membar(Sync); 694 bnp++; 695 if (bnp == IDR_CHEETAH_MAX_BN_PAIRS) 696 break; 697 } 698 } 699 while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 700 IDR_CHEETAH_ALL_BUSY) != 0) 701 ; 702 intr_restore(s); 703 bnp = 0; 704 for (cpu = 0; cpu < mp_ncpus; cpu++) { 705 if (CPU_ISSET(cpu, &cpus)) { 706 if ((ids & (IDR_NACK << (2 * bnp))) == 0) 707 CPU_CLR(cpu, &cpus); 708 bnp++; 709 } 710 } 711 if (CPU_EMPTY(&cpus)) 712 return; 713 /* 714 * Leave interrupts enabled for a bit before retrying 715 * in order to avoid deadlocks if the other CPUs are 716 * also trying to send IPIs. 717 */ 718 DELAY(2 * mp_ncpus); 719 } 720 if (kdb_active != 0 || panicstr != NULL) 721 printf("%s: couldn't send IPI (cpus=%s ids=0x%lu)\n", 722 __func__, cpusetobj_strprint(pbuf, &cpus), ids); 723 else 724 panic("%s: couldn't send IPI (cpus=%s ids=0x%lu)", 725 __func__, cpusetobj_strprint(pbuf, &cpus), ids); 726} 727 728static void 729jalapeno_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2) 730{ 731 register_t s; 732 u_long ids; 733 u_int busy, busynack, mid; 734 int i; 735 736 KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__)); 737 KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & 738 IDR_CHEETAH_ALL_BUSY) == 0, 739 ("%s: outstanding dispatch", __func__)); 740 mid = cpuid_to_mid[cpu]; 741 busy = IDR_BUSY << (2 * mid); 742 busynack = (IDR_BUSY | IDR_NACK) << (2 * mid); 743 for (i = 0; i < IPI_RETRIES; i++) { 744 s = intr_disable(); 745 stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 746 stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 747 stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 748 membar(Sync); 749 stxa(AA_INTR_SEND | (mid << IDC_ITID_SHIFT), 750 ASI_SDB_INTR_W, 0); 751 membar(Sync); 752 while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 753 busy) != 0) 754 ; 755 intr_restore(s); 756 if ((ids & busynack) == 0) 757 return; 758 /* 759 * Leave interrupts enabled for a bit before retrying 760 * in order to avoid deadlocks if the other CPU is also 761 * trying to send an IPI. 762 */ 763 DELAY(2); 764 } 765 if (kdb_active != 0 || panicstr != NULL) 766 printf("%s: couldn't send IPI to module 0x%u\n", 767 __func__, mid); 768 else 769 panic("%s: couldn't send IPI to module 0x%u", 770 __func__, mid); 771} 772 773static void 774jalapeno_ipi_selected(cpuset_t cpus, u_long d0, u_long d1, u_long d2) 775{ 776 char pbuf[CPUSETBUFSIZ]; 777 register_t s; 778 u_long ids; 779 u_int cpu; 780 int i; 781 782 KASSERT(!CPU_ISSET(curcpu, &cpus), ("%s: CPU can't IPI itself", 783 __func__)); 784 KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & 785 IDR_CHEETAH_ALL_BUSY) == 0, 786 ("%s: outstanding dispatch", __func__)); 787 if (CPU_EMPTY(&cpus)) 788 return; 789 ids = 0; 790 for (i = 0; i < IPI_RETRIES * mp_ncpus; i++) { 791 s = intr_disable(); 792 stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 793 stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 794 stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 795 membar(Sync); 796 for (cpu = 0; cpu < mp_ncpus; cpu++) { 797 if (CPU_ISSET(cpu, &cpus)) { 798 stxa(AA_INTR_SEND | (cpuid_to_mid[cpu] << 799 IDC_ITID_SHIFT), ASI_SDB_INTR_W, 0); 800 membar(Sync); 801 } 802 } 803 while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 804 IDR_CHEETAH_ALL_BUSY) != 0) 805 ; 806 intr_restore(s); 807 if ((ids & 808 (IDR_CHEETAH_ALL_BUSY | IDR_CHEETAH_ALL_NACK)) == 0) 809 return; 810 for (cpu = 0; cpu < mp_ncpus; cpu++) 811 if (CPU_ISSET(cpu, &cpus)) 812 if ((ids & (IDR_NACK << 813 (2 * cpuid_to_mid[cpu]))) == 0) 814 CPU_CLR(cpu, &cpus); 815 /* 816 * Leave interrupts enabled for a bit before retrying 817 * in order to avoid deadlocks if the other CPUs are 818 * also trying to send IPIs. 819 */ 820 DELAY(2 * mp_ncpus); 821 } 822 if (kdb_active != 0 || panicstr != NULL) 823 printf("%s: couldn't send IPI (cpus=%s ids=0x%lu)\n", 824 __func__, cpusetobj_strprint(pbuf, &cpus), ids); 825 else 826 panic("%s: couldn't send IPI (cpus=%s ids=0x%lu)", 827 __func__, cpusetobj_strprint(pbuf, &cpus), ids); 828} 829