189051Sjake/*- 289051Sjake * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved. 389051Sjake * 489051Sjake * Redistribution and use in source and binary forms, with or without 589051Sjake * modification, are permitted provided that the following conditions 689051Sjake * are met: 789051Sjake * 1. Redistributions of source code must retain the above copyright 889051Sjake * notice, this list of conditions and the following disclaimer. 989051Sjake * 2. Redistributions in binary form must reproduce the above copyright 1089051Sjake * notice, this list of conditions and the following disclaimer in the 1189051Sjake * documentation and/or other materials provided with the distribution. 1289051Sjake * 3. Berkeley Software Design Inc's name may not be used to endorse or 1389051Sjake * promote products derived from this software without specific prior 1489051Sjake * written permission. 1589051Sjake * 1689051Sjake * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 1789051Sjake * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1889051Sjake * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1989051Sjake * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 2089051Sjake * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2189051Sjake * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2289051Sjake * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2389051Sjake * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2489051Sjake * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2589051Sjake * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2689051Sjake * SUCH DAMAGE. 2789051Sjake * 2889051Sjake * from BSDI: locore.s,v 1.36.2.15 1999/08/23 22:34:41 cp Exp 2989051Sjake */ 3089051Sjake/*- 3189051Sjake * Copyright (c) 2002 Jake Burkholder. 32211050Smarius * Copyright (c) 2007 - 2010 Marius Strobl <marius@FreeBSD.org> 3389051Sjake * All rights reserved. 3489051Sjake * 3589051Sjake * Redistribution and use in source and binary forms, with or without 3689051Sjake * modification, are permitted provided that the following conditions 3789051Sjake * are met: 3889051Sjake * 1. Redistributions of source code must retain the above copyright 3989051Sjake * notice, this list of conditions and the following disclaimer. 4089051Sjake * 2. Redistributions in binary form must reproduce the above copyright 4189051Sjake * notice, this list of conditions and the following disclaimer in the 4289051Sjake * documentation and/or other materials provided with the distribution. 4389051Sjake * 4489051Sjake * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 4589051Sjake * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 4689051Sjake * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 4789051Sjake * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 4889051Sjake * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 4989051Sjake * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 5089051Sjake * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 5189051Sjake * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 5289051Sjake * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 5389051Sjake * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 5489051Sjake * SUCH DAMAGE. 5589051Sjake */ 5689051Sjake 57145150Smarius#include <sys/cdefs.h> 58145150Smarius__FBSDID("$FreeBSD$"); 59145150Smarius 6089051Sjake#include <sys/param.h> 6189051Sjake#include <sys/systm.h> 6289051Sjake#include <sys/lock.h> 63131950Smarcel#include <sys/kdb.h> 6489051Sjake#include <sys/kernel.h> 6589051Sjake#include <sys/ktr.h> 6689051Sjake#include <sys/mutex.h> 6789051Sjake#include <sys/pcpu.h> 6889051Sjake#include <sys/proc.h> 69170846Smarius#include <sys/sched.h> 7089051Sjake#include <sys/smp.h> 7189051Sjake 7289051Sjake#include <vm/vm.h> 7389051Sjake#include <vm/vm_param.h> 7489051Sjake#include <vm/pmap.h> 7589051Sjake#include <vm/vm_kern.h> 7689051Sjake#include <vm/vm_extern.h> 7789051Sjake#include <vm/vm_map.h> 7889051Sjake 7989051Sjake#include <dev/ofw/openfirm.h> 8089051Sjake 8189051Sjake#include <machine/asi.h> 8292205Sjake#include <machine/atomic.h> 83119696Smarcel#include <machine/bus.h> 84182730Smarius#include <machine/cpu.h> 8589051Sjake#include <machine/md_var.h> 8697511Sjake#include <machine/metadata.h> 8792205Sjake#include <machine/ofw_machdep.h> 88152022Sjhb#include <machine/pcb.h> 8989051Sjake#include <machine/smp.h> 9095132Sjake#include <machine/tick.h> 9191617Sjake#include <machine/tlb.h> 92216803Smarius#include <machine/tsb.h> 9389051Sjake#include <machine/tte.h> 94170846Smarius#include <machine/ver.h> 9589051Sjake 96183142Smarius#define SUNW_STARTCPU "SUNW,start-cpu" 97183142Smarius#define SUNW_STOPSELF "SUNW,stop-self" 98183142Smarius 9989051Sjakestatic ih_func_t cpu_ipi_ast; 100210601Smavstatic ih_func_t cpu_ipi_hardclock; 101178048Smariusstatic ih_func_t cpu_ipi_preempt; 10289051Sjakestatic ih_func_t cpu_ipi_stop; 10389051Sjake 10489051Sjake/* 10589051Sjake * Argument area used to pass data to non-boot processors as they start up. 106169796Smarius * This must be statically initialized with a known invalid CPU module ID, 107169796Smarius * since the other processors will use it before the boot CPU enters the 10889051Sjake * kernel. 10989051Sjake */ 110182730Smariusstruct cpu_start_args cpu_start_args = { 0, -1, -1, 0, 0, 0 }; 11197001Sjakestruct ipi_cache_args ipi_cache_args; 112211071Smariusstruct ipi_rd_args ipi_rd_args; 11391783Sjakestruct ipi_tlb_args ipi_tlb_args; 114152022Sjhbstruct pcb stoppcbs[MAXCPU]; 11589051Sjake 116170846Smariuscpu_ipi_selected_t *cpu_ipi_selected; 117211050Smariuscpu_ipi_single_t *cpu_ipi_single; 11891617Sjake 119170846Smariusstatic vm_offset_t mp_tramp; 120170846Smariusstatic u_int cpuid_to_mid[MAXCPU]; 121170846Smariusstatic int isjbus; 122222813Sattiliostatic volatile cpuset_t shutdown_cpus; 12392205Sjake 124204152Smariusstatic void ap_count(phandle_t node, u_int mid, u_int cpu_impl); 125204152Smariusstatic void ap_start(phandle_t node, u_int mid, u_int cpu_impl); 126169796Smariusstatic void cpu_mp_unleash(void *v); 127203838Smariusstatic void foreach_ap(phandle_t node, void (*func)(phandle_t node, 128204152Smarius u_int mid, u_int cpu_impl)); 129169796Smariusstatic void sun4u_startcpu(phandle_t cpu, void *func, u_long arg); 130169796Smarius 131170846Smariusstatic cpu_ipi_selected_t cheetah_ipi_selected; 132211050Smariusstatic cpu_ipi_single_t cheetah_ipi_single; 133211050Smariusstatic cpu_ipi_selected_t jalapeno_ipi_selected; 134211050Smariusstatic cpu_ipi_single_t jalapeno_ipi_single; 135170846Smariusstatic cpu_ipi_selected_t spitfire_ipi_selected; 136211050Smariusstatic cpu_ipi_single_t spitfire_ipi_single; 137170846Smarius 13891617SjakeSYSINIT(cpu_mp_unleash, SI_SUB_SMP, SI_ORDER_FIRST, cpu_mp_unleash, NULL); 13991617Sjake 140170846Smariusvoid 141204152Smariusmp_init(u_int cpu_impl) 14291617Sjake{ 14391617Sjake struct tte *tp; 14491617Sjake int i; 14591617Sjake 146170846Smarius mp_tramp = (vm_offset_t)OF_claim(NULL, PAGE_SIZE, PAGE_SIZE); 147170846Smarius if (mp_tramp == (vm_offset_t)-1) 148169796Smarius panic("%s", __func__); 149170846Smarius bcopy(mp_tramp_code, (void *)mp_tramp, mp_tramp_code_len); 150170846Smarius *(vm_offset_t *)(mp_tramp + mp_tramp_tlb_slots) = kernel_tlb_slots; 151170846Smarius *(vm_offset_t *)(mp_tramp + mp_tramp_func) = (vm_offset_t)mp_startup; 152170846Smarius tp = (struct tte *)(mp_tramp + mp_tramp_code_len); 15397511Sjake for (i = 0; i < kernel_tlb_slots; i++) { 154102042Sjake tp[i].tte_vpn = TV_VPN(kernel_tlbs[i].te_va, TS_4M); 15597511Sjake tp[i].tte_data = TD_V | TD_4M | TD_PA(kernel_tlbs[i].te_pa) | 15697511Sjake TD_L | TD_CP | TD_CV | TD_P | TD_W; 15797511Sjake } 158170846Smarius for (i = 0; i < PAGE_SIZE; i += sizeof(vm_offset_t)) 159170846Smarius flush(mp_tramp + i); 160170846Smarius 161170846Smarius /* 162170846Smarius * On UP systems cpu_ipi_selected() can be called while 163170846Smarius * cpu_mp_start() wasn't so initialize these here. 164170846Smarius */ 165170846Smarius if (cpu_impl == CPU_IMPL_ULTRASPARCIIIi || 166211050Smarius cpu_impl == CPU_IMPL_ULTRASPARCIIIip) { 167170846Smarius isjbus = 1; 168211050Smarius cpu_ipi_selected = jalapeno_ipi_selected; 169211050Smarius cpu_ipi_single = jalapeno_ipi_single; 170211050Smarius } else if (cpu_impl == CPU_IMPL_SPARC64V || 171211050Smarius cpu_impl >= CPU_IMPL_ULTRASPARCIII) { 172170846Smarius cpu_ipi_selected = cheetah_ipi_selected; 173211050Smarius cpu_ipi_single = cheetah_ipi_single; 174211050Smarius } else { 175170846Smarius cpu_ipi_selected = spitfire_ipi_selected; 176211050Smarius cpu_ipi_single = spitfire_ipi_single; 177211050Smarius } 17891617Sjake} 17991617Sjake 180203838Smariusstatic void 181204152Smariusforeach_ap(phandle_t node, void (*func)(phandle_t node, u_int mid, 182204152Smarius u_int cpu_impl)) 183203838Smarius{ 184203838Smarius char type[sizeof("cpu")]; 185203838Smarius phandle_t child; 186203838Smarius u_int cpuid; 187204152Smarius uint32_t cpu_impl; 188211050Smarius 189203838Smarius /* There's no need to traverse the whole OFW tree twice. */ 190203838Smarius if (mp_maxid > 0 && mp_ncpus >= mp_maxid + 1) 191203838Smarius return; 192203838Smarius 193203838Smarius for (; node != 0; node = OF_peer(node)) { 194203838Smarius child = OF_child(node); 195203838Smarius if (child > 0) 196203838Smarius foreach_ap(child, func); 197203838Smarius else { 198203838Smarius if (OF_getprop(node, "device_type", type, 199203838Smarius sizeof(type)) <= 0) 200203838Smarius continue; 201203838Smarius if (strcmp(type, "cpu") != 0) 202203838Smarius continue; 203204152Smarius if (OF_getprop(node, "implementation#", &cpu_impl, 204204152Smarius sizeof(cpu_impl)) <= 0) 205204152Smarius panic("%s: couldn't determine CPU " 206204152Smarius "implementation", __func__); 207204152Smarius if (OF_getprop(node, cpu_cpuid_prop(cpu_impl), &cpuid, 208211050Smarius sizeof(cpuid)) <= 0) 209204152Smarius panic("%s: couldn't determine CPU module ID", 210204152Smarius __func__); 211203838Smarius if (cpuid == PCPU_GET(mid)) 212203838Smarius continue; 213204152Smarius (*func)(node, cpuid, cpu_impl); 214203838Smarius } 215203838Smarius } 216203838Smarius} 217203838Smarius 21889051Sjake/* 219169796Smarius * Probe for other CPUs. 22089051Sjake */ 221122947Sjhbvoid 222203838Smariuscpu_mp_setmaxid() 22389051Sjake{ 22489051Sjake 225222813Sattilio CPU_SETOF(curcpu, &all_cpus); 22689051Sjake mp_ncpus = 1; 227203838Smarius mp_maxid = 0; 22889051Sjake 229203838Smarius foreach_ap(OF_child(OF_peer(0)), ap_count); 23089051Sjake} 23189051Sjake 232203838Smariusstatic void 233204152Smariusap_count(phandle_t node __unused, u_int mid __unused, u_int cpu_impl __unused) 234203838Smarius{ 235203838Smarius 236203838Smarius mp_maxid++; 237203838Smarius} 238203838Smarius 239122947Sjhbint 240122947Sjhbcpu_mp_probe(void) 241122947Sjhb{ 242122947Sjhb 243123126Sjhb return (mp_maxid > 0); 244122947Sjhb} 245122947Sjhb 246176734Sjeffstruct cpu_group * 247176734Sjeffcpu_topo(void) 248176734Sjeff{ 249176734Sjeff 250176994Smarius return (smp_topo_none()); 251176734Sjeff} 252176734Sjeff 25391617Sjakestatic void 25491617Sjakesun4u_startcpu(phandle_t cpu, void *func, u_long arg) 25591617Sjake{ 25691617Sjake static struct { 25791617Sjake cell_t name; 25891617Sjake cell_t nargs; 25991617Sjake cell_t nreturns; 26091617Sjake cell_t cpu; 26191617Sjake cell_t func; 26291617Sjake cell_t arg; 26391617Sjake } args = { 264183142Smarius (cell_t)SUNW_STARTCPU, 26591617Sjake 3, 26691617Sjake }; 26791617Sjake 26891617Sjake args.cpu = cpu; 26991617Sjake args.func = (cell_t)func; 27091617Sjake args.arg = (cell_t)arg; 271186347Snwhitehorn ofw_entry(&args); 27291617Sjake} 27391617Sjake 27489051Sjake/* 27589051Sjake * Fire up any non-boot processors. 27689051Sjake */ 27789051Sjakevoid 27889051Sjakecpu_mp_start(void) 27989051Sjake{ 28089051Sjake 28189051Sjake intr_setup(PIL_AST, cpu_ipi_ast, -1, NULL, NULL); 28289051Sjake intr_setup(PIL_RENDEZVOUS, (ih_func_t *)smp_rendezvous_action, 28389051Sjake -1, NULL, NULL); 28489051Sjake intr_setup(PIL_STOP, cpu_ipi_stop, -1, NULL, NULL); 285178048Smarius intr_setup(PIL_PREEMPT, cpu_ipi_preempt, -1, NULL, NULL); 286210601Smav intr_setup(PIL_HARDCLOCK, cpu_ipi_hardclock, -1, NULL, NULL); 28789051Sjake 288169796Smarius cpuid_to_mid[curcpu] = PCPU_GET(mid); 289157240Smarius 290203838Smarius foreach_ap(OF_child(OF_peer(0)), ap_start); 291203838Smarius KASSERT(!isjbus || mp_ncpus <= IDR_JALAPENO_MAX_BN_PAIRS, 292203838Smarius ("%s: can only IPI a maximum of %d JBus-CPUs", 293203838Smarius __func__, IDR_JALAPENO_MAX_BN_PAIRS)); 294203838Smarius smp_active = 1; 295203838Smarius} 296203838Smarius 297203838Smariusstatic void 298204152Smariusap_start(phandle_t node, u_int mid, u_int cpu_impl) 299203838Smarius{ 300203838Smarius volatile struct cpu_start_args *csa; 301203838Smarius struct pcpu *pc; 302203838Smarius register_t s; 303203838Smarius vm_offset_t va; 304203838Smarius u_int cpuid; 305204152Smarius uint32_t clock; 306203838Smarius 307203838Smarius if (mp_ncpus > MAXCPU) 308203838Smarius return; 309203838Smarius 310203838Smarius if (OF_getprop(node, "clock-frequency", &clock, sizeof(clock)) <= 0) 311204152Smarius panic("%s: couldn't determine CPU frequency", __func__); 312203838Smarius if (clock != PCPU_GET(clock)) 313214071Smarius tick_et_use_stick = 1; 314203838Smarius 31589051Sjake csa = &cpu_start_args; 316203838Smarius csa->csa_state = 0; 317203838Smarius sun4u_startcpu(node, (void *)mp_tramp, 0); 318203838Smarius s = intr_disable(); 319203838Smarius while (csa->csa_state != CPU_TICKSYNC) 320203838Smarius ; 321203838Smarius membar(StoreLoad); 322203838Smarius csa->csa_tick = rd(tick); 323207537Smarius if (cpu_impl == CPU_IMPL_SPARC64V || 324207537Smarius cpu_impl >= CPU_IMPL_ULTRASPARCIII) { 325203838Smarius while (csa->csa_state != CPU_STICKSYNC) 32691617Sjake ; 32791617Sjake membar(StoreLoad); 328203838Smarius csa->csa_stick = rdstick(); 329203838Smarius } 330203838Smarius while (csa->csa_state != CPU_INIT) 331203838Smarius ; 332203838Smarius csa->csa_tick = csa->csa_stick = 0; 333203838Smarius intr_restore(s); 33491617Sjake 335203838Smarius cpuid = mp_ncpus++; 336203838Smarius cpuid_to_mid[cpuid] = mid; 337203838Smarius cpu_identify(csa->csa_ver, clock, cpuid); 33891617Sjake 339254025Sjeff va = kmem_malloc(kernel_arena, PCPU_PAGES * PAGE_SIZE, 340254025Sjeff M_WAITOK | M_ZERO); 341203838Smarius pc = (struct pcpu *)(va + (PCPU_PAGES * PAGE_SIZE)) - 1; 342203838Smarius pcpu_init(pc, cpuid, sizeof(*pc)); 343254025Sjeff dpcpu_init((void *)kmem_malloc(kernel_arena, DPCPU_SIZE, 344254025Sjeff M_WAITOK | M_ZERO), cpuid); 345203838Smarius pc->pc_addr = va; 346203838Smarius pc->pc_clock = clock; 347204152Smarius pc->pc_impl = cpu_impl; 348203838Smarius pc->pc_mid = mid; 349203838Smarius pc->pc_node = node; 35089051Sjake 351203838Smarius cache_init(pc); 352182689Smarius 353222813Sattilio CPU_SET(cpuid, &all_cpus); 354203838Smarius intr_add_cpu(cpuid); 35589051Sjake} 35689051Sjake 35789051Sjakevoid 35889051Sjakecpu_mp_announce(void) 35989051Sjake{ 360169796Smarius 36189051Sjake} 36289051Sjake 363169796Smariusstatic void 36491617Sjakecpu_mp_unleash(void *v) 36589051Sjake{ 36691617Sjake volatile struct cpu_start_args *csa; 36791617Sjake struct pcpu *pc; 368169796Smarius register_t s; 36991617Sjake vm_offset_t va; 370113238Sjake vm_paddr_t pa; 371181701Smarius u_int ctx_inc; 37291617Sjake u_int ctx_min; 37391617Sjake int i; 37489051Sjake 37591783Sjake ctx_min = TLB_CTX_USER_MIN; 37691783Sjake ctx_inc = (TLB_CTX_USER_MAX - 1) / mp_ncpus; 37789051Sjake csa = &cpu_start_args; 37891783Sjake csa->csa_count = mp_ncpus; 379222531Snwhitehorn STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { 38091617Sjake pc->pc_tlb_ctx = ctx_min; 38191617Sjake pc->pc_tlb_ctx_min = ctx_min; 38291617Sjake pc->pc_tlb_ctx_max = ctx_min + ctx_inc; 38391617Sjake ctx_min += ctx_inc; 38489051Sjake 385169796Smarius if (pc->pc_cpuid == curcpu) 38691617Sjake continue; 38791617Sjake KASSERT(pc->pc_idlethread != NULL, 388169796Smarius ("%s: idlethread", __func__)); 389169796Smarius pc->pc_curthread = pc->pc_idlethread; 39091617Sjake pc->pc_curpcb = pc->pc_curthread->td_pcb; 39191617Sjake for (i = 0; i < PCPU_PAGES; i++) { 39291617Sjake va = pc->pc_addr + i * PAGE_SIZE; 39391617Sjake pa = pmap_kextract(va); 39491617Sjake if (pa == 0) 395169796Smarius panic("%s: pmap_kextract", __func__); 396102042Sjake csa->csa_ttes[i].tte_vpn = TV_VPN(va, TS_8K); 39791617Sjake csa->csa_ttes[i].tte_data = TD_V | TD_8K | TD_PA(pa) | 39891617Sjake TD_L | TD_CP | TD_CV | TD_P | TD_W; 39991617Sjake } 40091617Sjake csa->csa_state = 0; 40191783Sjake csa->csa_pcpu = pc->pc_addr; 40291617Sjake csa->csa_mid = pc->pc_mid; 40391617Sjake s = intr_disable(); 40491617Sjake while (csa->csa_state != CPU_BOOTSTRAP) 40591617Sjake ; 40691617Sjake intr_restore(s); 40791617Sjake } 40891783Sjake 40991783Sjake membar(StoreLoad); 410169796Smarius csa->csa_count = 0; 41192205Sjake smp_started = 1; 41291617Sjake} 41389051Sjake 41491617Sjakevoid 41591617Sjakecpu_mp_bootstrap(struct pcpu *pc) 41691617Sjake{ 41791617Sjake volatile struct cpu_start_args *csa; 41889051Sjake 41991617Sjake csa = &cpu_start_args; 420207248Smarius 421207248Smarius /* Do CPU-specific initialization. */ 422223719Smarius if (pc->pc_impl >= CPU_IMPL_ULTRASPARCIII) 423204152Smarius cheetah_init(pc->pc_impl); 424223719Smarius else if (pc->pc_impl == CPU_IMPL_SPARC64V) 425223719Smarius zeus_init(pc->pc_impl); 426223719Smarius 427207248Smarius /* 428207248Smarius * Enable the caches. Note that his may include applying workarounds. 429207248Smarius */ 430204152Smarius cache_enable(pc->pc_impl); 431207248Smarius 432213868Smarius /* 433213868Smarius * Clear (S)TICK timer(s) (including NPT) and ensure they are stopped. 434213868Smarius */ 435213868Smarius tick_clear(pc->pc_impl); 436213868Smarius tick_stop(pc->pc_impl); 437213868Smarius 438216803Smarius /* Set the kernel context. */ 439216803Smarius pmap_set_kctx(); 440207248Smarius 441216803Smarius /* Lock the kernel TSB in the TLB if necessary. */ 442216803Smarius if (tsb_kernel_ldd_phys == 0) 443216803Smarius pmap_map_tsb(); 444216803Smarius 445176994Smarius /* 446176994Smarius * Flush all non-locked TLB entries possibly left over by the 447176994Smarius * firmware. 448176994Smarius */ 449176994Smarius tlb_flush_nonlocked(); 450207248Smarius 451213868Smarius /* 452213868Smarius * Enable interrupts. 453213868Smarius * Note that the PIL we be lowered indirectly via sched_throw(NULL) 454213868Smarius * when fake spinlock held by the idle thread eventually is released. 455213868Smarius */ 456207248Smarius wrpr(pstate, 0, PSTATE_KERNEL); 457207248Smarius 45889051Sjake smp_cpus++; 459169796Smarius KASSERT(curthread != NULL, ("%s: curthread", __func__)); 460169796Smarius printf("SMP: AP CPU #%d Launched!\n", curcpu); 46189051Sjake 46291783Sjake csa->csa_count--; 46391783Sjake membar(StoreLoad); 46491617Sjake csa->csa_state = CPU_BOOTSTRAP; 46591783Sjake while (csa->csa_count != 0) 46691617Sjake ; 46789051Sjake 468210601Smav /* Start per-CPU event timers. */ 469210601Smav cpu_initclocks_ap(); 470210601Smav 471182020Smarius /* Ok, now enter the scheduler. */ 472170303Sjeff sched_throw(NULL); 47389051Sjake} 47489051Sjake 47592205Sjakevoid 47692205Sjakecpu_mp_shutdown(void) 47792205Sjake{ 478222813Sattilio cpuset_t cpus; 47992205Sjake int i; 48092205Sjake 48192205Sjake critical_enter(); 482223346Smarius shutdown_cpus = all_cpus; 483223346Smarius CPU_CLR(PCPU_GET(cpuid), &shutdown_cpus); 484222813Sattilio cpus = shutdown_cpus; 485222813Sattilio 486222813Sattilio /* XXX: Stop all the CPUs which aren't already. */ 487222813Sattilio if (CPU_CMP(&stopped_cpus, &cpus)) { 488222813Sattilio 489223346Smarius /* cpus is just a flat "on" mask without curcpu. */ 490222813Sattilio CPU_NAND(&cpus, &stopped_cpus); 491222813Sattilio stop_cpus(cpus); 492222813Sattilio } 49392205Sjake i = 0; 494222813Sattilio while (!CPU_EMPTY(&shutdown_cpus)) { 49592205Sjake if (i++ > 100000) { 49692205Sjake printf("timeout shutting down CPUs.\n"); 49792205Sjake break; 49892205Sjake } 49992205Sjake } 50092205Sjake critical_exit(); 50192205Sjake} 50292205Sjake 50389051Sjakestatic void 504239864Smariuscpu_ipi_ast(struct trapframe *tf __unused) 50589051Sjake{ 506169796Smarius 50789051Sjake} 50889051Sjake 50989051Sjakestatic void 510239864Smariuscpu_ipi_stop(struct trapframe *tf __unused) 51189051Sjake{ 512223346Smarius u_int cpuid; 51392205Sjake 514169796Smarius CTR2(KTR_SMP, "%s: stopped %d", __func__, curcpu); 515222813Sattilio sched_pin(); 516169796Smarius savectx(&stoppcbs[curcpu]); 517223346Smarius cpuid = PCPU_GET(cpuid); 518223346Smarius CPU_SET_ATOMIC(cpuid, &stopped_cpus); 519223346Smarius while (!CPU_ISSET(cpuid, &started_cpus)) { 520223346Smarius if (CPU_ISSET(cpuid, &shutdown_cpus)) { 521223346Smarius CPU_CLR_ATOMIC(cpuid, &shutdown_cpus); 522183142Smarius (void)intr_disable(); 523183142Smarius for (;;) 524183142Smarius ; 52592205Sjake } 52692205Sjake } 527223346Smarius CPU_CLR_ATOMIC(cpuid, &started_cpus); 528223346Smarius CPU_CLR_ATOMIC(cpuid, &stopped_cpus); 529222813Sattilio sched_unpin(); 530169796Smarius CTR2(KTR_SMP, "%s: restarted %d", __func__, curcpu); 53189051Sjake} 53289051Sjake 533170846Smariusstatic void 534178048Smariuscpu_ipi_preempt(struct trapframe *tf) 535178048Smarius{ 536178048Smarius 537178048Smarius sched_preempt(curthread); 538178048Smarius} 539178048Smarius 540178048Smariusstatic void 541210601Smavcpu_ipi_hardclock(struct trapframe *tf) 542210601Smav{ 543212541Smav struct trapframe *oldframe; 544212541Smav struct thread *td; 545210601Smav 546212541Smav critical_enter(); 547212541Smav td = curthread; 548212541Smav td->td_intr_nesting_level++; 549212541Smav oldframe = td->td_intr_frame; 550212541Smav td->td_intr_frame = tf; 551212541Smav hardclockintr(); 552212541Smav td->td_intr_frame = oldframe; 553212541Smav td->td_intr_nesting_level--; 554212541Smav critical_exit(); 555210601Smav} 556210601Smav 557210601Smavstatic void 558222813Sattiliospitfire_ipi_selected(cpuset_t cpus, u_long d0, u_long d1, u_long d2) 55989051Sjake{ 56089051Sjake u_int cpu; 56189051Sjake 562251703Sjeff while ((cpu = CPU_FFS(&cpus)) != 0) { 563222813Sattilio cpu--; 564222813Sattilio CPU_CLR(cpu, &cpus); 565211050Smarius spitfire_ipi_single(cpu, d0, d1, d2); 56689051Sjake } 56789051Sjake} 56889051Sjake 569169796Smariusstatic void 570211050Smariusspitfire_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2) 57189051Sjake{ 572169796Smarius register_t s; 573157240Smarius u_long ids; 574211050Smarius u_int mid; 57589051Sjake int i; 57689051Sjake 577211050Smarius KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__)); 57889051Sjake KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & IDR_BUSY) == 0, 579169796Smarius ("%s: outstanding dispatch", __func__)); 580211050Smarius mid = cpuid_to_mid[cpu]; 58189051Sjake for (i = 0; i < IPI_RETRIES; i++) { 58291783Sjake s = intr_disable(); 58389051Sjake stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 58489051Sjake stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 58589051Sjake stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 586170846Smarius membar(Sync); 587169796Smarius stxa(AA_INTR_SEND | (mid << IDC_ITID_SHIFT), 588169796Smarius ASI_SDB_INTR_W, 0); 589157240Smarius /* 590157240Smarius * Workaround for SpitFire erratum #54; do a dummy read 591157240Smarius * from a SDB internal register before the MEMBAR #Sync 592157240Smarius * for the write to ASI_SDB_INTR_W (requiring another 593157240Smarius * MEMBAR #Sync in order to make sure the write has 594157240Smarius * occurred before the load). 595157240Smarius */ 59689051Sjake membar(Sync); 597157240Smarius (void)ldxa(AA_SDB_CNTL_HIGH, ASI_SDB_CONTROL_R); 598157240Smarius membar(Sync); 599169796Smarius while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 600169796Smarius IDR_BUSY) != 0) 60189051Sjake ; 60291783Sjake intr_restore(s); 603170846Smarius if ((ids & (IDR_BUSY | IDR_NACK)) == 0) 60489051Sjake return; 605161966Smarius /* 606161966Smarius * Leave interrupts enabled for a bit before retrying 607161966Smarius * in order to avoid deadlocks if the other CPU is also 608161966Smarius * trying to send an IPI. 609161966Smarius */ 610161966Smarius DELAY(2); 61189051Sjake } 612190106Smarius if (kdb_active != 0 || panicstr != NULL) 613169796Smarius printf("%s: couldn't send IPI to module 0x%u\n", 614169796Smarius __func__, mid); 61592205Sjake else 616186395Smarius panic("%s: couldn't send IPI to module 0x%u", 617186395Smarius __func__, mid); 61889051Sjake} 61989051Sjake 620170846Smariusstatic void 621211050Smariuscheetah_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2) 622211050Smarius{ 623211050Smarius register_t s; 624211050Smarius u_long ids; 625211050Smarius u_int mid; 626211050Smarius int i; 627211050Smarius 628211050Smarius KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__)); 629211050Smarius KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & 630211050Smarius IDR_CHEETAH_ALL_BUSY) == 0, 631211050Smarius ("%s: outstanding dispatch", __func__)); 632211050Smarius mid = cpuid_to_mid[cpu]; 633211050Smarius for (i = 0; i < IPI_RETRIES; i++) { 634211050Smarius s = intr_disable(); 635211050Smarius stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 636211050Smarius stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 637211050Smarius stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 638211050Smarius membar(Sync); 639211050Smarius stxa(AA_INTR_SEND | (mid << IDC_ITID_SHIFT), 640211050Smarius ASI_SDB_INTR_W, 0); 641211050Smarius membar(Sync); 642211050Smarius while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 643211050Smarius IDR_BUSY) != 0) 644211050Smarius ; 645211050Smarius intr_restore(s); 646211050Smarius if ((ids & (IDR_BUSY | IDR_NACK)) == 0) 647211050Smarius return; 648211050Smarius /* 649211050Smarius * Leave interrupts enabled for a bit before retrying 650211050Smarius * in order to avoid deadlocks if the other CPU is also 651211050Smarius * trying to send an IPI. 652211050Smarius */ 653211050Smarius DELAY(2); 654211050Smarius } 655211050Smarius if (kdb_active != 0 || panicstr != NULL) 656211050Smarius printf("%s: couldn't send IPI to module 0x%u\n", 657211050Smarius __func__, mid); 658211050Smarius else 659211050Smarius panic("%s: couldn't send IPI to module 0x%u", 660211050Smarius __func__, mid); 661211050Smarius} 662211050Smarius 663211050Smariusstatic void 664222813Sattiliocheetah_ipi_selected(cpuset_t cpus, u_long d0, u_long d1, u_long d2) 665170846Smarius{ 666222813Sattilio char pbuf[CPUSETBUFSIZ]; 667170846Smarius register_t s; 668170846Smarius u_long ids; 669170846Smarius u_int bnp; 670170846Smarius u_int cpu; 671170846Smarius int i; 672170846Smarius 673222813Sattilio KASSERT(!CPU_ISSET(curcpu, &cpus), ("%s: CPU can't IPI itself", 674222813Sattilio __func__)); 675170846Smarius KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & 676170846Smarius IDR_CHEETAH_ALL_BUSY) == 0, 677170846Smarius ("%s: outstanding dispatch", __func__)); 678222813Sattilio if (CPU_EMPTY(&cpus)) 679170846Smarius return; 680170846Smarius ids = 0; 681170846Smarius for (i = 0; i < IPI_RETRIES * mp_ncpus; i++) { 682170846Smarius s = intr_disable(); 683170846Smarius stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 684170846Smarius stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 685170846Smarius stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 686170846Smarius membar(Sync); 687170846Smarius bnp = 0; 688170846Smarius for (cpu = 0; cpu < mp_ncpus; cpu++) { 689222813Sattilio if (CPU_ISSET(cpu, &cpus)) { 690211050Smarius stxa(AA_INTR_SEND | (cpuid_to_mid[cpu] << 691211050Smarius IDC_ITID_SHIFT) | bnp << IDC_BN_SHIFT, 692170846Smarius ASI_SDB_INTR_W, 0); 693170846Smarius membar(Sync); 694170846Smarius bnp++; 695223806Smarius if (bnp == IDR_CHEETAH_MAX_BN_PAIRS) 696223806Smarius break; 697170846Smarius } 698170846Smarius } 699170846Smarius while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 700170846Smarius IDR_CHEETAH_ALL_BUSY) != 0) 701170846Smarius ; 702170846Smarius intr_restore(s); 703170846Smarius bnp = 0; 704170846Smarius for (cpu = 0; cpu < mp_ncpus; cpu++) { 705222813Sattilio if (CPU_ISSET(cpu, &cpus)) { 706211050Smarius if ((ids & (IDR_NACK << (2 * bnp))) == 0) 707222813Sattilio CPU_CLR(cpu, &cpus); 708170846Smarius bnp++; 709170846Smarius } 710170846Smarius } 711222813Sattilio if (CPU_EMPTY(&cpus)) 712186395Smarius return; 713186395Smarius /* 714170846Smarius * Leave interrupts enabled for a bit before retrying 715170846Smarius * in order to avoid deadlocks if the other CPUs are 716170846Smarius * also trying to send IPIs. 717170846Smarius */ 718186395Smarius DELAY(2 * mp_ncpus); 719170846Smarius } 720190106Smarius if (kdb_active != 0 || panicstr != NULL) 721222813Sattilio printf("%s: couldn't send IPI (cpus=%s ids=0x%lu)\n", 722222813Sattilio __func__, cpusetobj_strprint(pbuf, &cpus), ids); 723170846Smarius else 724222813Sattilio panic("%s: couldn't send IPI (cpus=%s ids=0x%lu)", 725222813Sattilio __func__, cpusetobj_strprint(pbuf, &cpus), ids); 726170846Smarius} 727211050Smarius 728211050Smariusstatic void 729211050Smariusjalapeno_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2) 730211050Smarius{ 731211050Smarius register_t s; 732211050Smarius u_long ids; 733211050Smarius u_int busy, busynack, mid; 734211050Smarius int i; 735211050Smarius 736211050Smarius KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__)); 737211050Smarius KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & 738211050Smarius IDR_CHEETAH_ALL_BUSY) == 0, 739211050Smarius ("%s: outstanding dispatch", __func__)); 740211050Smarius mid = cpuid_to_mid[cpu]; 741211050Smarius busy = IDR_BUSY << (2 * mid); 742211050Smarius busynack = (IDR_BUSY | IDR_NACK) << (2 * mid); 743211050Smarius for (i = 0; i < IPI_RETRIES; i++) { 744211050Smarius s = intr_disable(); 745211050Smarius stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 746211050Smarius stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 747211050Smarius stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 748211050Smarius membar(Sync); 749211050Smarius stxa(AA_INTR_SEND | (mid << IDC_ITID_SHIFT), 750211050Smarius ASI_SDB_INTR_W, 0); 751211050Smarius membar(Sync); 752211050Smarius while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 753211050Smarius busy) != 0) 754211050Smarius ; 755211050Smarius intr_restore(s); 756211050Smarius if ((ids & busynack) == 0) 757211050Smarius return; 758211050Smarius /* 759211050Smarius * Leave interrupts enabled for a bit before retrying 760211050Smarius * in order to avoid deadlocks if the other CPU is also 761211050Smarius * trying to send an IPI. 762211050Smarius */ 763211050Smarius DELAY(2); 764211050Smarius } 765211050Smarius if (kdb_active != 0 || panicstr != NULL) 766211050Smarius printf("%s: couldn't send IPI to module 0x%u\n", 767211050Smarius __func__, mid); 768211050Smarius else 769211050Smarius panic("%s: couldn't send IPI to module 0x%u", 770211050Smarius __func__, mid); 771211050Smarius} 772211050Smarius 773211050Smariusstatic void 774222813Sattiliojalapeno_ipi_selected(cpuset_t cpus, u_long d0, u_long d1, u_long d2) 775211050Smarius{ 776222813Sattilio char pbuf[CPUSETBUFSIZ]; 777211050Smarius register_t s; 778211050Smarius u_long ids; 779211050Smarius u_int cpu; 780211050Smarius int i; 781211050Smarius 782222813Sattilio KASSERT(!CPU_ISSET(curcpu, &cpus), ("%s: CPU can't IPI itself", 783222813Sattilio __func__)); 784211050Smarius KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & 785211050Smarius IDR_CHEETAH_ALL_BUSY) == 0, 786211050Smarius ("%s: outstanding dispatch", __func__)); 787222813Sattilio if (CPU_EMPTY(&cpus)) 788211050Smarius return; 789211050Smarius ids = 0; 790211050Smarius for (i = 0; i < IPI_RETRIES * mp_ncpus; i++) { 791211050Smarius s = intr_disable(); 792211050Smarius stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 793211050Smarius stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 794211050Smarius stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 795211050Smarius membar(Sync); 796211050Smarius for (cpu = 0; cpu < mp_ncpus; cpu++) { 797222813Sattilio if (CPU_ISSET(cpu, &cpus)) { 798211050Smarius stxa(AA_INTR_SEND | (cpuid_to_mid[cpu] << 799211050Smarius IDC_ITID_SHIFT), ASI_SDB_INTR_W, 0); 800211050Smarius membar(Sync); 801211050Smarius } 802211050Smarius } 803211050Smarius while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 804211050Smarius IDR_CHEETAH_ALL_BUSY) != 0) 805211050Smarius ; 806211050Smarius intr_restore(s); 807211050Smarius if ((ids & 808211050Smarius (IDR_CHEETAH_ALL_BUSY | IDR_CHEETAH_ALL_NACK)) == 0) 809211050Smarius return; 810211050Smarius for (cpu = 0; cpu < mp_ncpus; cpu++) 811222813Sattilio if (CPU_ISSET(cpu, &cpus)) 812211050Smarius if ((ids & (IDR_NACK << 813211050Smarius (2 * cpuid_to_mid[cpu]))) == 0) 814222813Sattilio CPU_CLR(cpu, &cpus); 815211050Smarius /* 816211050Smarius * Leave interrupts enabled for a bit before retrying 817211050Smarius * in order to avoid deadlocks if the other CPUs are 818211050Smarius * also trying to send IPIs. 819211050Smarius */ 820211050Smarius DELAY(2 * mp_ncpus); 821211050Smarius } 822211050Smarius if (kdb_active != 0 || panicstr != NULL) 823222813Sattilio printf("%s: couldn't send IPI (cpus=%s ids=0x%lu)\n", 824222813Sattilio __func__, cpusetobj_strprint(pbuf, &cpus), ids); 825211050Smarius else 826222813Sattilio panic("%s: couldn't send IPI (cpus=%s ids=0x%lu)", 827222813Sattilio __func__, cpusetobj_strprint(pbuf, &cpus), ids); 828211050Smarius} 829