1239268Sgonzo/*- 2239268Sgonzo * Copyright (c) 2011 Semihalf. 3239268Sgonzo * All rights reserved. 4239268Sgonzo * 5239268Sgonzo * Redistribution and use in source and binary forms, with or without 6239268Sgonzo * modification, are permitted provided that the following conditions 7239268Sgonzo * are met: 8239268Sgonzo * 1. Redistributions of source code must retain the above copyright 9239268Sgonzo * notice, this list of conditions and the following disclaimer. 10239268Sgonzo * 2. Redistributions in binary form must reproduce the above copyright 11239268Sgonzo * notice, this list of conditions and the following disclaimer in the 12239268Sgonzo * documentation and/or other materials provided with the distribution. 13239268Sgonzo * 14239268Sgonzo * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15239268Sgonzo * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16239268Sgonzo * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17239268Sgonzo * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18239268Sgonzo * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19239268Sgonzo * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20239268Sgonzo * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21239268Sgonzo * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22239268Sgonzo * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23239268Sgonzo * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24239268Sgonzo * SUCH DAMAGE. 25239268Sgonzo */ 26239268Sgonzo#include <sys/cdefs.h> 27239268Sgonzo__FBSDID("$FreeBSD$"); 28239268Sgonzo#include <sys/param.h> 29239268Sgonzo#include <sys/systm.h> 30239268Sgonzo#include <sys/bus.h> 31239268Sgonzo#include <sys/kernel.h> 32239268Sgonzo#include <sys/lock.h> 33239268Sgonzo#include <sys/mutex.h> 34239268Sgonzo#include <sys/proc.h> 35239268Sgonzo#include <sys/pcpu.h> 36239268Sgonzo#include <sys/sched.h> 37239268Sgonzo#include <sys/smp.h> 38239268Sgonzo#include <sys/ktr.h> 39239268Sgonzo#include <sys/malloc.h> 40239268Sgonzo 41239268Sgonzo#include <vm/vm.h> 42239268Sgonzo#include <vm/vm_extern.h> 43239268Sgonzo#include <vm/vm_kern.h> 44239268Sgonzo#include <vm/pmap.h> 45239268Sgonzo 46239268Sgonzo#include <machine/cpu.h> 47266374Sian#include <machine/cpufunc.h> 48239268Sgonzo#include <machine/smp.h> 49239268Sgonzo#include <machine/pcb.h> 50239268Sgonzo#include <machine/pte.h> 51266194Sian#include <machine/physmem.h> 52239268Sgonzo#include <machine/intr.h> 53239268Sgonzo#include <machine/vmparam.h> 54254461Sandrew#ifdef VFP 55247339Scognet#include <machine/vfp.h> 56247339Scognet#endif 57266046Sian#ifdef CPU_MV_PJ4B 58266046Sian#include <arm/mv/mvwin.h> 59266046Sian#include <dev/fdt/fdt_common.h> 60266046Sian#endif 61239268Sgonzo 62239268Sgonzo#include "opt_smp.h" 63239268Sgonzo 64239268Sgonzovoid *temp_pagetable; 65239268Sgonzoextern struct pcpu __pcpu[]; 66239268Sgonzo/* used to hold the AP's until we are ready to release them */ 67239268Sgonzostruct mtx ap_boot_mtx; 68239268Sgonzostruct pcb stoppcbs[MAXCPU]; 69239268Sgonzo 70239268Sgonzo/* # of Applications processors */ 71239268Sgonzovolatile int mp_naps; 72239268Sgonzo 73239268Sgonzo/* Set to 1 once we're ready to let the APs out of the pen. */ 74239268Sgonzovolatile int aps_ready = 0; 75239268Sgonzo 76239268Sgonzostatic int ipi_handler(void *arg); 77239268Sgonzovoid set_stackptrs(int cpu); 78239268Sgonzo 79239268Sgonzo/* Temporary variables for init_secondary() */ 80239268Sgonzovoid *dpcpu[MAXCPU - 1]; 81239268Sgonzo 82239268Sgonzo/* Determine if we running MP machine */ 83239268Sgonzoint 84239268Sgonzocpu_mp_probe(void) 85239268Sgonzo{ 86239268Sgonzo CPU_SETOF(0, &all_cpus); 87239268Sgonzo 88239268Sgonzo return (platform_mp_probe()); 89239268Sgonzo} 90239268Sgonzo 91239268Sgonzo/* Start Application Processor via platform specific function */ 92239268Sgonzostatic int 93239268Sgonzocheck_ap(void) 94239268Sgonzo{ 95239268Sgonzo uint32_t ms; 96239268Sgonzo 97239268Sgonzo for (ms = 0; ms < 2000; ++ms) { 98239268Sgonzo if ((mp_naps + 1) == mp_ncpus) 99239268Sgonzo return (0); /* success */ 100239268Sgonzo else 101239268Sgonzo DELAY(1000); 102239268Sgonzo } 103239268Sgonzo 104239268Sgonzo return (-2); 105239268Sgonzo} 106239268Sgonzo 107239268Sgonzoextern unsigned char _end[]; 108239268Sgonzo 109239268Sgonzo/* Initialize and fire up non-boot processors */ 110239268Sgonzovoid 111239268Sgonzocpu_mp_start(void) 112239268Sgonzo{ 113239268Sgonzo int error, i; 114239268Sgonzo vm_offset_t temp_pagetable_va; 115239268Sgonzo vm_paddr_t addr, addr_end; 116239268Sgonzo 117239268Sgonzo mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN); 118239268Sgonzo 119239268Sgonzo /* Reserve memory for application processors */ 120239268Sgonzo for(i = 0; i < (mp_ncpus - 1); i++) 121254025Sjeff dpcpu[i] = (void *)kmem_malloc(kernel_arena, DPCPU_SIZE, 122254025Sjeff M_WAITOK | M_ZERO); 123239268Sgonzo temp_pagetable_va = (vm_offset_t)contigmalloc(L1_TABLE_SIZE, 124239268Sgonzo M_TEMP, 0, 0x0, 0xffffffff, L1_TABLE_SIZE, 0); 125266194Sian addr = arm_physmem_kernaddr; 126266194Sian addr_end = (vm_offset_t)&_end - KERNVIRTADDR + arm_physmem_kernaddr; 127239268Sgonzo addr_end &= ~L1_S_OFFSET; 128239268Sgonzo addr_end += L1_S_SIZE; 129239268Sgonzo bzero((void *)temp_pagetable_va, L1_TABLE_SIZE); 130266194Sian for (addr = arm_physmem_kernaddr; addr <= addr_end; addr += L1_S_SIZE) { 131239268Sgonzo ((int *)(temp_pagetable_va))[addr >> L1_S_SHIFT] = 132266203Sian L1_TYPE_S|L1_SHARED|L1_S_C|L1_S_B|L1_S_AP(AP_KRW)|L1_S_DOM(PMAP_DOMAIN_KERNEL)|addr; 133239268Sgonzo ((int *)(temp_pagetable_va))[(addr - 134266194Sian arm_physmem_kernaddr + KERNVIRTADDR) >> L1_S_SHIFT] = 135266203Sian L1_TYPE_S|L1_SHARED|L1_S_C|L1_S_B|L1_S_AP(AP_KRW)|L1_S_DOM(PMAP_DOMAIN_KERNEL)|addr; 136239268Sgonzo } 137250293Sgber 138250293Sgber#if defined(CPU_MV_PJ4B) 139250293Sgber /* Add ARMADAXP registers required for snoop filter initialization */ 140266046Sian ((int *)(temp_pagetable_va))[MV_BASE >> L1_S_SHIFT] = 141266046Sian L1_TYPE_S|L1_SHARED|L1_S_B|L1_S_AP(AP_KRW)|fdt_immr_pa; 142250293Sgber#endif 143250293Sgber 144239268Sgonzo temp_pagetable = (void*)(vtophys(temp_pagetable_va)); 145239268Sgonzo cpu_idcache_wbinv_all(); 146239268Sgonzo cpu_l2cache_wbinv_all(); 147239268Sgonzo 148239268Sgonzo /* Initialize boot code and start up processors */ 149239268Sgonzo platform_mp_start_ap(); 150239268Sgonzo 151239268Sgonzo /* Check if ap's started properly */ 152239268Sgonzo error = check_ap(); 153239268Sgonzo if (error) 154239268Sgonzo printf("WARNING: Some AP's failed to start\n"); 155239268Sgonzo else 156239268Sgonzo for (i = 1; i < mp_ncpus; i++) 157239268Sgonzo CPU_SET(i, &all_cpus); 158239268Sgonzo 159239268Sgonzo contigfree((void *)temp_pagetable_va, L1_TABLE_SIZE, M_TEMP); 160239268Sgonzo} 161239268Sgonzo 162239268Sgonzo/* Introduce rest of cores to the world */ 163239268Sgonzovoid 164239268Sgonzocpu_mp_announce(void) 165239268Sgonzo{ 166239268Sgonzo 167239268Sgonzo} 168239268Sgonzo 169239268Sgonzoextern vm_paddr_t pmap_pa; 170239268Sgonzovoid 171239268Sgonzoinit_secondary(int cpu) 172239268Sgonzo{ 173239268Sgonzo struct pcpu *pc; 174239268Sgonzo uint32_t loop_counter; 175239268Sgonzo int start = 0, end = 0; 176239268Sgonzo 177266203Sian cpu_idcache_inv_all(); 178266203Sian 179239268Sgonzo cpu_setup(NULL); 180239268Sgonzo setttb(pmap_pa); 181239268Sgonzo cpu_tlb_flushID(); 182239268Sgonzo 183239268Sgonzo pc = &__pcpu[cpu]; 184250294Sgber 185250294Sgber /* 186250294Sgber * pcpu_init() updates queue, so it should not be executed in parallel 187250294Sgber * on several cores 188250294Sgber */ 189250294Sgber while(mp_naps < (cpu - 1)) 190250294Sgber ; 191250294Sgber 192239268Sgonzo pcpu_init(pc, cpu, sizeof(struct pcpu)); 193239268Sgonzo dpcpu_init(dpcpu[cpu - 1], cpu); 194239268Sgonzo 195239268Sgonzo /* Provide stack pointers for other processor modes. */ 196239268Sgonzo set_stackptrs(cpu); 197239268Sgonzo 198239268Sgonzo /* Signal our startup to BSP */ 199239268Sgonzo atomic_add_rel_32(&mp_naps, 1); 200239268Sgonzo 201239268Sgonzo /* Spin until the BSP releases the APs */ 202239268Sgonzo while (!aps_ready) 203239268Sgonzo ; 204239268Sgonzo 205239268Sgonzo /* Initialize curthread */ 206239268Sgonzo KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread")); 207239268Sgonzo pc->pc_curthread = pc->pc_idlethread; 208239268Sgonzo pc->pc_curpcb = pc->pc_idlethread->td_pcb; 209266159Sian set_curthread(pc->pc_idlethread); 210254461Sandrew#ifdef VFP 211247339Scognet pc->pc_cpu = cpu; 212239268Sgonzo 213247339Scognet vfp_init(); 214247339Scognet#endif 215247339Scognet 216239268Sgonzo mtx_lock_spin(&ap_boot_mtx); 217239268Sgonzo 218239268Sgonzo atomic_add_rel_32(&smp_cpus, 1); 219239268Sgonzo 220239268Sgonzo if (smp_cpus == mp_ncpus) { 221239268Sgonzo /* enable IPI's, tlb shootdown, freezes etc */ 222239268Sgonzo atomic_store_rel_int(&smp_started, 1); 223239268Sgonzo } 224239268Sgonzo 225239268Sgonzo mtx_unlock_spin(&ap_boot_mtx); 226239268Sgonzo 227239268Sgonzo /* Enable ipi */ 228239268Sgonzo#ifdef IPI_IRQ_START 229239268Sgonzo start = IPI_IRQ_START; 230239268Sgonzo#ifdef IPI_IRQ_END 231239268Sgonzo end = IPI_IRQ_END; 232239268Sgonzo#else 233239268Sgonzo end = IPI_IRQ_START; 234239268Sgonzo#endif 235239268Sgonzo#endif 236239268Sgonzo 237239268Sgonzo for (int i = start; i <= end; i++) 238239268Sgonzo arm_unmask_irq(i); 239239268Sgonzo enable_interrupts(I32_bit); 240239268Sgonzo 241239268Sgonzo loop_counter = 0; 242239268Sgonzo while (smp_started == 0) { 243239268Sgonzo DELAY(100); 244239268Sgonzo loop_counter++; 245239268Sgonzo if (loop_counter == 1000) 246239268Sgonzo CTR0(KTR_SMP, "AP still wait for smp_started"); 247239268Sgonzo } 248239268Sgonzo /* Start per-CPU event timers. */ 249239268Sgonzo cpu_initclocks_ap(); 250239268Sgonzo 251239268Sgonzo CTR0(KTR_SMP, "go into scheduler"); 252239268Sgonzo platform_mp_init_secondary(); 253239268Sgonzo 254239268Sgonzo /* Enter the scheduler */ 255239268Sgonzo sched_throw(NULL); 256239268Sgonzo 257239268Sgonzo panic("scheduler returned us to %s", __func__); 258239268Sgonzo /* NOTREACHED */ 259239268Sgonzo} 260239268Sgonzo 261239268Sgonzostatic int 262239268Sgonzoipi_handler(void *arg) 263239268Sgonzo{ 264239268Sgonzo u_int cpu, ipi; 265239268Sgonzo 266239268Sgonzo cpu = PCPU_GET(cpuid); 267239268Sgonzo 268239268Sgonzo ipi = pic_ipi_get((int)arg); 269239268Sgonzo 270239268Sgonzo while ((ipi != 0x3ff)) { 271239268Sgonzo switch (ipi) { 272239268Sgonzo case IPI_RENDEZVOUS: 273239268Sgonzo CTR0(KTR_SMP, "IPI_RENDEZVOUS"); 274239268Sgonzo smp_rendezvous_action(); 275239268Sgonzo break; 276239268Sgonzo 277239268Sgonzo case IPI_AST: 278239268Sgonzo CTR0(KTR_SMP, "IPI_AST"); 279239268Sgonzo break; 280239268Sgonzo 281239268Sgonzo case IPI_STOP: 282239268Sgonzo /* 283239268Sgonzo * IPI_STOP_HARD is mapped to IPI_STOP so it is not 284239268Sgonzo * necessary to add it in the switch. 285239268Sgonzo */ 286239268Sgonzo CTR0(KTR_SMP, "IPI_STOP or IPI_STOP_HARD"); 287239268Sgonzo 288239268Sgonzo savectx(&stoppcbs[cpu]); 289239268Sgonzo 290266374Sian /* 291266374Sian * CPUs are stopped when entering the debugger and at 292266374Sian * system shutdown, both events which can precede a 293266374Sian * panic dump. For the dump to be correct, all caches 294266374Sian * must be flushed and invalidated, but on ARM there's 295266374Sian * no way to broadcast a wbinv_all to other cores. 296266374Sian * Instead, we have each core do the local wbinv_all as 297266374Sian * part of stopping the core. The core requesting the 298266374Sian * stop will do the l2 cache flush after all other cores 299266374Sian * have done their l1 flushes and stopped. 300266374Sian */ 301266374Sian cpu_idcache_wbinv_all(); 302266374Sian 303239268Sgonzo /* Indicate we are stopped */ 304239268Sgonzo CPU_SET_ATOMIC(cpu, &stopped_cpus); 305239268Sgonzo 306239268Sgonzo /* Wait for restart */ 307239268Sgonzo while (!CPU_ISSET(cpu, &started_cpus)) 308239268Sgonzo cpu_spinwait(); 309239268Sgonzo 310239268Sgonzo CPU_CLR_ATOMIC(cpu, &started_cpus); 311239268Sgonzo CPU_CLR_ATOMIC(cpu, &stopped_cpus); 312239268Sgonzo CTR0(KTR_SMP, "IPI_STOP (restart)"); 313239268Sgonzo break; 314239268Sgonzo case IPI_PREEMPT: 315239268Sgonzo CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__); 316239268Sgonzo sched_preempt(curthread); 317239268Sgonzo break; 318239268Sgonzo case IPI_HARDCLOCK: 319239268Sgonzo CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__); 320239268Sgonzo hardclockintr(); 321239268Sgonzo break; 322239268Sgonzo case IPI_TLB: 323239268Sgonzo CTR1(KTR_SMP, "%s: IPI_TLB", __func__); 324239268Sgonzo cpufuncs.cf_tlb_flushID(); 325239268Sgonzo break; 326239268Sgonzo default: 327239268Sgonzo panic("Unknown IPI 0x%0x on cpu %d", ipi, curcpu); 328239268Sgonzo } 329239268Sgonzo 330239268Sgonzo pic_ipi_clear(ipi); 331239268Sgonzo ipi = pic_ipi_get(-1); 332239268Sgonzo } 333239268Sgonzo 334239268Sgonzo return (FILTER_HANDLED); 335239268Sgonzo} 336239268Sgonzo 337239268Sgonzostatic void 338239268Sgonzorelease_aps(void *dummy __unused) 339239268Sgonzo{ 340239268Sgonzo uint32_t loop_counter; 341239268Sgonzo int start = 0, end = 0; 342239268Sgonzo 343239268Sgonzo if (mp_ncpus == 1) 344239268Sgonzo return; 345239268Sgonzo#ifdef IPI_IRQ_START 346239268Sgonzo start = IPI_IRQ_START; 347239268Sgonzo#ifdef IPI_IRQ_END 348239268Sgonzo end = IPI_IRQ_END; 349239268Sgonzo#else 350239268Sgonzo end = IPI_IRQ_START; 351239268Sgonzo#endif 352239268Sgonzo#endif 353239268Sgonzo 354239268Sgonzo for (int i = start; i <= end; i++) { 355239268Sgonzo /* 356239268Sgonzo * IPI handler 357239268Sgonzo */ 358239268Sgonzo /* 359239268Sgonzo * Use 0xdeadbeef as the argument value for irq 0, 360239268Sgonzo * if we used 0, the intr code will give the trap frame 361239268Sgonzo * pointer instead. 362239268Sgonzo */ 363239268Sgonzo arm_setup_irqhandler("ipi", ipi_handler, NULL, (void *)i, i, 364239268Sgonzo INTR_TYPE_MISC | INTR_EXCL, NULL); 365239268Sgonzo 366239268Sgonzo /* Enable ipi */ 367239268Sgonzo arm_unmask_irq(i); 368239268Sgonzo } 369239268Sgonzo atomic_store_rel_int(&aps_ready, 1); 370239268Sgonzo 371239268Sgonzo printf("Release APs\n"); 372239268Sgonzo 373239268Sgonzo for (loop_counter = 0; loop_counter < 2000; loop_counter++) { 374239268Sgonzo if (smp_started) 375239268Sgonzo return; 376239268Sgonzo DELAY(1000); 377239268Sgonzo } 378239268Sgonzo printf("AP's not started\n"); 379239268Sgonzo} 380239268Sgonzo 381239268SgonzoSYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); 382239268Sgonzo 383239268Sgonzostruct cpu_group * 384239268Sgonzocpu_topo(void) 385239268Sgonzo{ 386239268Sgonzo 387266207Sian return (smp_topo_1level(CG_SHARE_L2, mp_ncpus, 0)); 388239268Sgonzo} 389239268Sgonzo 390239268Sgonzovoid 391239268Sgonzocpu_mp_setmaxid(void) 392239268Sgonzo{ 393239268Sgonzo 394239268Sgonzo platform_mp_setmaxid(); 395239268Sgonzo} 396239268Sgonzo 397239268Sgonzo/* Sending IPI */ 398239268Sgonzovoid 399239268Sgonzoipi_all_but_self(u_int ipi) 400239268Sgonzo{ 401239268Sgonzo cpuset_t other_cpus; 402239268Sgonzo 403239268Sgonzo other_cpus = all_cpus; 404239268Sgonzo CPU_CLR(PCPU_GET(cpuid), &other_cpus); 405239268Sgonzo CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); 406239268Sgonzo platform_ipi_send(other_cpus, ipi); 407239268Sgonzo} 408239268Sgonzo 409239268Sgonzovoid 410239268Sgonzoipi_cpu(int cpu, u_int ipi) 411239268Sgonzo{ 412239268Sgonzo cpuset_t cpus; 413239268Sgonzo 414239268Sgonzo CPU_ZERO(&cpus); 415239268Sgonzo CPU_SET(cpu, &cpus); 416239268Sgonzo 417239268Sgonzo CTR3(KTR_SMP, "%s: cpu: %d, ipi: %x", __func__, cpu, ipi); 418239268Sgonzo platform_ipi_send(cpus, ipi); 419239268Sgonzo} 420239268Sgonzo 421239268Sgonzovoid 422239268Sgonzoipi_selected(cpuset_t cpus, u_int ipi) 423239268Sgonzo{ 424239268Sgonzo 425239268Sgonzo CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); 426239268Sgonzo platform_ipi_send(cpus, ipi); 427239268Sgonzo} 428239268Sgonzo 429239268Sgonzovoid 430239268Sgonzotlb_broadcast(int ipi) 431239268Sgonzo{ 432239268Sgonzo 433239268Sgonzo if (smp_started) 434239268Sgonzo ipi_all_but_self(ipi); 435239268Sgonzo} 436