mp_machdep.c revision 265606 
1/*-
2 * Copyright (c) 2011 Semihalf.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26#include <sys/cdefs.h>
27__FBSDID("$FreeBSD: stable/10/sys/arm/arm/mp_machdep.c 265606 2014-05-07 20:28:27Z scottl $");
28#include <sys/param.h>
29#include <sys/systm.h>
30#include <sys/bus.h>
31#include <sys/kernel.h>
32#include <sys/lock.h>
33#include <sys/mutex.h>
34#include <sys/proc.h>
35#include <sys/pcpu.h>
36#include <sys/sched.h>
37#include <sys/smp.h>
38#include <sys/ktr.h>
39#include <sys/malloc.h>
40
41#include <vm/vm.h>
42#include <vm/vm_extern.h>
43#include <vm/vm_kern.h>
44#include <vm/pmap.h>
45
46#include <machine/cpu.h>
47#include <machine/smp.h>
48#include <machine/pcb.h>
49#include <machine/pte.h>
50#include <machine/intr.h>
51#include <machine/vmparam.h>
52#ifdef VFP
53#include <machine/vfp.h>
54#endif
55
56#include "opt_smp.h"
57
58void *temp_pagetable;
59extern struct pcpu __pcpu[];
60/* used to hold the AP's until we are ready to release them */
61struct mtx ap_boot_mtx;
62struct pcb stoppcbs[MAXCPU];
63
64/* # of Applications processors */
65volatile int mp_naps;
66
67/* Set to 1 once we're ready to let the APs out of the pen. */
68volatile int aps_ready = 0;
69
70static int ipi_handler(void *arg);
71void set_stackptrs(int cpu);
72
73/* Temporary variables for init_secondary()  */
74void *dpcpu[MAXCPU - 1];
75
76/* Determine if we running MP machine */
77int
78cpu_mp_probe(void)
79{
80	CPU_SETOF(0, &all_cpus);
81
82	return (platform_mp_probe());
83}
84
85/* Start Application Processor via platform specific function */
86static int
87check_ap(void)
88{
89	uint32_t ms;
90
91	for (ms = 0; ms < 2000; ++ms) {
92		if ((mp_naps + 1) == mp_ncpus)
93			return (0);		/* success */
94		else
95			DELAY(1000);
96	}
97
98	return (-2);
99}
100
101extern unsigned char _end[];
102
103/* Initialize and fire up non-boot processors */
104void
105cpu_mp_start(void)
106{
107	int error, i;
108	vm_offset_t temp_pagetable_va;
109	vm_paddr_t addr, addr_end;
110
111	mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN);
112
113	/* Reserve memory for application processors */
114	for(i = 0; i < (mp_ncpus - 1); i++)
115		dpcpu[i] = (void *)kmem_malloc(kernel_arena, DPCPU_SIZE,
116		    M_WAITOK | M_ZERO);
117	temp_pagetable_va = (vm_offset_t)contigmalloc(L1_TABLE_SIZE,
118	    M_TEMP, 0, 0x0, 0xffffffff, L1_TABLE_SIZE, 0);
119	addr = KERNPHYSADDR;
120	addr_end = (vm_offset_t)&_end - KERNVIRTADDR + KERNPHYSADDR;
121	addr_end &= ~L1_S_OFFSET;
122	addr_end += L1_S_SIZE;
123	bzero((void *)temp_pagetable_va,  L1_TABLE_SIZE);
124	for (addr = KERNPHYSADDR; addr <= addr_end; addr += L1_S_SIZE) {
125		((int *)(temp_pagetable_va))[addr >> L1_S_SHIFT] =
126		    L1_TYPE_S|L1_SHARED|L1_S_C|L1_S_AP(AP_KRW)|L1_S_DOM(PMAP_DOMAIN_KERNEL)|addr;
127		((int *)(temp_pagetable_va))[(addr -
128			KERNPHYSADDR + KERNVIRTADDR) >> L1_S_SHIFT] =
129		    L1_TYPE_S|L1_SHARED|L1_S_C|L1_S_AP(AP_KRW)|L1_S_DOM(PMAP_DOMAIN_KERNEL)|addr;
130	}
131
132#if defined(CPU_MV_PJ4B)
133	/* Add ARMADAXP registers required for snoop filter initialization */
134	((int *)(temp_pagetable_va))[0xf1000000 >> L1_S_SHIFT] =
135	    L1_TYPE_S|L1_SHARED|L1_S_B|L1_S_AP(AP_KRW)|0xd0000000;
136#endif
137
138	temp_pagetable = (void*)(vtophys(temp_pagetable_va));
139	cpu_idcache_wbinv_all();
140	cpu_l2cache_wbinv_all();
141
142	/* Initialize boot code and start up processors */
143	platform_mp_start_ap();
144
145	/*  Check if ap's started properly */
146	error = check_ap();
147	if (error)
148		printf("WARNING: Some AP's failed to start\n");
149	else
150		for (i = 1; i < mp_ncpus; i++)
151			CPU_SET(i, &all_cpus);
152
153	contigfree((void *)temp_pagetable_va, L1_TABLE_SIZE, M_TEMP);
154}
155
156/* Introduce rest of cores to the world */
157void
158cpu_mp_announce(void)
159{
160
161}
162
163extern vm_paddr_t pmap_pa;
164void
165init_secondary(int cpu)
166{
167	struct pcpu *pc;
168	uint32_t loop_counter;
169	int start = 0, end = 0;
170
171	cpu_setup(NULL);
172	setttb(pmap_pa);
173	cpu_tlb_flushID();
174
175	pc = &__pcpu[cpu];
176	set_pcpu(pc);
177
178	/*
179	 * pcpu_init() updates queue, so it should not be executed in parallel
180	 * on several cores
181	 */
182	while(mp_naps < (cpu - 1))
183		;
184
185	pcpu_init(pc, cpu, sizeof(struct pcpu));
186	dpcpu_init(dpcpu[cpu - 1], cpu);
187
188	/* Provide stack pointers for other processor modes. */
189	set_stackptrs(cpu);
190
191	/* Signal our startup to BSP */
192	atomic_add_rel_32(&mp_naps, 1);
193
194	/* Spin until the BSP releases the APs */
195	while (!aps_ready)
196		;
197
198	/* Initialize curthread */
199	KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread"));
200	pc->pc_curthread = pc->pc_idlethread;
201	pc->pc_curpcb = pc->pc_idlethread->td_pcb;
202#ifdef VFP
203	pc->pc_cpu = cpu;
204
205	vfp_init();
206#endif
207
208	mtx_lock_spin(&ap_boot_mtx);
209
210	atomic_add_rel_32(&smp_cpus, 1);
211
212	if (smp_cpus == mp_ncpus) {
213		/* enable IPI's, tlb shootdown, freezes etc */
214		atomic_store_rel_int(&smp_started, 1);
215	}
216
217	mtx_unlock_spin(&ap_boot_mtx);
218
219	/* Enable ipi */
220#ifdef IPI_IRQ_START
221	start = IPI_IRQ_START;
222#ifdef IPI_IRQ_END
223  	end = IPI_IRQ_END;
224#else
225	end = IPI_IRQ_START;
226#endif
227#endif
228
229	for (int i = start; i <= end; i++)
230		arm_unmask_irq(i);
231	enable_interrupts(I32_bit);
232
233	loop_counter = 0;
234	while (smp_started == 0) {
235		DELAY(100);
236		loop_counter++;
237		if (loop_counter == 1000)
238			CTR0(KTR_SMP, "AP still wait for smp_started");
239	}
240	/* Start per-CPU event timers. */
241	cpu_initclocks_ap();
242
243	CTR0(KTR_SMP, "go into scheduler");
244	platform_mp_init_secondary();
245
246	/* Enter the scheduler */
247	sched_throw(NULL);
248
249	panic("scheduler returned us to %s", __func__);
250	/* NOTREACHED */
251}
252
253static int
254ipi_handler(void *arg)
255{
256	u_int	cpu, ipi;
257
258	cpu = PCPU_GET(cpuid);
259
260	ipi = pic_ipi_get((int)arg);
261
262	while ((ipi != 0x3ff)) {
263		switch (ipi) {
264		case IPI_RENDEZVOUS:
265			CTR0(KTR_SMP, "IPI_RENDEZVOUS");
266			smp_rendezvous_action();
267			break;
268
269		case IPI_AST:
270			CTR0(KTR_SMP, "IPI_AST");
271			break;
272
273		case IPI_STOP:
274		case IPI_STOP_HARD:
275			/*
276			 * IPI_STOP_HARD is mapped to IPI_STOP so it is not
277			 * necessary to add it in the switch.
278			 */
279			CTR0(KTR_SMP, "IPI_STOP or IPI_STOP_HARD");
280
281			savectx(&stoppcbs[cpu]);
282
283			/* Indicate we are stopped */
284			CPU_SET_ATOMIC(cpu, &stopped_cpus);
285
286			/* Wait for restart */
287			while (!CPU_ISSET(cpu, &started_cpus))
288				cpu_spinwait();
289
290			CPU_CLR_ATOMIC(cpu, &started_cpus);
291			CPU_CLR_ATOMIC(cpu, &stopped_cpus);
292			CTR0(KTR_SMP, "IPI_STOP (restart)");
293			break;
294		case IPI_PREEMPT:
295			CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__);
296			sched_preempt(curthread);
297			break;
298		case IPI_HARDCLOCK:
299			CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__);
300			hardclockintr();
301			break;
302		case IPI_TLB:
303			CTR1(KTR_SMP, "%s: IPI_TLB", __func__);
304			cpufuncs.cf_tlb_flushID();
305			break;
306		default:
307			panic("Unknown IPI 0x%0x on cpu %d", ipi, curcpu);
308		}
309
310		pic_ipi_clear(ipi);
311		ipi = pic_ipi_get(-1);
312	}
313
314	return (FILTER_HANDLED);
315}
316
317static void
318release_aps(void *dummy __unused)
319{
320	uint32_t loop_counter;
321	int start = 0, end = 0;
322
323	if (mp_ncpus == 1)
324		return;
325#ifdef IPI_IRQ_START
326	start = IPI_IRQ_START;
327#ifdef IPI_IRQ_END
328	end = IPI_IRQ_END;
329#else
330	end = IPI_IRQ_START;
331#endif
332#endif
333
334	for (int i = start; i <= end; i++) {
335		/*
336		 * IPI handler
337		 */
338		/*
339		 * Use 0xdeadbeef as the argument value for irq 0,
340		 * if we used 0, the intr code will give the trap frame
341		 * pointer instead.
342		 */
343		arm_setup_irqhandler("ipi", ipi_handler, NULL, (void *)i, i,
344		    INTR_TYPE_MISC | INTR_EXCL, NULL);
345
346		/* Enable ipi */
347		arm_unmask_irq(i);
348	}
349	atomic_store_rel_int(&aps_ready, 1);
350
351	printf("Release APs\n");
352
353	for (loop_counter = 0; loop_counter < 2000; loop_counter++) {
354		if (smp_started)
355			return;
356		DELAY(1000);
357	}
358	printf("AP's not started\n");
359}
360
361SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL);
362
363struct cpu_group *
364cpu_topo(void)
365{
366
367	return (smp_topo_1level(CG_SHARE_L2, 1, 0));
368}
369
370void
371cpu_mp_setmaxid(void)
372{
373
374	platform_mp_setmaxid();
375}
376
377/* Sending IPI */
378void
379ipi_all_but_self(u_int ipi)
380{
381	cpuset_t other_cpus;
382
383	other_cpus = all_cpus;
384	CPU_CLR(PCPU_GET(cpuid), &other_cpus);
385	CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
386	platform_ipi_send(other_cpus, ipi);
387}
388
389void
390ipi_cpu(int cpu, u_int ipi)
391{
392	cpuset_t cpus;
393
394	CPU_ZERO(&cpus);
395	CPU_SET(cpu, &cpus);
396
397	CTR3(KTR_SMP, "%s: cpu: %d, ipi: %x", __func__, cpu, ipi);
398	platform_ipi_send(cpus, ipi);
399}
400
401void
402ipi_selected(cpuset_t cpus, u_int ipi)
403{
404
405	CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
406	platform_ipi_send(cpus, ipi);
407}
408
409void
410tlb_broadcast(int ipi)
411{
412
413	if (smp_started)
414		ipi_all_but_self(ipi);
415}
416