180708Sjake/*- 280708Sjake * Copyright (c) 2001 Jake Burkholder. 3223126Smarius * Copyright (c) 2007 - 2011 Marius Strobl <marius@FreeBSD.org> 480708Sjake * All rights reserved. 580708Sjake * 680708Sjake * Redistribution and use in source and binary forms, with or without 780708Sjake * modification, are permitted provided that the following conditions 880708Sjake * are met: 980708Sjake * 1. Redistributions of source code must retain the above copyright 1080708Sjake * notice, this list of conditions and the following disclaimer. 1180708Sjake * 2. Redistributions in binary form must reproduce the above copyright 1280708Sjake * notice, this list of conditions and the following disclaimer in the 1380708Sjake * documentation and/or other materials provided with the distribution. 1480708Sjake * 1581334Sobrien * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1680708Sjake * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1780708Sjake * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1881334Sobrien * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1980708Sjake * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2080708Sjake * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2180708Sjake * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2280708Sjake * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2380708Sjake * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2480708Sjake * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2580708Sjake * SUCH DAMAGE. 2680708Sjake * 2780708Sjake * $FreeBSD$ 2880708Sjake */ 2980708Sjake 3080708Sjake#ifndef _MACHINE_SMP_H_ 3180708Sjake#define _MACHINE_SMP_H_ 3280708Sjake 33183142Smarius#ifdef SMP 34183142Smarius 35182730Smarius#define CPU_TICKSYNC 1 36182730Smarius#define CPU_STICKSYNC 2 37182730Smarius#define CPU_INIT 3 38182730Smarius#define CPU_BOOTSTRAP 4 3989051Sjake 4089051Sjake#ifndef LOCORE 4189051Sjake 42222813Sattilio#include <sys/cpuset.h> 43209695Smarius#include <sys/proc.h> 44209695Smarius#include <sys/sched.h> 45245850Smarius#include <sys/smp.h> 46209695Smarius 4791157Sjake#include <machine/intr_machdep.h> 48169730Skan#include <machine/pcb.h> 4991617Sjake#include <machine/tte.h> 5091157Sjake 51170846Smarius#define IDR_BUSY 0x0000000000000001ULL 52170846Smarius#define IDR_NACK 0x0000000000000002ULL 53170846Smarius#define IDR_CHEETAH_ALL_BUSY 0x5555555555555555ULL 54170846Smarius#define IDR_CHEETAH_ALL_NACK (~IDR_CHEETAH_ALL_BUSY) 55170846Smarius#define IDR_CHEETAH_MAX_BN_PAIRS 32 56170846Smarius#define IDR_JALAPENO_MAX_BN_PAIRS 4 5789051Sjake 58169796Smarius#define IDC_ITID_SHIFT 14 59170846Smarius#define IDC_BN_SHIFT 24 60169796Smarius 6189051Sjake#define IPI_AST PIL_AST 6289051Sjake#define IPI_RENDEZVOUS PIL_RENDEZVOUS 63178048Smarius#define IPI_PREEMPT PIL_PREEMPT 64210601Smav#define IPI_HARDCLOCK PIL_HARDCLOCK 6589051Sjake#define IPI_STOP PIL_STOP 66196196Sattilio#define IPI_STOP_HARD PIL_STOP 6789051Sjake 68135943Skensmith#define IPI_RETRIES 5000 6989051Sjake 7089051Sjakestruct cpu_start_args { 7191783Sjake u_int csa_count; 7289051Sjake u_int csa_mid; 7389051Sjake u_int csa_state; 7491783Sjake vm_offset_t csa_pcpu; 7591617Sjake u_long csa_tick; 76182730Smarius u_long csa_stick; 7791617Sjake u_long csa_ver; 7891617Sjake struct tte csa_ttes[PCPU_PAGES]; 7989051Sjake}; 8089051Sjake 8197001Sjakestruct ipi_cache_args { 82222813Sattilio cpuset_t ica_mask; 83113238Sjake vm_paddr_t ica_pa; 8497001Sjake}; 8597001Sjake 86211071Smariusstruct ipi_rd_args { 87222813Sattilio cpuset_t ira_mask; 88211071Smarius register_t *ira_val; 89211071Smarius}; 90211071Smarius 9189051Sjakestruct ipi_tlb_args { 92222813Sattilio cpuset_t ita_mask; 9391783Sjake struct pmap *ita_pmap; 9489051Sjake u_long ita_start; 9589051Sjake u_long ita_end; 9689051Sjake}; 9789051Sjake#define ita_va ita_start 9889051Sjake 9989051Sjakestruct pcpu; 10089051Sjake 101152022Sjhbextern struct pcb stoppcbs[]; 102152022Sjhb 10389051Sjakevoid cpu_mp_bootstrap(struct pcpu *pc); 10492199Sjakevoid cpu_mp_shutdown(void); 10589051Sjake 106222813Sattiliotypedef void cpu_ipi_selected_t(cpuset_t, u_long, u_long, u_long); 107170846Smariusextern cpu_ipi_selected_t *cpu_ipi_selected; 108211050Smariustypedef void cpu_ipi_single_t(u_int, u_long, u_long, u_long); 109211050Smariusextern cpu_ipi_single_t *cpu_ipi_single; 11089051Sjake 111204152Smariusvoid mp_init(u_int cpu_impl); 11291617Sjake 113108187Sjakeextern struct ipi_cache_args ipi_cache_args; 114211071Smariusextern struct ipi_rd_args ipi_rd_args; 115108187Sjakeextern struct ipi_tlb_args ipi_tlb_args; 11689051Sjake 11791617Sjakeextern char *mp_tramp_code; 11891617Sjakeextern u_long mp_tramp_code_len; 11991617Sjakeextern u_long mp_tramp_tlb_slots; 12091617Sjakeextern u_long mp_tramp_func; 12191617Sjake 12291617Sjakeextern void mp_startup(void); 12391617Sjake 124112399Sjakeextern char tl_ipi_cheetah_dcache_page_inval[]; 125112399Sjakeextern char tl_ipi_spitfire_dcache_page_inval[]; 126112399Sjakeextern char tl_ipi_spitfire_icache_page_inval[]; 127112399Sjake 12889051Sjakeextern char tl_ipi_level[]; 129211071Smarius 130211071Smariusextern char tl_ipi_stick_rd[]; 131211071Smariusextern char tl_ipi_tick_rd[]; 132211071Smarius 13389051Sjakeextern char tl_ipi_tlb_context_demap[]; 13489051Sjakeextern char tl_ipi_tlb_page_demap[]; 13589051Sjakeextern char tl_ipi_tlb_range_demap[]; 13689051Sjake 137183142Smariusstatic __inline void 138183142Smariusipi_all_but_self(u_int ipi) 139183142Smarius{ 140223126Smarius cpuset_t cpus; 14189051Sjake 142223126Smarius cpus = all_cpus; 143223126Smarius CPU_CLR(PCPU_GET(cpuid), &cpus); 144223126Smarius cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_level, ipi); 145183142Smarius} 146183142Smarius 147183142Smariusstatic __inline void 148222813Sattilioipi_selected(cpuset_t cpus, u_int ipi) 149183142Smarius{ 150183142Smarius 151183142Smarius cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_level, ipi); 152183142Smarius} 153183142Smarius 154210939Sjhbstatic __inline void 155210939Sjhbipi_cpu(int cpu, u_int ipi) 156210939Sjhb{ 157210939Sjhb 158211050Smarius cpu_ipi_single(cpu, 0, (u_long)tl_ipi_level, ipi); 159210939Sjhb} 160210939Sjhb 161108187Sjake#if defined(_MACHINE_PMAP_H_) && defined(_SYS_MUTEX_H_) 162108187Sjake 16397001Sjakestatic __inline void * 164113238Sjakeipi_dcache_page_inval(void *func, vm_paddr_t pa) 16597001Sjake{ 16697001Sjake struct ipi_cache_args *ica; 16797001Sjake 16897001Sjake if (smp_cpus == 1) 16997001Sjake return (NULL); 170209695Smarius sched_pin(); 17197001Sjake ica = &ipi_cache_args; 172239864Smarius mtx_lock_spin(&smp_ipi_mtx); 17397001Sjake ica->ica_mask = all_cpus; 174223126Smarius CPU_CLR(PCPU_GET(cpuid), &ica->ica_mask); 17597001Sjake ica->ica_pa = pa; 176223126Smarius cpu_ipi_selected(ica->ica_mask, 0, (u_long)func, (u_long)ica); 17797001Sjake return (&ica->ica_mask); 17897001Sjake} 17997001Sjake 18097001Sjakestatic __inline void * 181113238Sjakeipi_icache_page_inval(void *func, vm_paddr_t pa) 18297001Sjake{ 18397001Sjake struct ipi_cache_args *ica; 18497001Sjake 18597001Sjake if (smp_cpus == 1) 18697001Sjake return (NULL); 187209695Smarius sched_pin(); 18897001Sjake ica = &ipi_cache_args; 189239864Smarius mtx_lock_spin(&smp_ipi_mtx); 19097001Sjake ica->ica_mask = all_cpus; 191223126Smarius CPU_CLR(PCPU_GET(cpuid), &ica->ica_mask); 19297001Sjake ica->ica_pa = pa; 193223126Smarius cpu_ipi_selected(ica->ica_mask, 0, (u_long)func, (u_long)ica); 19497001Sjake return (&ica->ica_mask); 19597001Sjake} 19697001Sjake 19789051Sjakestatic __inline void * 198211071Smariusipi_rd(u_int cpu, void *func, u_long *val) 199211071Smarius{ 200211071Smarius struct ipi_rd_args *ira; 201211071Smarius 202211071Smarius if (smp_cpus == 1) 203211071Smarius return (NULL); 204211071Smarius sched_pin(); 205211071Smarius ira = &ipi_rd_args; 206245850Smarius mtx_lock_spin(&smp_ipi_mtx); 207223126Smarius CPU_SETOF(cpu, &ira->ira_mask); 208211071Smarius ira->ira_val = val; 209211071Smarius cpu_ipi_single(cpu, 0, (u_long)func, (u_long)ira); 210211071Smarius return (&ira->ira_mask); 211211071Smarius} 212211071Smarius 213211071Smariusstatic __inline void * 21491783Sjakeipi_tlb_context_demap(struct pmap *pm) 21589051Sjake{ 21689051Sjake struct ipi_tlb_args *ita; 217222813Sattilio cpuset_t cpus; 21889051Sjake 21991783Sjake if (smp_cpus == 1) 22089051Sjake return (NULL); 221209695Smarius sched_pin(); 222222813Sattilio cpus = pm->pm_active; 223223126Smarius CPU_AND(&cpus, &all_cpus); 224223126Smarius CPU_CLR(PCPU_GET(cpuid), &cpus); 225222813Sattilio if (CPU_EMPTY(&cpus)) { 226209695Smarius sched_unpin(); 22791783Sjake return (NULL); 228209695Smarius } 22989051Sjake ita = &ipi_tlb_args; 230239864Smarius mtx_lock_spin(&smp_ipi_mtx); 231222813Sattilio ita->ita_mask = cpus; 23291783Sjake ita->ita_pmap = pm; 23391783Sjake cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_tlb_context_demap, 23491783Sjake (u_long)ita); 23592199Sjake return (&ita->ita_mask); 23689051Sjake} 23789051Sjake 23889051Sjakestatic __inline void * 239100718Sjakeipi_tlb_page_demap(struct pmap *pm, vm_offset_t va) 24089051Sjake{ 24189051Sjake struct ipi_tlb_args *ita; 242222813Sattilio cpuset_t cpus; 24389051Sjake 24491783Sjake if (smp_cpus == 1) 24589051Sjake return (NULL); 246209695Smarius sched_pin(); 247222813Sattilio cpus = pm->pm_active; 248223126Smarius CPU_AND(&cpus, &all_cpus); 249223126Smarius CPU_CLR(PCPU_GET(cpuid), &cpus); 250222813Sattilio if (CPU_EMPTY(&cpus)) { 251209695Smarius sched_unpin(); 25291783Sjake return (NULL); 253209695Smarius } 25489051Sjake ita = &ipi_tlb_args; 255239864Smarius mtx_lock_spin(&smp_ipi_mtx); 256222813Sattilio ita->ita_mask = cpus; 25791783Sjake ita->ita_pmap = pm; 25889051Sjake ita->ita_va = va; 25991783Sjake cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_tlb_page_demap, (u_long)ita); 26092199Sjake return (&ita->ita_mask); 26189051Sjake} 26289051Sjake 26389051Sjakestatic __inline void * 26491783Sjakeipi_tlb_range_demap(struct pmap *pm, vm_offset_t start, vm_offset_t end) 26589051Sjake{ 26689051Sjake struct ipi_tlb_args *ita; 267222813Sattilio cpuset_t cpus; 26889051Sjake 26991783Sjake if (smp_cpus == 1) 27089051Sjake return (NULL); 271209695Smarius sched_pin(); 272222813Sattilio cpus = pm->pm_active; 273223126Smarius CPU_AND(&cpus, &all_cpus); 274223126Smarius CPU_CLR(PCPU_GET(cpuid), &cpus); 275222813Sattilio if (CPU_EMPTY(&cpus)) { 276209695Smarius sched_unpin(); 27791783Sjake return (NULL); 278209695Smarius } 27989051Sjake ita = &ipi_tlb_args; 280239864Smarius mtx_lock_spin(&smp_ipi_mtx); 281222813Sattilio ita->ita_mask = cpus; 28291783Sjake ita->ita_pmap = pm; 28389051Sjake ita->ita_start = start; 28489051Sjake ita->ita_end = end; 285211050Smarius cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_tlb_range_demap, 286211050Smarius (u_long)ita); 28792199Sjake return (&ita->ita_mask); 28889051Sjake} 28989051Sjake 29089051Sjakestatic __inline void 29189051Sjakeipi_wait(void *cookie) 29289051Sjake{ 293222813Sattilio volatile cpuset_t *mask; 29489051Sjake 29592199Sjake if ((mask = cookie) != NULL) { 296222813Sattilio while (!CPU_EMPTY(mask)) 29791617Sjake ; 298239864Smarius mtx_unlock_spin(&smp_ipi_mtx); 299209695Smarius sched_unpin(); 30089051Sjake } 30189051Sjake} 30289051Sjake 303108187Sjake#endif /* _MACHINE_PMAP_H_ && _SYS_MUTEX_H_ */ 30491783Sjake 305183142Smarius#endif /* !LOCORE */ 306183142Smarius 30789051Sjake#else 30889051Sjake 309183142Smarius#ifndef LOCORE 310183142Smarius 31189051Sjakestatic __inline void * 312209695Smariusipi_dcache_page_inval(void *func __unused, vm_paddr_t pa __unused) 31397001Sjake{ 314169796Smarius 31597001Sjake return (NULL); 31697001Sjake} 31797001Sjake 31897001Sjakestatic __inline void * 319209695Smariusipi_icache_page_inval(void *func __unused, vm_paddr_t pa __unused) 32097001Sjake{ 321169796Smarius 32297001Sjake return (NULL); 32397001Sjake} 32497001Sjake 32597001Sjakestatic __inline void * 326211071Smariusipi_rd(u_int cpu __unused, void *func __unused, u_long *val __unused) 327211071Smarius{ 328211071Smarius 329211071Smarius return (NULL); 330211071Smarius} 331211071Smarius 332211071Smariusstatic __inline void * 333209695Smariusipi_tlb_context_demap(struct pmap *pm __unused) 33489051Sjake{ 335169796Smarius 33689051Sjake return (NULL); 33789051Sjake} 33889051Sjake 33989051Sjakestatic __inline void * 340209695Smariusipi_tlb_page_demap(struct pmap *pm __unused, vm_offset_t va __unused) 34189051Sjake{ 342169796Smarius 34389051Sjake return (NULL); 34489051Sjake} 34589051Sjake 34689051Sjakestatic __inline void * 347209695Smariusipi_tlb_range_demap(struct pmap *pm __unused, vm_offset_t start __unused, 348209695Smarius __unused vm_offset_t end) 34989051Sjake{ 350169796Smarius 35189051Sjake return (NULL); 35289051Sjake} 35389051Sjake 35489051Sjakestatic __inline void 355239864Smariusipi_wait(void *cookie __unused) 35689051Sjake{ 357169796Smarius 35889051Sjake} 35989051Sjake 360183142Smariusstatic __inline void 361183142Smariustl_ipi_cheetah_dcache_page_inval(void) 362183142Smarius{ 36389051Sjake 364183142Smarius} 365183142Smarius 366183142Smariusstatic __inline void 367183142Smariustl_ipi_spitfire_dcache_page_inval(void) 368183142Smarius{ 369183142Smarius 370183142Smarius} 371183142Smarius 372183142Smariusstatic __inline void 373183142Smariustl_ipi_spitfire_icache_page_inval(void) 374183142Smarius{ 375183142Smarius 376183142Smarius} 377183142Smarius 37889051Sjake#endif /* !LOCORE */ 37989051Sjake 380183142Smarius#endif /* SMP */ 381183142Smarius 38280708Sjake#endif /* !_MACHINE_SMP_H_ */ 383