1/* 2 * Copyright 2002-2005, Axel D��rfler, axeld@pinc-software.de. 3 * Distributed under the terms of the MIT License. 4 * 5 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. 6 * Distributed under the terms of the NewOS License. 7 */ 8#ifndef KERNEL_SMP_H 9#define KERNEL_SMP_H 10 11 12#include <arch/atomic.h> 13#include <boot/kernel_args.h> 14#include <kernel.h> 15 16#include <KernelExport.h> 17 18#include <string.h> 19 20 21struct kernel_args; 22 23 24// intercpu messages 25enum { 26 SMP_MSG_INVALIDATE_PAGE_RANGE = 0, 27 SMP_MSG_INVALIDATE_PAGE_LIST, 28 SMP_MSG_USER_INVALIDATE_PAGES, 29 SMP_MSG_GLOBAL_INVALIDATE_PAGES, 30 SMP_MSG_CPU_HALT, 31 SMP_MSG_CALL_FUNCTION, 32 SMP_MSG_RESCHEDULE 33}; 34 35enum { 36 SMP_MSG_FLAG_ASYNC = 0x0, 37 SMP_MSG_FLAG_SYNC = 0x1, 38 SMP_MSG_FLAG_FREE_ARG = 0x2, 39}; 40 41typedef void (*smp_call_func)(addr_t data1, int32 currentCPU, addr_t data2, addr_t data3); 42 43class CPUSet { 44public: 45 inline CPUSet(); 46 47 inline void ClearAll(); 48 inline void SetAll(); 49 50 inline void SetBit(int32 cpu); 51 inline void ClearBit(int32 cpu); 52 53 inline void SetBitAtomic(int32 cpu); 54 inline void ClearBitAtomic(int32 cpu); 55 56 inline bool GetBit(int32 cpu) const; 57 58 inline bool IsEmpty() const; 59 60private: 61 static const int kArraySize = ROUNDUP(SMP_MAX_CPUS, 32) / 32; 62 63 uint32 fBitmap[kArraySize]; 64}; 65 66 67#ifdef __cplusplus 68extern "C" { 69#endif 70 71bool try_acquire_spinlock(spinlock* lock); 72 73status_t smp_init(struct kernel_args *args); 74status_t smp_per_cpu_init(struct kernel_args *args, int32 cpu); 75status_t smp_init_post_generic_syscalls(void); 76bool smp_trap_non_boot_cpus(int32 cpu, uint32* rendezVous); 77void smp_wake_up_non_boot_cpus(void); 78void smp_cpu_rendezvous(uint32* var); 79void smp_send_ici(int32 targetCPU, int32 message, addr_t data, addr_t data2, addr_t data3, 80 void *data_ptr, uint32 flags); 81void smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data, 82 addr_t data2, addr_t data3, void *data_ptr, uint32 flags); 83void smp_send_broadcast_ici(int32 message, addr_t data, addr_t data2, addr_t data3, 84 void *data_ptr, uint32 flags); 85void smp_send_broadcast_ici_interrupts_disabled(int32 currentCPU, int32 message, 86 addr_t data, addr_t data2, addr_t data3, void *data_ptr, uint32 flags); 87 88int32 smp_get_num_cpus(void); 89void smp_set_num_cpus(int32 numCPUs); 90int32 smp_get_current_cpu(void); 91 92int smp_intercpu_int_handler(int32 cpu); 93 94void call_single_cpu(uint32 targetCPU, void (*func)(void*, int), void* cookie); 95void call_single_cpu_sync(uint32 targetCPU, void (*func)(void*, int), 96 void* cookie); 97 98 99#ifdef __cplusplus 100} 101#endif 102 103 104inline 105CPUSet::CPUSet() 106{ 107 memset(fBitmap, 0, sizeof(fBitmap)); 108} 109 110 111inline void 112CPUSet::ClearAll() 113{ 114 memset(fBitmap, 0, sizeof(fBitmap)); 115} 116 117 118inline void 119CPUSet::SetAll() 120{ 121 memset(fBitmap, ~uint8(0), sizeof(fBitmap)); 122} 123 124 125inline void 126CPUSet::SetBit(int32 cpu) 127{ 128 int32* element = (int32*)&fBitmap[cpu % kArraySize]; 129 *element |= 1u << (cpu / kArraySize); 130} 131 132 133inline void 134CPUSet::ClearBit(int32 cpu) 135{ 136 int32* element = (int32*)&fBitmap[cpu % kArraySize]; 137 *element &= ~uint32(1u << (cpu / kArraySize)); 138} 139 140 141inline void 142CPUSet::SetBitAtomic(int32 cpu) 143{ 144 int32* element = (int32*)&fBitmap[cpu % kArraySize]; 145 atomic_or(element, 1u << (cpu / kArraySize)); 146} 147 148 149inline void 150CPUSet::ClearBitAtomic(int32 cpu) 151{ 152 int32* element = (int32*)&fBitmap[cpu % kArraySize]; 153 atomic_and(element, ~uint32(1u << (cpu / kArraySize))); 154} 155 156 157inline bool 158CPUSet::GetBit(int32 cpu) const 159{ 160 int32* element = (int32*)&fBitmap[cpu % kArraySize]; 161 return ((uint32)atomic_get(element) & (1u << (cpu / kArraySize))) != 0; 162} 163 164 165inline bool 166CPUSet::IsEmpty() const 167{ 168 for (int i = 0; i < kArraySize; i++) { 169 if (fBitmap[i] != 0) 170 return false; 171 } 172 173 return true; 174} 175 176 177// Unless spinlock debug features are enabled, try to inline 178// {acquire,release}_spinlock(). 179#if !DEBUG_SPINLOCKS && !B_DEBUG_SPINLOCK_CONTENTION 180 181 182static inline bool 183try_acquire_spinlock_inline(spinlock* lock) 184{ 185 return atomic_get_and_set(&lock->lock, 1) == 0; 186} 187 188 189static inline void 190acquire_spinlock_inline(spinlock* lock) 191{ 192 if (try_acquire_spinlock_inline(lock)) 193 return; 194 acquire_spinlock(lock); 195} 196 197 198static inline void 199release_spinlock_inline(spinlock* lock) 200{ 201 atomic_set(&lock->lock, 0); 202} 203 204 205#define try_acquire_spinlock(lock) try_acquire_spinlock_inline(lock) 206#define acquire_spinlock(lock) acquire_spinlock_inline(lock) 207#define release_spinlock(lock) release_spinlock_inline(lock) 208 209 210static inline bool 211try_acquire_write_spinlock_inline(rw_spinlock* lock) 212{ 213 return atomic_test_and_set(&lock->lock, 1u << 31, 0) == 0; 214} 215 216 217static inline void 218acquire_write_spinlock_inline(rw_spinlock* lock) 219{ 220 if (try_acquire_write_spinlock(lock)) 221 return; 222 acquire_write_spinlock(lock); 223} 224 225 226static inline void 227release_write_spinlock_inline(rw_spinlock* lock) 228{ 229 atomic_set(&lock->lock, 0); 230} 231 232 233static inline bool 234try_acquire_read_spinlock_inline(rw_spinlock* lock) 235{ 236 uint32 previous = atomic_add(&lock->lock, 1); 237 return (previous & (1u << 31)) == 0; 238} 239 240 241static inline void 242acquire_read_spinlock_inline(rw_spinlock* lock) 243{ 244 if (try_acquire_read_spinlock(lock)) 245 return; 246 acquire_read_spinlock(lock); 247} 248 249 250static inline void 251release_read_spinlock_inline(rw_spinlock* lock) 252{ 253 atomic_add(&lock->lock, -1); 254} 255 256 257#define try_acquire_read_spinlock(lock) try_acquire_read_spinlock_inline(lock) 258#define acquire_read_spinlock(lock) acquire_read_spinlock_inline(lock) 259#define release_read_spinlock(lock) release_read_spinlock_inline(lock) 260#define try_acquire_write_spinlock(lock) \ 261 try_acquire_write_spinlock(lock) 262#define acquire_write_spinlock(lock) acquire_write_spinlock_inline(lock) 263#define release_write_spinlock(lock) release_write_spinlock_inline(lock) 264 265 266static inline bool 267try_acquire_write_seqlock_inline(seqlock* lock) { 268 bool succeed = try_acquire_spinlock(&lock->lock); 269 if (succeed) 270 atomic_add((int32*)&lock->count, 1); 271 return succeed; 272} 273 274 275static inline void 276acquire_write_seqlock_inline(seqlock* lock) { 277 acquire_spinlock(&lock->lock); 278 atomic_add((int32*)&lock->count, 1); 279} 280 281 282static inline void 283release_write_seqlock_inline(seqlock* lock) { 284 atomic_add((int32*)&lock->count, 1); 285 release_spinlock(&lock->lock); 286} 287 288 289static inline uint32 290acquire_read_seqlock_inline(seqlock* lock) 291{ 292 return (uint32)atomic_get((int32*)&lock->count); 293} 294 295 296static inline bool 297release_read_seqlock_inline(seqlock* lock, uint32 count) 298{ 299 uint32 current = (uint32)atomic_get((int32*)&lock->count); 300 301 return count % 2 == 0 && current == count; 302} 303 304 305#define try_acquire_write_seqlock(lock) try_acquire_write_seqlock_inline(lock) 306#define acquire_write_seqlock(lock) acquire_write_seqlock_inline(lock) 307#define release_write_seqlock(lock) release_write_seqlock_inline(lock) 308#define acquire_read_seqlock(lock) acquire_read_seqlock_inline(lock) 309#define release_read_seqlock(lock, count) \ 310 release_read_seqlock_inline(lock, count) 311 312 313#endif // !DEBUG_SPINLOCKS && !B_DEBUG_SPINLOCK_CONTENTION 314 315 316#endif /* KERNEL_SMP_H */ 317