1//===-- tsan_rtl.h ----------------------------------------------*- C++ -*-===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file is a part of ThreadSanitizer (TSan), a race detector. 10// 11// Main internal TSan header file. 12// 13// Ground rules: 14// - C++ run-time should not be used (static CTORs, RTTI, exceptions, static 15// function-scope locals) 16// - All functions/classes/etc reside in namespace __tsan, except for those 17// declared in tsan_interface.h. 18// - Platform-specific files should be used instead of ifdefs (*). 19// - No system headers included in header files (*). 20// - Platform specific headres included only into platform-specific files (*). 21// 22// (*) Except when inlining is critical for performance. 23//===----------------------------------------------------------------------===// 24 25#ifndef TSAN_RTL_H 26#define TSAN_RTL_H 27 28#include "sanitizer_common/sanitizer_allocator.h" 29#include "sanitizer_common/sanitizer_allocator_internal.h" 30#include "sanitizer_common/sanitizer_asm.h" 31#include "sanitizer_common/sanitizer_common.h" 32#include "sanitizer_common/sanitizer_deadlock_detector_interface.h" 33#include "sanitizer_common/sanitizer_libignore.h" 34#include "sanitizer_common/sanitizer_suppressions.h" 35#include "sanitizer_common/sanitizer_thread_registry.h" 36#include "sanitizer_common/sanitizer_vector.h" 37#include "tsan_defs.h" 38#include "tsan_flags.h" 39#include "tsan_ignoreset.h" 40#include "tsan_ilist.h" 41#include "tsan_mman.h" 42#include "tsan_mutexset.h" 43#include "tsan_platform.h" 44#include "tsan_report.h" 45#include "tsan_shadow.h" 46#include "tsan_stack_trace.h" 47#include "tsan_sync.h" 48#include "tsan_trace.h" 49#include "tsan_vector_clock.h" 50 51#if SANITIZER_WORDSIZE != 64 52# error "ThreadSanitizer is supported only on 64-bit platforms" 53#endif 54 55namespace __tsan { 56 57#if !SANITIZER_GO 58struct MapUnmapCallback; 59# if defined(__mips64) || defined(__aarch64__) || defined(__loongarch__) || \ 60 defined(__powerpc__) || SANITIZER_RISCV64 61 62struct AP32 { 63 static const uptr kSpaceBeg = 0; 64 static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE; 65 static const uptr kMetadataSize = 0; 66 typedef __sanitizer::CompactSizeClassMap SizeClassMap; 67 static const uptr kRegionSizeLog = 20; 68 using AddressSpaceView = LocalAddressSpaceView; 69 typedef __tsan::MapUnmapCallback MapUnmapCallback; 70 static const uptr kFlags = 0; 71}; 72typedef SizeClassAllocator32<AP32> PrimaryAllocator; 73#else 74struct AP64 { // Allocator64 parameters. Deliberately using a short name. 75# if defined(__s390x__) 76 typedef MappingS390x Mapping; 77# else 78 typedef Mapping48AddressSpace Mapping; 79# endif 80 static const uptr kSpaceBeg = Mapping::kHeapMemBeg; 81 static const uptr kSpaceSize = Mapping::kHeapMemEnd - Mapping::kHeapMemBeg; 82 static const uptr kMetadataSize = 0; 83 typedef DefaultSizeClassMap SizeClassMap; 84 typedef __tsan::MapUnmapCallback MapUnmapCallback; 85 static const uptr kFlags = 0; 86 using AddressSpaceView = LocalAddressSpaceView; 87}; 88typedef SizeClassAllocator64<AP64> PrimaryAllocator; 89#endif 90typedef CombinedAllocator<PrimaryAllocator> Allocator; 91typedef Allocator::AllocatorCache AllocatorCache; 92Allocator *allocator(); 93#endif 94 95struct ThreadSignalContext; 96 97struct JmpBuf { 98 uptr sp; 99 int int_signal_send; 100 bool in_blocking_func; 101 uptr in_signal_handler; 102 uptr *shadow_stack_pos; 103}; 104 105// A Processor represents a physical thread, or a P for Go. 106// It is used to store internal resources like allocate cache, and does not 107// participate in race-detection logic (invisible to end user). 108// In C++ it is tied to an OS thread just like ThreadState, however ideally 109// it should be tied to a CPU (this way we will have fewer allocator caches). 110// In Go it is tied to a P, so there are significantly fewer Processor's than 111// ThreadState's (which are tied to Gs). 112// A ThreadState must be wired with a Processor to handle events. 113struct Processor { 114 ThreadState *thr; // currently wired thread, or nullptr 115#if !SANITIZER_GO 116 AllocatorCache alloc_cache; 117 InternalAllocatorCache internal_alloc_cache; 118#endif 119 DenseSlabAllocCache block_cache; 120 DenseSlabAllocCache sync_cache; 121 DDPhysicalThread *dd_pt; 122}; 123 124#if !SANITIZER_GO 125// ScopedGlobalProcessor temporary setups a global processor for the current 126// thread, if it does not have one. Intended for interceptors that can run 127// at the very thread end, when we already destroyed the thread processor. 128struct ScopedGlobalProcessor { 129 ScopedGlobalProcessor(); 130 ~ScopedGlobalProcessor(); 131}; 132#endif 133 134struct TidEpoch { 135 Tid tid; 136 Epoch epoch; 137}; 138 139struct TidSlot { 140 Mutex mtx; 141 Sid sid; 142 atomic_uint32_t raw_epoch; 143 ThreadState *thr; 144 Vector<TidEpoch> journal; 145 INode node; 146 147 Epoch epoch() const { 148 return static_cast<Epoch>(atomic_load(&raw_epoch, memory_order_relaxed)); 149 } 150 151 void SetEpoch(Epoch v) { 152 atomic_store(&raw_epoch, static_cast<u32>(v), memory_order_relaxed); 153 } 154 155 TidSlot(); 156} ALIGNED(SANITIZER_CACHE_LINE_SIZE); 157 158// This struct is stored in TLS. 159struct ThreadState { 160 FastState fast_state; 161 int ignore_sync; 162#if !SANITIZER_GO 163 int ignore_interceptors; 164#endif 165 uptr *shadow_stack_pos; 166 167 // Current position in tctx->trace.Back()->events (Event*). 168 atomic_uintptr_t trace_pos; 169 // PC of the last memory access, used to compute PC deltas in the trace. 170 uptr trace_prev_pc; 171 172 // Technically `current` should be a separate THREADLOCAL variable; 173 // but it is placed here in order to share cache line with previous fields. 174 ThreadState* current; 175 176 atomic_sint32_t pending_signals; 177 178 VectorClock clock; 179 180 // This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read. 181 // We do not distinguish beteween ignoring reads and writes 182 // for better performance. 183 int ignore_reads_and_writes; 184 int suppress_reports; 185 // Go does not support ignores. 186#if !SANITIZER_GO 187 IgnoreSet mop_ignore_set; 188 IgnoreSet sync_ignore_set; 189#endif 190 uptr *shadow_stack; 191 uptr *shadow_stack_end; 192#if !SANITIZER_GO 193 Vector<JmpBuf> jmp_bufs; 194 int in_symbolizer; 195 atomic_uintptr_t in_blocking_func; 196 bool in_ignored_lib; 197 bool is_inited; 198#endif 199 MutexSet mset; 200 bool is_dead; 201 const Tid tid; 202 uptr stk_addr; 203 uptr stk_size; 204 uptr tls_addr; 205 uptr tls_size; 206 ThreadContext *tctx; 207 208 DDLogicalThread *dd_lt; 209 210 TidSlot *slot; 211 uptr slot_epoch; 212 bool slot_locked; 213 214 // Current wired Processor, or nullptr. Required to handle any events. 215 Processor *proc1; 216#if !SANITIZER_GO 217 Processor *proc() { return proc1; } 218#else 219 Processor *proc(); 220#endif 221 222 atomic_uintptr_t in_signal_handler; 223 atomic_uintptr_t signal_ctx; 224 225#if !SANITIZER_GO 226 StackID last_sleep_stack_id; 227 VectorClock last_sleep_clock; 228#endif 229 230 // Set in regions of runtime that must be signal-safe and fork-safe. 231 // If set, malloc must not be called. 232 int nomalloc; 233 234 const ReportDesc *current_report; 235 236 explicit ThreadState(Tid tid); 237} ALIGNED(SANITIZER_CACHE_LINE_SIZE); 238 239#if !SANITIZER_GO 240#if SANITIZER_APPLE || SANITIZER_ANDROID 241ThreadState *cur_thread(); 242void set_cur_thread(ThreadState *thr); 243void cur_thread_finalize(); 244inline ThreadState *cur_thread_init() { return cur_thread(); } 245# else 246__attribute__((tls_model("initial-exec"))) 247extern THREADLOCAL char cur_thread_placeholder[]; 248inline ThreadState *cur_thread() { 249 return reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current; 250} 251inline ThreadState *cur_thread_init() { 252 ThreadState *thr = reinterpret_cast<ThreadState *>(cur_thread_placeholder); 253 if (UNLIKELY(!thr->current)) 254 thr->current = thr; 255 return thr->current; 256} 257inline void set_cur_thread(ThreadState *thr) { 258 reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current = thr; 259} 260inline void cur_thread_finalize() { } 261# endif // SANITIZER_APPLE || SANITIZER_ANDROID 262#endif // SANITIZER_GO 263 264class ThreadContext final : public ThreadContextBase { 265 public: 266 explicit ThreadContext(Tid tid); 267 ~ThreadContext(); 268 ThreadState *thr; 269 StackID creation_stack_id; 270 VectorClock *sync; 271 uptr sync_epoch; 272 Trace trace; 273 274 // Override superclass callbacks. 275 void OnDead() override; 276 void OnJoined(void *arg) override; 277 void OnFinished() override; 278 void OnStarted(void *arg) override; 279 void OnCreated(void *arg) override; 280 void OnReset() override; 281 void OnDetached(void *arg) override; 282}; 283 284struct RacyStacks { 285 MD5Hash hash[2]; 286 bool operator==(const RacyStacks &other) const; 287}; 288 289struct RacyAddress { 290 uptr addr_min; 291 uptr addr_max; 292}; 293 294struct FiredSuppression { 295 ReportType type; 296 uptr pc_or_addr; 297 Suppression *supp; 298}; 299 300struct Context { 301 Context(); 302 303 bool initialized; 304#if !SANITIZER_GO 305 bool after_multithreaded_fork; 306#endif 307 308 MetaMap metamap; 309 310 Mutex report_mtx; 311 int nreported; 312 atomic_uint64_t last_symbolize_time_ns; 313 314 void *background_thread; 315 atomic_uint32_t stop_background_thread; 316 317 ThreadRegistry thread_registry; 318 319 // This is used to prevent a very unlikely but very pathological behavior. 320 // Since memory access handling is not synchronized with DoReset, 321 // a thread running concurrently with DoReset can leave a bogus shadow value 322 // that will be later falsely detected as a race. For such false races 323 // RestoreStack will return false and we will not report it. 324 // However, consider that a thread leaves a whole lot of such bogus values 325 // and these values are later read by a whole lot of threads. 326 // This will cause massive amounts of ReportRace calls and lots of 327 // serialization. In very pathological cases the resulting slowdown 328 // can be >100x. This is very unlikely, but it was presumably observed 329 // in practice: https://github.com/google/sanitizers/issues/1552 330 // If this happens, previous access sid+epoch will be the same for all of 331 // these false races b/c if the thread will try to increment epoch, it will 332 // notice that DoReset has happened and will stop producing bogus shadow 333 // values. So, last_spurious_race is used to remember the last sid+epoch 334 // for which RestoreStack returned false. Then it is used to filter out 335 // races with the same sid+epoch very early and quickly. 336 // It is of course possible that multiple threads left multiple bogus shadow 337 // values and all of them are read by lots of threads at the same time. 338 // In such case last_spurious_race will only be able to deduplicate a few 339 // races from one thread, then few from another and so on. An alternative 340 // would be to hold an array of such sid+epoch, but we consider such scenario 341 // as even less likely. 342 // Note: this can lead to some rare false negatives as well: 343 // 1. When a legit access with the same sid+epoch participates in a race 344 // as the "previous" memory access, it will be wrongly filtered out. 345 // 2. When RestoreStack returns false for a legit memory access because it 346 // was already evicted from the thread trace, we will still remember it in 347 // last_spurious_race. Then if there is another racing memory access from 348 // the same thread that happened in the same epoch, but was stored in the 349 // next thread trace part (which is still preserved in the thread trace), 350 // we will also wrongly filter it out while RestoreStack would actually 351 // succeed for that second memory access. 352 RawShadow last_spurious_race; 353 354 Mutex racy_mtx; 355 Vector<RacyStacks> racy_stacks; 356 // Number of fired suppressions may be large enough. 357 Mutex fired_suppressions_mtx; 358 InternalMmapVector<FiredSuppression> fired_suppressions; 359 DDetector *dd; 360 361 Flags flags; 362 fd_t memprof_fd; 363 364 // The last slot index (kFreeSid) is used to denote freed memory. 365 TidSlot slots[kThreadSlotCount - 1]; 366 367 // Protects global_epoch, slot_queue, trace_part_recycle. 368 Mutex slot_mtx; 369 uptr global_epoch; // guarded by slot_mtx and by all slot mutexes 370 bool resetting; // global reset is in progress 371 IList<TidSlot, &TidSlot::node> slot_queue SANITIZER_GUARDED_BY(slot_mtx); 372 IList<TraceHeader, &TraceHeader::global, TracePart> trace_part_recycle 373 SANITIZER_GUARDED_BY(slot_mtx); 374 uptr trace_part_total_allocated SANITIZER_GUARDED_BY(slot_mtx); 375 uptr trace_part_recycle_finished SANITIZER_GUARDED_BY(slot_mtx); 376 uptr trace_part_finished_excess SANITIZER_GUARDED_BY(slot_mtx); 377#if SANITIZER_GO 378 uptr mapped_shadow_begin; 379 uptr mapped_shadow_end; 380#endif 381}; 382 383extern Context *ctx; // The one and the only global runtime context. 384 385ALWAYS_INLINE Flags *flags() { 386 return &ctx->flags; 387} 388 389struct ScopedIgnoreInterceptors { 390 ScopedIgnoreInterceptors() { 391#if !SANITIZER_GO 392 cur_thread()->ignore_interceptors++; 393#endif 394 } 395 396 ~ScopedIgnoreInterceptors() { 397#if !SANITIZER_GO 398 cur_thread()->ignore_interceptors--; 399#endif 400 } 401}; 402 403const char *GetObjectTypeFromTag(uptr tag); 404const char *GetReportHeaderFromTag(uptr tag); 405uptr TagFromShadowStackFrame(uptr pc); 406 407class ScopedReportBase { 408 public: 409 void AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, Tid tid, 410 StackTrace stack, const MutexSet *mset); 411 void AddStack(StackTrace stack, bool suppressable = false); 412 void AddThread(const ThreadContext *tctx, bool suppressable = false); 413 void AddThread(Tid tid, bool suppressable = false); 414 void AddUniqueTid(Tid unique_tid); 415 int AddMutex(uptr addr, StackID creation_stack_id); 416 void AddLocation(uptr addr, uptr size); 417 void AddSleep(StackID stack_id); 418 void SetCount(int count); 419 void SetSigNum(int sig); 420 421 const ReportDesc *GetReport() const; 422 423 protected: 424 ScopedReportBase(ReportType typ, uptr tag); 425 ~ScopedReportBase(); 426 427 private: 428 ReportDesc *rep_; 429 // Symbolizer makes lots of intercepted calls. If we try to process them, 430 // at best it will cause deadlocks on internal mutexes. 431 ScopedIgnoreInterceptors ignore_interceptors_; 432 433 ScopedReportBase(const ScopedReportBase &) = delete; 434 void operator=(const ScopedReportBase &) = delete; 435}; 436 437class ScopedReport : public ScopedReportBase { 438 public: 439 explicit ScopedReport(ReportType typ, uptr tag = kExternalTagNone); 440 ~ScopedReport(); 441 442 private: 443 ScopedErrorReportLock lock_; 444}; 445 446bool ShouldReport(ThreadState *thr, ReportType typ); 447ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack); 448 449// The stack could look like: 450// <start> | <main> | <foo> | tag | <bar> 451// This will extract the tag and keep: 452// <start> | <main> | <foo> | <bar> 453template<typename StackTraceTy> 454void ExtractTagFromStack(StackTraceTy *stack, uptr *tag = nullptr) { 455 if (stack->size < 2) return; 456 uptr possible_tag_pc = stack->trace[stack->size - 2]; 457 uptr possible_tag = TagFromShadowStackFrame(possible_tag_pc); 458 if (possible_tag == kExternalTagNone) return; 459 stack->trace_buffer[stack->size - 2] = stack->trace_buffer[stack->size - 1]; 460 stack->size -= 1; 461 if (tag) *tag = possible_tag; 462} 463 464template<typename StackTraceTy> 465void ObtainCurrentStack(ThreadState *thr, uptr toppc, StackTraceTy *stack, 466 uptr *tag = nullptr) { 467 uptr size = thr->shadow_stack_pos - thr->shadow_stack; 468 uptr start = 0; 469 if (size + !!toppc > kStackTraceMax) { 470 start = size + !!toppc - kStackTraceMax; 471 size = kStackTraceMax - !!toppc; 472 } 473 stack->Init(&thr->shadow_stack[start], size, toppc); 474 ExtractTagFromStack(stack, tag); 475} 476 477#define GET_STACK_TRACE_FATAL(thr, pc) \ 478 VarSizeStackTrace stack; \ 479 ObtainCurrentStack(thr, pc, &stack); \ 480 stack.ReverseOrder(); 481 482void MapShadow(uptr addr, uptr size); 483void MapThreadTrace(uptr addr, uptr size, const char *name); 484void DontNeedShadowFor(uptr addr, uptr size); 485void UnmapShadow(ThreadState *thr, uptr addr, uptr size); 486void InitializeShadowMemory(); 487void DontDumpShadow(uptr addr, uptr size); 488void InitializeInterceptors(); 489void InitializeLibIgnore(); 490void InitializeDynamicAnnotations(); 491 492void ForkBefore(ThreadState *thr, uptr pc); 493void ForkParentAfter(ThreadState *thr, uptr pc); 494void ForkChildAfter(ThreadState *thr, uptr pc, bool start_thread); 495 496void ReportRace(ThreadState *thr, RawShadow *shadow_mem, Shadow cur, Shadow old, 497 AccessType typ); 498bool OutputReport(ThreadState *thr, const ScopedReport &srep); 499bool IsFiredSuppression(Context *ctx, ReportType type, StackTrace trace); 500bool IsExpectedReport(uptr addr, uptr size); 501 502#if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 1 503# define DPrintf Printf 504#else 505# define DPrintf(...) 506#endif 507 508#if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 2 509# define DPrintf2 Printf 510#else 511# define DPrintf2(...) 512#endif 513 514StackID CurrentStackId(ThreadState *thr, uptr pc); 515ReportStack *SymbolizeStackId(StackID stack_id); 516void PrintCurrentStack(ThreadState *thr, uptr pc); 517void PrintCurrentStackSlow(uptr pc); // uses libunwind 518MBlock *JavaHeapBlock(uptr addr, uptr *start); 519 520void Initialize(ThreadState *thr); 521void MaybeSpawnBackgroundThread(); 522int Finalize(ThreadState *thr); 523 524void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write); 525void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write); 526 527void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size, 528 AccessType typ); 529void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size, 530 AccessType typ); 531// This creates 2 non-inlined specialized versions of MemoryAccessRange. 532template <bool is_read> 533void MemoryAccessRangeT(ThreadState *thr, uptr pc, uptr addr, uptr size); 534 535ALWAYS_INLINE 536void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size, 537 bool is_write) { 538 if (size == 0) 539 return; 540 if (is_write) 541 MemoryAccessRangeT<false>(thr, pc, addr, size); 542 else 543 MemoryAccessRangeT<true>(thr, pc, addr, size); 544} 545 546void ShadowSet(RawShadow *p, RawShadow *end, RawShadow v); 547void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size); 548void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size); 549void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size); 550void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr, 551 uptr size); 552 553void ThreadIgnoreBegin(ThreadState *thr, uptr pc); 554void ThreadIgnoreEnd(ThreadState *thr); 555void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc); 556void ThreadIgnoreSyncEnd(ThreadState *thr); 557 558Tid ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached); 559void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id, 560 ThreadType thread_type); 561void ThreadFinish(ThreadState *thr); 562Tid ThreadConsumeTid(ThreadState *thr, uptr pc, uptr uid); 563void ThreadJoin(ThreadState *thr, uptr pc, Tid tid); 564void ThreadDetach(ThreadState *thr, uptr pc, Tid tid); 565void ThreadFinalize(ThreadState *thr); 566void ThreadSetName(ThreadState *thr, const char *name); 567int ThreadCount(ThreadState *thr); 568void ProcessPendingSignalsImpl(ThreadState *thr); 569void ThreadNotJoined(ThreadState *thr, uptr pc, Tid tid, uptr uid); 570 571Processor *ProcCreate(); 572void ProcDestroy(Processor *proc); 573void ProcWire(Processor *proc, ThreadState *thr); 574void ProcUnwire(Processor *proc, ThreadState *thr); 575 576// Note: the parameter is called flagz, because flags is already taken 577// by the global function that returns flags. 578void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0); 579void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0); 580void MutexPreLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0); 581void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0, 582 int rec = 1); 583int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0); 584void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0); 585void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz = 0); 586void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr); 587void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr); 588void MutexRepair(ThreadState *thr, uptr pc, uptr addr); // call on EOWNERDEAD 589void MutexInvalidAccess(ThreadState *thr, uptr pc, uptr addr); 590 591void Acquire(ThreadState *thr, uptr pc, uptr addr); 592// AcquireGlobal synchronizes the current thread with all other threads. 593// In terms of happens-before relation, it draws a HB edge from all threads 594// (where they happen to execute right now) to the current thread. We use it to 595// handle Go finalizers. Namely, finalizer goroutine executes AcquireGlobal 596// right before executing finalizers. This provides a coarse, but simple 597// approximation of the actual required synchronization. 598void AcquireGlobal(ThreadState *thr); 599void Release(ThreadState *thr, uptr pc, uptr addr); 600void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr); 601void ReleaseStore(ThreadState *thr, uptr pc, uptr addr); 602void AfterSleep(ThreadState *thr, uptr pc); 603void IncrementEpoch(ThreadState *thr); 604 605#if !SANITIZER_GO 606uptr ALWAYS_INLINE HeapEnd() { 607 return HeapMemEnd() + PrimaryAllocator::AdditionalSize(); 608} 609#endif 610 611void SlotAttachAndLock(ThreadState *thr) SANITIZER_ACQUIRE(thr->slot->mtx); 612void SlotDetach(ThreadState *thr); 613void SlotLock(ThreadState *thr) SANITIZER_ACQUIRE(thr->slot->mtx); 614void SlotUnlock(ThreadState *thr) SANITIZER_RELEASE(thr->slot->mtx); 615void DoReset(ThreadState *thr, uptr epoch); 616void FlushShadowMemory(); 617 618ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags); 619void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber); 620void FiberSwitch(ThreadState *thr, uptr pc, ThreadState *fiber, unsigned flags); 621 622// These need to match __tsan_switch_to_fiber_* flags defined in 623// tsan_interface.h. See documentation there as well. 624enum FiberSwitchFlags { 625 FiberSwitchFlagNoSync = 1 << 0, // __tsan_switch_to_fiber_no_sync 626}; 627 628class SlotLocker { 629 public: 630 ALWAYS_INLINE 631 SlotLocker(ThreadState *thr, bool recursive = false) 632 : thr_(thr), locked_(recursive ? thr->slot_locked : false) { 633#if !SANITIZER_GO 634 // We are in trouble if we are here with in_blocking_func set. 635 // If in_blocking_func is set, all signals will be delivered synchronously, 636 // which means we can't lock slots since the signal handler will try 637 // to lock it recursively and deadlock. 638 DCHECK(!atomic_load(&thr->in_blocking_func, memory_order_relaxed)); 639#endif 640 if (!locked_) 641 SlotLock(thr_); 642 } 643 644 ALWAYS_INLINE 645 ~SlotLocker() { 646 if (!locked_) 647 SlotUnlock(thr_); 648 } 649 650 private: 651 ThreadState *thr_; 652 bool locked_; 653}; 654 655class SlotUnlocker { 656 public: 657 SlotUnlocker(ThreadState *thr) : thr_(thr), locked_(thr->slot_locked) { 658 if (locked_) 659 SlotUnlock(thr_); 660 } 661 662 ~SlotUnlocker() { 663 if (locked_) 664 SlotLock(thr_); 665 } 666 667 private: 668 ThreadState *thr_; 669 bool locked_; 670}; 671 672ALWAYS_INLINE void ProcessPendingSignals(ThreadState *thr) { 673 if (UNLIKELY(atomic_load_relaxed(&thr->pending_signals))) 674 ProcessPendingSignalsImpl(thr); 675} 676 677extern bool is_initialized; 678 679ALWAYS_INLINE 680void LazyInitialize(ThreadState *thr) { 681 // If we can use .preinit_array, assume that __tsan_init 682 // called from .preinit_array initializes runtime before 683 // any instrumented code except when tsan is used as a 684 // shared library. 685#if (!SANITIZER_CAN_USE_PREINIT_ARRAY || defined(SANITIZER_SHARED)) 686 if (UNLIKELY(!is_initialized)) 687 Initialize(thr); 688#endif 689} 690 691void TraceResetForTesting(); 692void TraceSwitchPart(ThreadState *thr); 693void TraceSwitchPartImpl(ThreadState *thr); 694bool RestoreStack(EventType type, Sid sid, Epoch epoch, uptr addr, uptr size, 695 AccessType typ, Tid *ptid, VarSizeStackTrace *pstk, 696 MutexSet *pmset, uptr *ptag); 697 698template <typename EventT> 699ALWAYS_INLINE WARN_UNUSED_RESULT bool TraceAcquire(ThreadState *thr, 700 EventT **ev) { 701 // TraceSwitchPart accesses shadow_stack, but it's called infrequently, 702 // so we check it here proactively. 703 DCHECK(thr->shadow_stack); 704 Event *pos = reinterpret_cast<Event *>(atomic_load_relaxed(&thr->trace_pos)); 705#if SANITIZER_DEBUG 706 // TraceSwitch acquires these mutexes, 707 // so we lock them here to detect deadlocks more reliably. 708 { Lock lock(&ctx->slot_mtx); } 709 { Lock lock(&thr->tctx->trace.mtx); } 710 TracePart *current = thr->tctx->trace.parts.Back(); 711 if (current) { 712 DCHECK_GE(pos, ¤t->events[0]); 713 DCHECK_LE(pos, ¤t->events[TracePart::kSize]); 714 } else { 715 DCHECK_EQ(pos, nullptr); 716 } 717#endif 718 // TracePart is allocated with mmap and is at least 4K aligned. 719 // So the following check is a faster way to check for part end. 720 // It may have false positives in the middle of the trace, 721 // they are filtered out in TraceSwitch. 722 if (UNLIKELY(((uptr)(pos + 1) & TracePart::kAlignment) == 0)) 723 return false; 724 *ev = reinterpret_cast<EventT *>(pos); 725 return true; 726} 727 728template <typename EventT> 729ALWAYS_INLINE void TraceRelease(ThreadState *thr, EventT *evp) { 730 DCHECK_LE(evp + 1, &thr->tctx->trace.parts.Back()->events[TracePart::kSize]); 731 atomic_store_relaxed(&thr->trace_pos, (uptr)(evp + 1)); 732} 733 734template <typename EventT> 735void TraceEvent(ThreadState *thr, EventT ev) { 736 EventT *evp; 737 if (!TraceAcquire(thr, &evp)) { 738 TraceSwitchPart(thr); 739 UNUSED bool res = TraceAcquire(thr, &evp); 740 DCHECK(res); 741 } 742 *evp = ev; 743 TraceRelease(thr, evp); 744} 745 746ALWAYS_INLINE WARN_UNUSED_RESULT bool TryTraceFunc(ThreadState *thr, 747 uptr pc = 0) { 748 if (!kCollectHistory) 749 return true; 750 EventFunc *ev; 751 if (UNLIKELY(!TraceAcquire(thr, &ev))) 752 return false; 753 ev->is_access = 0; 754 ev->is_func = 1; 755 ev->pc = pc; 756 TraceRelease(thr, ev); 757 return true; 758} 759 760WARN_UNUSED_RESULT 761bool TryTraceMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size, 762 AccessType typ); 763WARN_UNUSED_RESULT 764bool TryTraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size, 765 AccessType typ); 766void TraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size, 767 AccessType typ); 768void TraceFunc(ThreadState *thr, uptr pc = 0); 769void TraceMutexLock(ThreadState *thr, EventType type, uptr pc, uptr addr, 770 StackID stk); 771void TraceMutexUnlock(ThreadState *thr, uptr addr); 772void TraceTime(ThreadState *thr); 773 774void TraceRestartFuncExit(ThreadState *thr); 775void TraceRestartFuncEntry(ThreadState *thr, uptr pc); 776 777void GrowShadowStack(ThreadState *thr); 778 779ALWAYS_INLINE 780void FuncEntry(ThreadState *thr, uptr pc) { 781 DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.sid(), (void *)pc); 782 if (UNLIKELY(!TryTraceFunc(thr, pc))) 783 return TraceRestartFuncEntry(thr, pc); 784 DCHECK_GE(thr->shadow_stack_pos, thr->shadow_stack); 785#if !SANITIZER_GO 786 DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end); 787#else 788 if (thr->shadow_stack_pos == thr->shadow_stack_end) 789 GrowShadowStack(thr); 790#endif 791 thr->shadow_stack_pos[0] = pc; 792 thr->shadow_stack_pos++; 793} 794 795ALWAYS_INLINE 796void FuncExit(ThreadState *thr) { 797 DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.sid()); 798 if (UNLIKELY(!TryTraceFunc(thr, 0))) 799 return TraceRestartFuncExit(thr); 800 DCHECK_GT(thr->shadow_stack_pos, thr->shadow_stack); 801#if !SANITIZER_GO 802 DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end); 803#endif 804 thr->shadow_stack_pos--; 805} 806 807#if !SANITIZER_GO 808extern void (*on_initialize)(void); 809extern int (*on_finalize)(int); 810#endif 811} // namespace __tsan 812 813#endif // TSAN_RTL_H 814