xray_interface.cpp revision 360784
1//===-- xray_interface.cpp --------------------------------------*- C++ -*-===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file is a part of XRay, a dynamic runtime instrumentation system. 10// 11// Implementation of the API functions. 12// 13//===----------------------------------------------------------------------===// 14 15#include "xray_interface_internal.h" 16 17#include <cstdint> 18#include <cstdio> 19#include <errno.h> 20#include <limits> 21#include <string.h> 22#include <sys/mman.h> 23 24#if SANITIZER_FUCHSIA 25#include <zircon/process.h> 26#include <zircon/sanitizer.h> 27#include <zircon/status.h> 28#include <zircon/syscalls.h> 29#endif 30 31#include "sanitizer_common/sanitizer_addrhashmap.h" 32#include "sanitizer_common/sanitizer_common.h" 33 34#include "xray_defs.h" 35#include "xray_flags.h" 36 37extern __sanitizer::SpinMutex XRayInstrMapMutex; 38extern __sanitizer::atomic_uint8_t XRayInitialized; 39extern __xray::XRaySledMap XRayInstrMap; 40 41namespace __xray { 42 43#if defined(__x86_64__) 44static const int16_t cSledLength = 12; 45#elif defined(__aarch64__) 46static const int16_t cSledLength = 32; 47#elif defined(__arm__) 48static const int16_t cSledLength = 28; 49#elif SANITIZER_MIPS32 50static const int16_t cSledLength = 48; 51#elif SANITIZER_MIPS64 52static const int16_t cSledLength = 64; 53#elif defined(__powerpc64__) 54static const int16_t cSledLength = 8; 55#else 56#error "Unsupported CPU Architecture" 57#endif /* CPU architecture */ 58 59// This is the function to call when we encounter the entry or exit sleds. 60atomic_uintptr_t XRayPatchedFunction{0}; 61 62// This is the function to call from the arg1-enabled sleds/trampolines. 63atomic_uintptr_t XRayArgLogger{0}; 64 65// This is the function to call when we encounter a custom event log call. 66atomic_uintptr_t XRayPatchedCustomEvent{0}; 67 68// This is the function to call when we encounter a typed event log call. 69atomic_uintptr_t XRayPatchedTypedEvent{0}; 70 71// This is the global status to determine whether we are currently 72// patching/unpatching. 73atomic_uint8_t XRayPatching{0}; 74 75struct TypeDescription { 76 uint32_t type_id; 77 std::size_t description_string_length; 78}; 79 80using TypeDescriptorMapType = AddrHashMap<TypeDescription, 11>; 81// An address map from immutable descriptors to type ids. 82TypeDescriptorMapType TypeDescriptorAddressMap{}; 83 84atomic_uint32_t TypeEventDescriptorCounter{0}; 85 86// MProtectHelper is an RAII wrapper for calls to mprotect(...) that will 87// undo any successful mprotect(...) changes. This is used to make a page 88// writeable and executable, and upon destruction if it was successful in 89// doing so returns the page into a read-only and executable page. 90// 91// This is only used specifically for runtime-patching of the XRay 92// instrumentation points. This assumes that the executable pages are 93// originally read-and-execute only. 94class MProtectHelper { 95 void *PageAlignedAddr; 96 std::size_t MProtectLen; 97 bool MustCleanup; 98 99public: 100 explicit MProtectHelper(void *PageAlignedAddr, 101 std::size_t MProtectLen, 102 std::size_t PageSize) XRAY_NEVER_INSTRUMENT 103 : PageAlignedAddr(PageAlignedAddr), 104 MProtectLen(MProtectLen), 105 MustCleanup(false) { 106#if SANITIZER_FUCHSIA 107 MProtectLen = RoundUpTo(MProtectLen, PageSize); 108#endif 109 } 110 111 int MakeWriteable() XRAY_NEVER_INSTRUMENT { 112#if SANITIZER_FUCHSIA 113 auto R = __sanitizer_change_code_protection( 114 reinterpret_cast<uintptr_t>(PageAlignedAddr), MProtectLen, true); 115 if (R != ZX_OK) { 116 Report("XRay: cannot change code protection: %s\n", 117 _zx_status_get_string(R)); 118 return -1; 119 } 120 MustCleanup = true; 121 return 0; 122#else 123 auto R = mprotect(PageAlignedAddr, MProtectLen, 124 PROT_READ | PROT_WRITE | PROT_EXEC); 125 if (R != -1) 126 MustCleanup = true; 127 return R; 128#endif 129 } 130 131 ~MProtectHelper() XRAY_NEVER_INSTRUMENT { 132 if (MustCleanup) { 133#if SANITIZER_FUCHSIA 134 auto R = __sanitizer_change_code_protection( 135 reinterpret_cast<uintptr_t>(PageAlignedAddr), MProtectLen, false); 136 if (R != ZX_OK) { 137 Report("XRay: cannot change code protection: %s\n", 138 _zx_status_get_string(R)); 139 } 140#else 141 mprotect(PageAlignedAddr, MProtectLen, PROT_READ | PROT_EXEC); 142#endif 143 } 144 } 145}; 146 147namespace { 148 149bool patchSled(const XRaySledEntry &Sled, bool Enable, 150 int32_t FuncId) XRAY_NEVER_INSTRUMENT { 151 bool Success = false; 152 switch (Sled.Kind) { 153 case XRayEntryType::ENTRY: 154 Success = patchFunctionEntry(Enable, FuncId, Sled, __xray_FunctionEntry); 155 break; 156 case XRayEntryType::EXIT: 157 Success = patchFunctionExit(Enable, FuncId, Sled); 158 break; 159 case XRayEntryType::TAIL: 160 Success = patchFunctionTailExit(Enable, FuncId, Sled); 161 break; 162 case XRayEntryType::LOG_ARGS_ENTRY: 163 Success = patchFunctionEntry(Enable, FuncId, Sled, __xray_ArgLoggerEntry); 164 break; 165 case XRayEntryType::CUSTOM_EVENT: 166 Success = patchCustomEvent(Enable, FuncId, Sled); 167 break; 168 case XRayEntryType::TYPED_EVENT: 169 Success = patchTypedEvent(Enable, FuncId, Sled); 170 break; 171 default: 172 Report("Unsupported sled kind '%d' @%04x\n", Sled.Address, int(Sled.Kind)); 173 return false; 174 } 175 return Success; 176} 177 178XRayPatchingStatus patchFunction(int32_t FuncId, 179 bool Enable) XRAY_NEVER_INSTRUMENT { 180 if (!atomic_load(&XRayInitialized, 181 memory_order_acquire)) 182 return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized. 183 184 uint8_t NotPatching = false; 185 if (!atomic_compare_exchange_strong( 186 &XRayPatching, &NotPatching, true, memory_order_acq_rel)) 187 return XRayPatchingStatus::ONGOING; // Already patching. 188 189 // Next, we look for the function index. 190 XRaySledMap InstrMap; 191 { 192 SpinMutexLock Guard(&XRayInstrMapMutex); 193 InstrMap = XRayInstrMap; 194 } 195 196 // If we don't have an index, we can't patch individual functions. 197 if (InstrMap.Functions == 0) 198 return XRayPatchingStatus::NOT_INITIALIZED; 199 200 // FuncId must be a positive number, less than the number of functions 201 // instrumented. 202 if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions) { 203 Report("Invalid function id provided: %d\n", FuncId); 204 return XRayPatchingStatus::FAILED; 205 } 206 207 // Now we patch ths sleds for this specific function. 208 auto SledRange = InstrMap.SledsIndex[FuncId - 1]; 209 auto *f = SledRange.Begin; 210 auto *e = SledRange.End; 211 212 bool SucceedOnce = false; 213 while (f != e) 214 SucceedOnce |= patchSled(*f++, Enable, FuncId); 215 216 atomic_store(&XRayPatching, false, 217 memory_order_release); 218 219 if (!SucceedOnce) { 220 Report("Failed patching any sled for function '%d'.", FuncId); 221 return XRayPatchingStatus::FAILED; 222 } 223 224 return XRayPatchingStatus::SUCCESS; 225} 226 227// controlPatching implements the common internals of the patching/unpatching 228// implementation. |Enable| defines whether we're enabling or disabling the 229// runtime XRay instrumentation. 230XRayPatchingStatus controlPatching(bool Enable) XRAY_NEVER_INSTRUMENT { 231 if (!atomic_load(&XRayInitialized, 232 memory_order_acquire)) 233 return XRayPatchingStatus::NOT_INITIALIZED; // Not initialized. 234 235 uint8_t NotPatching = false; 236 if (!atomic_compare_exchange_strong( 237 &XRayPatching, &NotPatching, true, memory_order_acq_rel)) 238 return XRayPatchingStatus::ONGOING; // Already patching. 239 240 uint8_t PatchingSuccess = false; 241 auto XRayPatchingStatusResetter = 242 at_scope_exit([&PatchingSuccess] { 243 if (!PatchingSuccess) 244 atomic_store(&XRayPatching, false, 245 memory_order_release); 246 }); 247 248 XRaySledMap InstrMap; 249 { 250 SpinMutexLock Guard(&XRayInstrMapMutex); 251 InstrMap = XRayInstrMap; 252 } 253 if (InstrMap.Entries == 0) 254 return XRayPatchingStatus::NOT_INITIALIZED; 255 256 uint32_t FuncId = 1; 257 uint64_t CurFun = 0; 258 259 // First we want to find the bounds for which we have instrumentation points, 260 // and try to get as few calls to mprotect(...) as possible. We're assuming 261 // that all the sleds for the instrumentation map are contiguous as a single 262 // set of pages. When we do support dynamic shared object instrumentation, 263 // we'll need to do this for each set of page load offsets per DSO loaded. For 264 // now we're assuming we can mprotect the whole section of text between the 265 // minimum sled address and the maximum sled address (+ the largest sled 266 // size). 267 auto MinSled = InstrMap.Sleds[0]; 268 auto MaxSled = InstrMap.Sleds[InstrMap.Entries - 1]; 269 for (std::size_t I = 0; I < InstrMap.Entries; I++) { 270 const auto &Sled = InstrMap.Sleds[I]; 271 if (Sled.Address < MinSled.Address) 272 MinSled = Sled; 273 if (Sled.Address > MaxSled.Address) 274 MaxSled = Sled; 275 } 276 277 const size_t PageSize = flags()->xray_page_size_override > 0 278 ? flags()->xray_page_size_override 279 : GetPageSizeCached(); 280 if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) { 281 Report("System page size is not a power of two: %lld\n", PageSize); 282 return XRayPatchingStatus::FAILED; 283 } 284 285 void *PageAlignedAddr = 286 reinterpret_cast<void *>(MinSled.Address & ~(PageSize - 1)); 287 size_t MProtectLen = 288 (MaxSled.Address - reinterpret_cast<uptr>(PageAlignedAddr)) + cSledLength; 289 MProtectHelper Protector(PageAlignedAddr, MProtectLen, PageSize); 290 if (Protector.MakeWriteable() == -1) { 291 Report("Failed mprotect: %d\n", errno); 292 return XRayPatchingStatus::FAILED; 293 } 294 295 for (std::size_t I = 0; I < InstrMap.Entries; ++I) { 296 auto &Sled = InstrMap.Sleds[I]; 297 auto F = Sled.Function; 298 if (CurFun == 0) 299 CurFun = F; 300 if (F != CurFun) { 301 ++FuncId; 302 CurFun = F; 303 } 304 patchSled(Sled, Enable, FuncId); 305 } 306 atomic_store(&XRayPatching, false, 307 memory_order_release); 308 PatchingSuccess = true; 309 return XRayPatchingStatus::SUCCESS; 310} 311 312XRayPatchingStatus mprotectAndPatchFunction(int32_t FuncId, 313 bool Enable) XRAY_NEVER_INSTRUMENT { 314 XRaySledMap InstrMap; 315 { 316 SpinMutexLock Guard(&XRayInstrMapMutex); 317 InstrMap = XRayInstrMap; 318 } 319 320 // FuncId must be a positive number, less than the number of functions 321 // instrumented. 322 if (FuncId <= 0 || static_cast<size_t>(FuncId) > InstrMap.Functions) { 323 Report("Invalid function id provided: %d\n", FuncId); 324 return XRayPatchingStatus::FAILED; 325 } 326 327 const size_t PageSize = flags()->xray_page_size_override > 0 328 ? flags()->xray_page_size_override 329 : GetPageSizeCached(); 330 if ((PageSize == 0) || ((PageSize & (PageSize - 1)) != 0)) { 331 Report("Provided page size is not a power of two: %lld\n", PageSize); 332 return XRayPatchingStatus::FAILED; 333 } 334 335 // Here we compute the minumum sled and maximum sled associated with a 336 // particular function ID. 337 auto SledRange = InstrMap.SledsIndex[FuncId - 1]; 338 auto *f = SledRange.Begin; 339 auto *e = SledRange.End; 340 auto MinSled = *f; 341 auto MaxSled = *(SledRange.End - 1); 342 while (f != e) { 343 if (f->Address < MinSled.Address) 344 MinSled = *f; 345 if (f->Address > MaxSled.Address) 346 MaxSled = *f; 347 ++f; 348 } 349 350 void *PageAlignedAddr = 351 reinterpret_cast<void *>(MinSled.Address & ~(PageSize - 1)); 352 size_t MProtectLen = 353 (MaxSled.Address - reinterpret_cast<uptr>(PageAlignedAddr)) + cSledLength; 354 MProtectHelper Protector(PageAlignedAddr, MProtectLen, PageSize); 355 if (Protector.MakeWriteable() == -1) { 356 Report("Failed mprotect: %d\n", errno); 357 return XRayPatchingStatus::FAILED; 358 } 359 return patchFunction(FuncId, Enable); 360} 361 362} // namespace 363 364} // namespace __xray 365 366using namespace __xray; 367 368// The following functions are declared `extern "C" {...}` in the header, hence 369// they're defined in the global namespace. 370 371int __xray_set_handler(void (*entry)(int32_t, 372 XRayEntryType)) XRAY_NEVER_INSTRUMENT { 373 if (atomic_load(&XRayInitialized, 374 memory_order_acquire)) { 375 376 atomic_store(&__xray::XRayPatchedFunction, 377 reinterpret_cast<uintptr_t>(entry), 378 memory_order_release); 379 return 1; 380 } 381 return 0; 382} 383 384int __xray_set_customevent_handler(void (*entry)(void *, size_t)) 385 XRAY_NEVER_INSTRUMENT { 386 if (atomic_load(&XRayInitialized, 387 memory_order_acquire)) { 388 atomic_store(&__xray::XRayPatchedCustomEvent, 389 reinterpret_cast<uintptr_t>(entry), 390 memory_order_release); 391 return 1; 392 } 393 return 0; 394} 395 396int __xray_set_typedevent_handler(void (*entry)( 397 uint16_t, const void *, size_t)) XRAY_NEVER_INSTRUMENT { 398 if (atomic_load(&XRayInitialized, 399 memory_order_acquire)) { 400 atomic_store(&__xray::XRayPatchedTypedEvent, 401 reinterpret_cast<uintptr_t>(entry), 402 memory_order_release); 403 return 1; 404 } 405 return 0; 406} 407 408int __xray_remove_handler() XRAY_NEVER_INSTRUMENT { 409 return __xray_set_handler(nullptr); 410} 411 412int __xray_remove_customevent_handler() XRAY_NEVER_INSTRUMENT { 413 return __xray_set_customevent_handler(nullptr); 414} 415 416int __xray_remove_typedevent_handler() XRAY_NEVER_INSTRUMENT { 417 return __xray_set_typedevent_handler(nullptr); 418} 419 420uint16_t __xray_register_event_type( 421 const char *const event_type) XRAY_NEVER_INSTRUMENT { 422 TypeDescriptorMapType::Handle h(&TypeDescriptorAddressMap, (uptr)event_type); 423 if (h.created()) { 424 h->type_id = atomic_fetch_add( 425 &TypeEventDescriptorCounter, 1, memory_order_acq_rel); 426 h->description_string_length = strnlen(event_type, 1024); 427 } 428 return h->type_id; 429} 430 431XRayPatchingStatus __xray_patch() XRAY_NEVER_INSTRUMENT { 432 return controlPatching(true); 433} 434 435XRayPatchingStatus __xray_unpatch() XRAY_NEVER_INSTRUMENT { 436 return controlPatching(false); 437} 438 439XRayPatchingStatus __xray_patch_function(int32_t FuncId) XRAY_NEVER_INSTRUMENT { 440 return mprotectAndPatchFunction(FuncId, true); 441} 442 443XRayPatchingStatus 444__xray_unpatch_function(int32_t FuncId) XRAY_NEVER_INSTRUMENT { 445 return mprotectAndPatchFunction(FuncId, false); 446} 447 448int __xray_set_handler_arg1(void (*entry)(int32_t, XRayEntryType, uint64_t)) { 449 if (!atomic_load(&XRayInitialized, 450 memory_order_acquire)) 451 return 0; 452 453 // A relaxed write might not be visible even if the current thread gets 454 // scheduled on a different CPU/NUMA node. We need to wait for everyone to 455 // have this handler installed for consistency of collected data across CPUs. 456 atomic_store(&XRayArgLogger, reinterpret_cast<uint64_t>(entry), 457 memory_order_release); 458 return 1; 459} 460 461int __xray_remove_handler_arg1() { return __xray_set_handler_arg1(nullptr); } 462 463uintptr_t __xray_function_address(int32_t FuncId) XRAY_NEVER_INSTRUMENT { 464 SpinMutexLock Guard(&XRayInstrMapMutex); 465 if (FuncId <= 0 || static_cast<size_t>(FuncId) > XRayInstrMap.Functions) 466 return 0; 467 return XRayInstrMap.SledsIndex[FuncId - 1].Begin->Function 468// On PPC, function entries are always aligned to 16 bytes. The beginning of a 469// sled might be a local entry, which is always +8 based on the global entry. 470// Always return the global entry. 471#ifdef __PPC__ 472 & ~0xf 473#endif 474 ; 475} 476 477size_t __xray_max_function_id() XRAY_NEVER_INSTRUMENT { 478 SpinMutexLock Guard(&XRayInstrMapMutex); 479 return XRayInstrMap.Functions; 480} 481