1/* 2 * Copyright (c) 2013 Apple Inc. All rights reserved. 3 * 4 * @APPLE_APACHE_LICENSE_HEADER_START@ 5 * 6 * Licensed under the Apache License, Version 2.0 (the "License"); 7 * you may not use this file except in compliance with the License. 8 * You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 * 18 * @APPLE_APACHE_LICENSE_HEADER_END@ 19 */ 20 21/* 22 * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch 23 * which are subject to change in future releases of Mac OS X. Any applications 24 * relying on these interfaces WILL break. 25 */ 26 27#ifndef __DISPATCH_VOUCHER_INTERNAL__ 28#define __DISPATCH_VOUCHER_INTERNAL__ 29 30#ifndef __DISPATCH_INDIRECT__ 31#error "Please #include <dispatch/dispatch.h> instead of this file directly." 32#include <dispatch/base.h> // for HeaderDoc 33#endif 34 35#pragma mark - 36#pragma mark voucher_recipe_t (disabled) 37 38#if VOUCHER_ENABLE_RECIPE_OBJECTS 39/*! 40 * @group Voucher Creation SPI 41 * SPI intended for clients that need to create vouchers. 42 */ 43 44#if OS_OBJECT_USE_OBJC 45OS_OBJECT_DECL(voucher_recipe); 46#else 47typedef struct voucher_recipe_s *voucher_recipe_t; 48#endif 49 50/*! 51 * @function voucher_create 52 * 53 * @abstract 54 * Creates a new voucher object from a recipe. 55 * 56 * @discussion 57 * Error handling TBD 58 * 59 * @result 60 * The newly created voucher object. 61 */ 62__OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0) 63OS_EXPORT OS_OBJECT_RETURNS_RETAINED OS_WARN_RESULT OS_NOTHROW 64voucher_t 65voucher_create(voucher_recipe_t recipe); 66#endif // VOUCHER_ENABLE_RECIPE_OBJECTS 67 68#if VOUCHER_ENABLE_GET_MACH_VOUCHER 69/*! 70 * @function voucher_get_mach_voucher 71 * 72 * @abstract 73 * Returns the mach voucher port underlying the specified voucher object. 74 * 75 * @discussion 76 * The caller must either maintain a reference on the voucher object while the 77 * returned mach voucher port is in use to ensure it stays valid for the 78 * duration, or it must retain the mach voucher port with mach_port_mod_refs(). 79 * 80 * @param voucher 81 * The voucher object to query. 82 * 83 * @result 84 * A mach voucher port. 85 */ 86__OSX_AVAILABLE_STARTING(__MAC_10_10,__IPHONE_8_0) 87OS_VOUCHER_EXPORT OS_WARN_RESULT OS_NOTHROW 88mach_voucher_t 89voucher_get_mach_voucher(voucher_t voucher); 90#endif // VOUCHER_ENABLE_GET_MACH_VOUCHER 91 92#pragma mark - 93#pragma mark voucher_t 94 95#if TARGET_IPHONE_SIMULATOR && \ 96 IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 101000 97#undef VOUCHER_USE_MACH_VOUCHER 98#define VOUCHER_USE_MACH_VOUCHER 0 99#endif 100#ifndef VOUCHER_USE_MACH_VOUCHER 101#if __has_include(<mach/mach_voucher.h>) 102#define VOUCHER_USE_MACH_VOUCHER 1 103#endif 104#endif 105 106#if VOUCHER_USE_MACH_VOUCHER 107#undef DISPATCH_USE_IMPORTANCE_ASSERTION 108#define DISPATCH_USE_IMPORTANCE_ASSERTION 0 109#else 110#undef MACH_RCV_VOUCHER 111#define MACH_RCV_VOUCHER 0 112#endif // VOUCHER_USE_MACH_VOUCHER 113 114void _voucher_init(void); 115void _voucher_atfork_child(void); 116void _voucher_activity_heap_pressure_warn(void); 117void _voucher_activity_heap_pressure_normal(void); 118void _voucher_xref_dispose(voucher_t voucher); 119void _voucher_dispose(voucher_t voucher); 120size_t _voucher_debug(voucher_t v, char* buf, size_t bufsiz); 121void _voucher_thread_cleanup(void *voucher); 122mach_voucher_t _voucher_get_mach_voucher(voucher_t voucher); 123voucher_t _voucher_create_without_importance(voucher_t voucher); 124mach_voucher_t _voucher_create_mach_voucher_with_priority(voucher_t voucher, 125 pthread_priority_t priority); 126voucher_t _voucher_create_with_priority_and_mach_voucher(voucher_t voucher, 127 pthread_priority_t priority, mach_voucher_t kv); 128void _voucher_dealloc_mach_voucher(mach_voucher_t kv); 129 130#if OS_OBJECT_USE_OBJC 131_OS_OBJECT_DECL_SUBCLASS_INTERFACE(voucher, object) 132#if VOUCHER_ENABLE_RECIPE_OBJECTS 133_OS_OBJECT_DECL_SUBCLASS_INTERFACE(voucher_recipe, object) 134#endif 135#endif 136 137#define _TAILQ_IS_ENQUEUED(elm, field) \ 138 ((elm)->field.tqe_prev != NULL) 139#define _TAILQ_MARK_NOT_ENQUEUED(elm, field) \ 140 do { (elm)->field.tqe_prev = NULL; } while (0) 141 142#define VOUCHER_NO_MACH_VOUCHER MACH_PORT_DEAD 143 144#if VOUCHER_USE_MACH_VOUCHER 145 146#if DISPATCH_DEBUG 147#define DISPATCH_VOUCHER_DEBUG 1 148#define DISPATCH_VOUCHER_ACTIVITY_DEBUG 1 149#endif 150 151typedef struct voucher_s { 152 _OS_OBJECT_HEADER( 153 void *os_obj_isa, 154 os_obj_ref_cnt, 155 os_obj_xref_cnt); 156 TAILQ_ENTRY(voucher_s) v_list; 157 mach_voucher_t v_kvoucher, v_ipc_kvoucher; // if equal, only one reference 158 voucher_t v_kvbase; // if non-NULL, v_kvoucher is a borrowed reference 159 _voucher_activity_t v_activity; 160#if VOUCHER_ENABLE_RECIPE_OBJECTS 161 size_t v_recipe_extra_offset; 162 mach_voucher_attr_recipe_size_t v_recipe_extra_size; 163#endif 164 unsigned int v_has_priority:1; 165 unsigned int v_activities; 166 mach_voucher_attr_recipe_data_t v_recipes[]; 167} voucher_s; 168 169#if VOUCHER_ENABLE_RECIPE_OBJECTS 170typedef struct voucher_recipe_s { 171 _OS_OBJECT_HEADER( 172 const _os_object_class_s *os_obj_isa, 173 os_obj_ref_cnt, 174 os_obj_xref_cnt); 175 size_t vr_allocation_size; 176 mach_voucher_attr_recipe_size_t volatile vr_size; 177 mach_voucher_attr_recipe_t vr_data; 178} voucher_recipe_s; 179#endif 180 181#define _voucher_recipes_base(r) (r[0]) 182#define _voucher_recipes_atm(r) (r[1]) 183#define _voucher_recipes_bits(r) (r[2]) 184#define _voucher_base_recipe(v) (_voucher_recipes_base((v)->v_recipes)) 185#define _voucher_atm_recipe(v) (_voucher_recipes_atm((v)->v_recipes)) 186#define _voucher_bits_recipe(v) (_voucher_recipes_bits((v)->v_recipes)) 187#define _voucher_recipes_size() (3 * sizeof(mach_voucher_attr_recipe_data_t)) 188 189#if TARGET_OS_EMBEDDED 190#define VL_HASH_SIZE 64u // must be a power of two 191#else 192#define VL_HASH_SIZE 256u // must be a power of two 193#endif 194#define VL_HASH(kv) (MACH_PORT_INDEX(kv) & (VL_HASH_SIZE - 1)) 195 196typedef uint32_t _voucher_magic_t; 197const _voucher_magic_t _voucher_magic_v1 = 0x0190cefa; // little-endian FACE9001 198#define _voucher_recipes_magic(r) ((_voucher_magic_t*) \ 199 (_voucher_recipes_bits(r).content)) 200#define _voucher_magic(v) _voucher_recipes_magic((v)->v_recipes) 201typedef uint32_t _voucher_priority_t; 202#define _voucher_recipes_priority(r) ((_voucher_priority_t*) \ 203 (_voucher_recipes_bits(r).content + sizeof(_voucher_magic_t))) 204#define _voucher_priority(v) _voucher_recipes_priority((v)->v_recipes) 205#define _voucher_activity_ids(v) ((voucher_activity_id_t*) \ 206 (_voucher_bits_recipe(v).content + sizeof(_voucher_magic_t) + \ 207 sizeof(_voucher_priority_t))) 208#define _voucher_bits_size(activities) \ 209 (sizeof(_voucher_magic_t) + sizeof(_voucher_priority_t) + \ 210 (activities) * sizeof(voucher_activity_id_t)) 211 212#if VOUCHER_ENABLE_RECIPE_OBJECTS 213#define _voucher_extra_size(v) ((v)->v_recipe_extra_size) 214#define _voucher_extra_recipes(v) ((char*)(v) + (v)->v_recipe_extra_offset) 215#else 216#define _voucher_extra_size(v) 0 217#define _voucher_extra_recipes(v) NULL 218#endif 219 220#if DISPATCH_DEBUG && DISPATCH_VOUCHER_DEBUG 221#define _dispatch_voucher_debug(msg, v, ...) \ 222 _dispatch_debug("voucher[%p]: " msg, v, ##__VA_ARGS__) 223#define _dispatch_kvoucher_debug(msg, kv, ...) \ 224 _dispatch_debug("kvoucher[0x%08x]: " msg, kv, ##__VA_ARGS__) 225#define _dispatch_voucher_debug_machport(name) \ 226 dispatch_debug_machport((name), __func__) 227#else 228#define _dispatch_voucher_debug(msg, v, ...) 229#define _dispatch_kvoucher_debug(msg, kv, ...) 230#define _dispatch_voucher_debug_machport(name) ((void)(name)) 231#endif 232 233#if !(USE_OBJC && __OBJC2__) 234 235DISPATCH_ALWAYS_INLINE 236static inline voucher_t 237_voucher_retain(voucher_t voucher) 238{ 239#if !DISPATCH_VOUCHER_OBJC_DEBUG 240 int xref_cnt = dispatch_atomic_inc2o(voucher, os_obj_xref_cnt, relaxed); 241 _dispatch_voucher_debug("retain -> %d", voucher, xref_cnt + 1); 242 if (slowpath(xref_cnt <= 0)) { 243 _dispatch_voucher_debug("resurrection", voucher); 244 DISPATCH_CRASH("Voucher resurrection"); 245 } 246#else 247 os_retain(voucher); 248 _dispatch_voucher_debug("retain -> %d", voucher, 249 voucher->os_obj_xref_cnt + 1); 250#endif // DISPATCH_DEBUG 251 return voucher; 252} 253 254DISPATCH_ALWAYS_INLINE 255static inline void 256_voucher_release(voucher_t voucher) 257{ 258#if !DISPATCH_VOUCHER_OBJC_DEBUG 259 int xref_cnt = dispatch_atomic_dec2o(voucher, os_obj_xref_cnt, relaxed); 260 _dispatch_voucher_debug("release -> %d", voucher, xref_cnt + 1); 261 if (fastpath(xref_cnt >= 0)) { 262 return; 263 } 264 if (slowpath(xref_cnt < -1)) { 265 _dispatch_voucher_debug("overrelease", voucher); 266 DISPATCH_CRASH("Voucher overrelease"); 267 } 268 return _os_object_xref_dispose((_os_object_t)voucher); 269#else 270 _dispatch_voucher_debug("release -> %d", voucher, voucher->os_obj_xref_cnt); 271 return os_release(voucher); 272#endif // DISPATCH_DEBUG 273} 274 275DISPATCH_ALWAYS_INLINE 276static inline voucher_t 277_voucher_get(void) 278{ 279 return _dispatch_thread_getspecific(dispatch_voucher_key); 280} 281 282DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT 283static inline voucher_t 284_voucher_copy(void) 285{ 286 voucher_t voucher = _voucher_get(); 287 if (voucher) _voucher_retain(voucher); 288 return voucher; 289} 290 291DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT 292static inline voucher_t 293_voucher_copy_without_importance(void) 294{ 295 voucher_t voucher = _voucher_get(); 296 if (voucher) voucher = _voucher_create_without_importance(voucher); 297 return voucher; 298} 299 300DISPATCH_ALWAYS_INLINE 301static inline void 302_voucher_mach_voucher_set(mach_voucher_t kv) 303{ 304 if (kv == VOUCHER_NO_MACH_VOUCHER) return; 305 _dispatch_set_priority_and_mach_voucher(0, kv); 306} 307 308DISPATCH_ALWAYS_INLINE 309static inline mach_voucher_t 310_voucher_swap_and_get_mach_voucher(voucher_t ov, voucher_t voucher) 311{ 312 if (ov == voucher) return VOUCHER_NO_MACH_VOUCHER; 313 _dispatch_voucher_debug("swap from voucher[%p]", voucher, ov); 314 _dispatch_thread_setspecific(dispatch_voucher_key, voucher); 315 mach_voucher_t kv = voucher ? voucher->v_kvoucher : MACH_VOUCHER_NULL; 316 mach_voucher_t okv = ov ? ov->v_kvoucher : MACH_VOUCHER_NULL; 317 return (kv != okv) ? kv : VOUCHER_NO_MACH_VOUCHER; 318} 319 320DISPATCH_ALWAYS_INLINE 321static inline void 322_voucher_swap(voucher_t ov, voucher_t voucher) 323{ 324 _voucher_mach_voucher_set(_voucher_swap_and_get_mach_voucher(ov, voucher)); 325 if (ov) _voucher_release(ov); 326} 327 328DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT 329static inline voucher_t 330_voucher_adopt(voucher_t voucher) 331{ 332 voucher_t ov = _voucher_get(); 333 _voucher_mach_voucher_set(_voucher_swap_and_get_mach_voucher(ov, voucher)); 334 return ov; 335} 336 337DISPATCH_ALWAYS_INLINE 338static inline void 339_voucher_replace(voucher_t voucher) 340{ 341 voucher_t ov = _voucher_get(); 342 _voucher_swap(ov, voucher); 343} 344 345DISPATCH_ALWAYS_INLINE 346static inline void 347_voucher_clear(void) 348{ 349 _voucher_replace(NULL); 350} 351 352DISPATCH_ALWAYS_INLINE 353static inline pthread_priority_t 354_voucher_get_priority(voucher_t voucher) 355{ 356 return voucher && voucher->v_has_priority ? 357 (pthread_priority_t)*_voucher_priority(voucher) : 0; 358} 359 360void _voucher_task_mach_voucher_init(void* ctxt); 361extern dispatch_once_t _voucher_task_mach_voucher_pred; 362extern mach_voucher_t _voucher_task_mach_voucher; 363 364DISPATCH_ALWAYS_INLINE 365static inline mach_voucher_t 366_voucher_get_task_mach_voucher(void) 367{ 368 dispatch_once_f(&_voucher_task_mach_voucher_pred, NULL, 369 _voucher_task_mach_voucher_init); 370 return _voucher_task_mach_voucher; 371} 372 373DISPATCH_ALWAYS_INLINE 374static inline bool 375_voucher_mach_msg_set_mach_voucher(mach_msg_header_t *msg, mach_voucher_t kv, 376 bool move_send) 377{ 378 if (MACH_MSGH_BITS_HAS_VOUCHER(msg->msgh_bits)) return false; 379 if (!kv) return false; 380 msg->msgh_voucher_port = kv; 381 msg->msgh_bits |= MACH_MSGH_BITS_SET_PORTS(0, 0, move_send ? 382 MACH_MSG_TYPE_MOVE_SEND : MACH_MSG_TYPE_COPY_SEND); 383 _dispatch_kvoucher_debug("msg[%p] set %s", kv, msg, move_send ? 384 "move-send" : "copy-send"); 385 _dispatch_voucher_debug_machport(kv); 386 return true; 387} 388 389DISPATCH_ALWAYS_INLINE 390static inline bool 391_voucher_mach_msg_set(mach_msg_header_t *msg, voucher_t voucher) 392{ 393 if (MACH_MSGH_BITS_HAS_VOUCHER(msg->msgh_bits)) return false; 394 mach_voucher_t kv; 395 if (voucher) { 396 kv = _voucher_get_mach_voucher(voucher); 397 } else { 398 kv = _voucher_get_task_mach_voucher(); 399 } 400 return _voucher_mach_msg_set_mach_voucher(msg, kv, false); 401} 402 403DISPATCH_ALWAYS_INLINE 404static inline mach_voucher_t 405_voucher_mach_msg_get(mach_msg_header_t *msg) 406{ 407 if (!MACH_MSGH_BITS_HAS_VOUCHER(msg->msgh_bits)) return MACH_VOUCHER_NULL; 408 mach_voucher_t kv = msg->msgh_voucher_port; 409 msg->msgh_voucher_port = MACH_VOUCHER_NULL; 410 msg->msgh_bits &= (mach_msg_bits_t)~MACH_MSGH_BITS_VOUCHER_MASK; 411 return kv; 412} 413 414DISPATCH_ALWAYS_INLINE 415static inline mach_voucher_t 416_voucher_mach_msg_clear(mach_msg_header_t *msg, bool move_send) 417{ 418 mach_msg_bits_t kvbits = MACH_MSGH_BITS_VOUCHER(msg->msgh_bits); 419 mach_voucher_t kv = msg->msgh_voucher_port, kvm = MACH_VOUCHER_NULL; 420 if ((kvbits == MACH_MSG_TYPE_COPY_SEND || 421 kvbits == MACH_MSG_TYPE_MOVE_SEND) && kv) { 422 _dispatch_kvoucher_debug("msg[%p] clear %s", kv, msg, move_send ? 423 "move-send" : "copy-send"); 424 _dispatch_voucher_debug_machport(kv); 425 if (kvbits == MACH_MSG_TYPE_MOVE_SEND) { 426 // <rdar://problem/15694142> return/drop received or pseudo-received 427 // voucher reference (e.g. due to send failure). 428 if (move_send) { 429 kvm = kv; 430 } else { 431 _voucher_dealloc_mach_voucher(kv); 432 } 433 } 434 msg->msgh_voucher_port = MACH_VOUCHER_NULL; 435 msg->msgh_bits &= (mach_msg_bits_t)~MACH_MSGH_BITS_VOUCHER_MASK; 436 } 437 return kvm; 438} 439 440#pragma mark - 441#pragma mark dispatch_continuation_t + voucher_t 442 443#if DISPATCH_USE_KDEBUG_TRACE 444DISPATCH_ALWAYS_INLINE 445static inline void 446_dispatch_voucher_ktrace(int code, natural_t voucher, void *container) 447{ 448 if (!voucher) return; 449 __kdebug_trace(APPSDBG_CODE(DBG_MACH_CHUD, (0xfac >> 2)) | DBG_FUNC_NONE, 450 code, (int)voucher, (int)(uintptr_t)container, 451#ifdef __LP64__ 452 (int)((uintptr_t)container >> 32) 453#else 454 0 455#endif 456 ); 457} 458#define _dispatch_voucher_ktrace_dc_push(dc) \ 459 _dispatch_voucher_ktrace(0x1, (dc)->dc_voucher ? \ 460 (dc)->dc_voucher->v_kvoucher : MACH_VOUCHER_NULL, (dc)) 461#define _dispatch_voucher_ktrace_dc_pop(dc) \ 462 _dispatch_voucher_ktrace(0x2, (dc)->dc_voucher ? \ 463 (dc)->dc_voucher->v_kvoucher : MACH_VOUCHER_NULL, (dc)) 464#define _dispatch_voucher_ktrace_dmsg_push(dmsg) \ 465 _dispatch_voucher_ktrace(0x3, (dmsg)->dmsg_voucher ? \ 466 (dmsg)->dmsg_voucher->v_kvoucher : MACH_VOUCHER_NULL, (dmsg)) 467#define _dispatch_voucher_ktrace_dmsg_pop(dmsg) \ 468 _dispatch_voucher_ktrace(0x4, (dmsg)->dmsg_voucher ? \ 469 (dmsg)->dmsg_voucher->v_kvoucher : MACH_VOUCHER_NULL, (dmsg)) 470#else 471#define _dispatch_voucher_ktrace_dc_push(dc) 472#define _dispatch_voucher_ktrace_dc_pop(dc) 473#define _dispatch_voucher_ktrace_dmsg_push(dmsg) 474#define _dispatch_voucher_ktrace_dmsg_pop(dmsg) 475#endif // DISPATCH_USE_KDEBUG_TRACE 476 477DISPATCH_ALWAYS_INLINE 478static inline void 479_dispatch_continuation_voucher_set(dispatch_continuation_t dc, 480 dispatch_block_flags_t flags) 481{ 482 unsigned long bits = (unsigned long)dc->do_vtable; 483 voucher_t v = NULL; 484 485 if (flags & DISPATCH_BLOCK_HAS_VOUCHER) { 486 bits |= DISPATCH_OBJ_HAS_VOUCHER_BIT; 487 } else if (!(flags & DISPATCH_BLOCK_NO_VOUCHER)) { 488 v = _voucher_copy(); 489 } 490 dc->do_vtable = (void*)bits; 491 dc->dc_voucher = v; 492 _dispatch_voucher_debug("continuation[%p] set", dc->dc_voucher, dc); 493 _dispatch_voucher_ktrace_dc_push(dc); 494} 495 496DISPATCH_ALWAYS_INLINE 497static inline void 498_dispatch_continuation_voucher_adopt(dispatch_continuation_t dc) 499{ 500 unsigned long bits = (unsigned long)dc->do_vtable; 501 voucher_t v = DISPATCH_NO_VOUCHER; 502 if (!(bits & DISPATCH_OBJ_HAS_VOUCHER_BIT)) { 503 _dispatch_voucher_ktrace_dc_pop(dc); 504 _dispatch_voucher_debug("continuation[%p] adopt", dc->dc_voucher, dc); 505 v = dc->dc_voucher; 506 dc->dc_voucher = NULL; 507 } 508 _dispatch_adopt_priority_and_replace_voucher(dc->dc_priority, v, 0); 509} 510 511#pragma mark - 512#pragma mark _voucher_activity_heap 513 514typedef uint32_t _voucher_atm_subid_t; 515static const size_t _voucher_activity_hash_bits = 6; 516static const size_t _voucher_activity_hash_size = 517 1 << _voucher_activity_hash_bits; 518#define VACTID_HASH(x) ((((uint32_t)((x) >> 32) + (uint32_t)(x)) * \ 519 2654435761u) >> (32-_voucher_activity_hash_bits)) 520#define VATMID_HASH(x) \ 521 (((uint32_t)(x) * 2654435761u) >> (32-_voucher_activity_hash_bits)) 522#define VATMID2ACTID(x) ((uint64_t)(x) << 32) 523#define VACTID_BASEID(x) ((uint64_t)(x) & (((uint64_t)UINT32_MAX) << 32)) 524#define VACTID_SUBID(x) ((uint32_t)(x)) 525#define VATM_ACTID(vatm, subid) (VATMID2ACTID((vatm)->vatm_id) + (subid)) 526#define VATM_SUBID_BITS2MAX(bits) ((1u << (bits)) - 1) 527#define VATM_SUBID_MAXBITS (32) 528#define VATM_SUBID_MAX (ATM_SUBAID32_MAX) 529#define MAILBOX_OFFSET_UNSET UINT64_MAX 530 531static const size_t _voucher_activity_buffers_per_heap = 512; 532typedef unsigned long _voucher_activity_bitmap_base_t; 533static const size_t _voucher_activity_bits_per_bitmap_base_t = 534 8 * sizeof(_voucher_activity_bitmap_base_t); 535static const size_t _voucher_activity_bitmaps_per_heap = 536 _voucher_activity_buffers_per_heap / 537 _voucher_activity_bits_per_bitmap_base_t; 538typedef _voucher_activity_bitmap_base_t 539 _voucher_activity_bitmap_t[_voucher_activity_bitmaps_per_heap]; 540 541typedef struct _voucher_activity_metadata_s { 542 _voucher_activity_buffer_t vam_kernel_metadata; 543 _voucher_activity_buffer_t vam_client_metadata; 544 struct _voucher_activity_self_metadata_s vam_self_metadata; 545#if __LP64__ 546 uintptr_t vam_pad0[7]; 547#else 548 uintptr_t vam_pad0[15]; 549#endif 550 // cacheline 551 _voucher_activity_bitmap_t volatile vam_atm_mbox_bitmap; 552 _voucher_activity_bitmap_t volatile vam_buffer_bitmap; 553 _voucher_activity_bitmap_t volatile vam_pressure_locked_bitmap; 554 // cacheline 555 _voucher_atm_subid_t vam_base_atm_subid; 556 _voucher_atm_subid_t vam_base_atm_subid_max; 557 _voucher_atm_subid_t vam_nested_atm_subid; 558 _voucher_atm_t vam_default_activity_atm; 559 _voucher_atm_t volatile vam_base_atm; 560 voucher_activity_id_t volatile vam_nested_atm_id; 561#if __LP64__ 562 uintptr_t vam_pad2[3]; 563#else 564 uintptr_t vam_pad2[1]; 565#endif 566 _voucher_activity_lock_s vam_base_atm_lock; 567 _voucher_activity_lock_s vam_nested_atm_lock; 568 _voucher_activity_lock_s vam_atms_lock; 569 _voucher_activity_lock_s vam_activities_lock; 570 // cacheline 571 TAILQ_HEAD(, _voucher_atm_s) vam_atms[_voucher_activity_hash_size]; 572 TAILQ_HEAD(, _voucher_activity_s) 573 vam_activities[_voucher_activity_hash_size]; 574} *_voucher_activity_metadata_t; 575 576#pragma mark - 577#pragma mark _voucher_activity_t 578 579_voucher_activity_tracepoint_t _voucher_activity_tracepoint_get_slow( 580 unsigned int slots); 581extern _voucher_activity_t _voucher_activity_default; 582extern voucher_activity_mode_t _voucher_activity_mode; 583 584#if DISPATCH_DEBUG && DISPATCH_VOUCHER_ACTIVITY_DEBUG 585#define _dispatch_voucher_activity_debug(msg, act, ...) \ 586 _dispatch_debug("activity[%p] <0x%x>: atm[%p] <%lld>: " msg, (act), \ 587 (act) ? VACTID_SUBID((act)->va_id) : 0, (act) ? (act)->va_atm : NULL, \ 588 (act) && (act)->va_atm ? (act)->va_atm->vatm_id : 0, ##__VA_ARGS__) 589#define _dispatch_voucher_atm_debug(msg, atm, ...) \ 590 _dispatch_debug("atm[%p] <%lld> kvoucher[0x%08x]: " msg, (atm), \ 591 (atm) ? (atm)->vatm_id : 0, (atm) ? (atm)->vatm_kvoucher : 0, \ 592 ##__VA_ARGS__) 593#else 594#define _dispatch_voucher_activity_debug(msg, act, ...) 595#define _dispatch_voucher_atm_debug(msg, atm, ...) 596#endif 597 598DISPATCH_ALWAYS_INLINE 599static inline uint64_t 600_voucher_activity_timestamp(void) 601{ 602#if TARGET_IPHONE_SIMULATOR && \ 603 IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED < 101000 604 return mach_absolute_time(); 605#else 606 return mach_approximate_time(); 607#endif 608} 609 610DISPATCH_ALWAYS_INLINE 611static inline uint64_t 612_voucher_activity_thread_id(void) 613{ 614 uint64_t thread_id; 615 pthread_threadid_np(NULL, &thread_id); // TODO: 15923074: use TSD thread_id 616 return thread_id; 617} 618 619DISPATCH_ALWAYS_INLINE 620static inline _voucher_activity_tracepoint_t 621_voucher_activity_buffer_tracepoint_get(_voucher_activity_buffer_header_t vab, 622 unsigned int slots) 623{ 624 uint32_t idx = dispatch_atomic_add2o(vab, vabh_next_tracepoint_idx, 625 slots, relaxed); 626 if (idx <= _voucher_activity_tracepoints_per_buffer) { 627 return (_voucher_activity_tracepoint_t)vab + (idx - slots); 628 } 629 return NULL; 630} 631 632DISPATCH_ALWAYS_INLINE 633static inline _voucher_activity_tracepoint_t 634_voucher_activity_tracepoint_get_from_activity(_voucher_activity_t va, 635 unsigned int slots) 636{ 637 _voucher_activity_buffer_header_t vab = va ? va->va_current_buffer : NULL; 638 return vab ? _voucher_activity_buffer_tracepoint_get(vab, slots) : NULL; 639} 640 641DISPATCH_ALWAYS_INLINE 642static inline _voucher_activity_tracepoint_t 643_voucher_activity_tracepoint_get(unsigned int slots) 644{ 645 _voucher_activity_t va; 646 voucher_t v = _voucher_get(); 647 va = v && v->v_activity ? v->v_activity : _voucher_activity_default; 648 return _voucher_activity_tracepoint_get_from_activity(va, slots); 649} 650 651DISPATCH_ALWAYS_INLINE 652static inline uint64_t 653_voucher_activity_tracepoint_init(_voucher_activity_tracepoint_t vat, 654 uint8_t type, uint8_t code_namespace, uint32_t code, uint64_t location) 655{ 656 if (!location) location = (uint64_t)__builtin_return_address(0); 657 uint64_t timestamp = _voucher_activity_timestamp(); 658 vat->vat_flags = _voucher_activity_trace_flag_tracepoint, 659 vat->vat_type = type, 660 vat->vat_namespace = code_namespace, 661 vat->vat_code = code, 662 vat->vat_timestamp = timestamp, 663 vat->vat_thread = _voucher_activity_thread_id(), 664 vat->vat_location = location; 665 return timestamp; 666} 667 668DISPATCH_ALWAYS_INLINE 669static inline uint64_t 670_voucher_activity_tracepoint_init_with_id(_voucher_activity_tracepoint_t vat, 671 voucher_activity_trace_id_t trace_id, uint64_t location) 672{ 673 uint8_t type = (uint8_t)(trace_id >> _voucher_activity_trace_id_type_shift); 674 uint8_t cns = (uint8_t)(trace_id >> 675 _voucher_activity_trace_id_code_namespace_shift); 676 uint32_t code = (uint32_t)trace_id; 677 return _voucher_activity_tracepoint_init(vat, type, cns, code, location); 678} 679 680DISPATCH_ALWAYS_INLINE 681static inline bool 682_voucher_activity_trace_id_is_subtype(voucher_activity_trace_id_t trace_id, 683 uint8_t type) 684{ 685 voucher_activity_trace_id_t type_id = voucher_activity_trace_id(type, 0, 0); 686 return (trace_id & type_id) == type_id; 687} 688#define _voucher_activity_trace_id_is_subtype(trace_id, name) \ 689 _voucher_activity_trace_id_is_subtype(trace_id, \ 690 voucher_activity_tracepoint_type_ ## name) 691 692DISPATCH_ALWAYS_INLINE 693static inline bool 694_voucher_activity_trace_id_enabled(voucher_activity_trace_id_t trace_id) 695{ 696 switch (_voucher_activity_mode) { 697 case voucher_activity_mode_release: 698 return _voucher_activity_trace_id_is_subtype(trace_id, release); 699 case voucher_activity_mode_stream: 700 case voucher_activity_mode_debug: 701 return _voucher_activity_trace_id_is_subtype(trace_id, debug) || 702 _voucher_activity_trace_id_is_subtype(trace_id, release); 703 } 704 return false; 705} 706 707DISPATCH_ALWAYS_INLINE 708static inline bool 709_voucher_activity_trace_type_enabled(uint8_t type) 710{ 711 voucher_activity_trace_id_t type_id = voucher_activity_trace_id(type, 0, 0); 712 return _voucher_activity_trace_id_enabled(type_id); 713} 714 715DISPATCH_ALWAYS_INLINE 716static inline bool 717_voucher_activity_disabled(void) 718{ 719 return slowpath(_voucher_activity_mode == voucher_activity_mode_disable); 720} 721 722DISPATCH_ALWAYS_INLINE 723static inline _voucher_activity_tracepoint_t 724_voucher_activity_trace_args_inline(uint8_t type, uint8_t code_namespace, 725 uint32_t code, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, 726 uintptr_t arg4) 727{ 728 if (!_voucher_activity_trace_type_enabled(type)) return NULL; 729 _voucher_activity_tracepoint_t vat; 730 vat = _voucher_activity_tracepoint_get(1); 731 if (!vat) return NULL; 732 _voucher_activity_tracepoint_init(vat, type, code_namespace, code, 0); 733 vat->vat_flags |= _voucher_activity_trace_flag_tracepoint_args; 734 vat->vat_data[0] = arg1; 735 vat->vat_data[1] = arg2; 736 vat->vat_data[2] = arg3; 737 vat->vat_data[3] = arg4; 738 return vat; 739} 740 741DISPATCH_ALWAYS_INLINE 742static inline _voucher_activity_tracepoint_t 743_voucher_activity_trace_with_id_inline(voucher_activity_trace_id_t trace_id) 744{ 745 _voucher_activity_tracepoint_t vat = _voucher_activity_tracepoint_get(1); 746 if (!vat) return NULL; 747 _voucher_activity_tracepoint_init_with_id(vat, trace_id, 0); 748 return vat; 749} 750 751DISPATCH_ALWAYS_INLINE 752static inline _voucher_activity_tracepoint_t 753_voucher_activity_trace_with_id(voucher_activity_trace_id_t trace_id) 754{ 755 _voucher_activity_tracepoint_t vat = _voucher_activity_tracepoint_get(1); 756 if (!vat) vat = _voucher_activity_tracepoint_get_slow(1); 757 if (!vat) return NULL; 758 _voucher_activity_tracepoint_init_with_id(vat, trace_id, 0); 759 return vat; 760} 761 762DISPATCH_ALWAYS_INLINE 763static inline void 764_voucher_activity_trace_msg(voucher_t v, mach_msg_header_t *msg, uint32_t code) 765{ 766 if (!v || !v->v_activity) return; // Don't use default activity for IPC 767 const uint8_t type = voucher_activity_tracepoint_type_release; 768 const uint8_t code_namespace = _voucher_activity_tracepoint_namespace_ipc; 769 if (!_voucher_activity_trace_type_enabled(type)) return; 770 _voucher_activity_tracepoint_t vat; 771 vat = _voucher_activity_tracepoint_get_from_activity(v->v_activity, 1); 772 if (!vat) return; // TODO: slowpath ? 773 _voucher_activity_tracepoint_init(vat, type, code_namespace, code, 0); 774 vat->vat_flags |= _voucher_activity_trace_flag_libdispatch; 775#if __has_extension(c_static_assert) 776 _Static_assert(sizeof(mach_msg_header_t) <= sizeof(vat->vat_data), 777 "mach_msg_header_t too large"); 778#endif 779 memcpy(vat->vat_data, msg, sizeof(mach_msg_header_t)); 780} 781#define _voucher_activity_trace_msg(v, msg, type) \ 782 _voucher_activity_trace_msg(v, msg, \ 783 _voucher_activity_tracepoint_namespace_ipc_ ## type) 784 785#endif // !(USE_OBJC && __OBJC2__) 786 787#else // VOUCHER_USE_MACH_VOUCHER 788 789#pragma mark - 790#pragma mark Simulator / vouchers disabled 791 792#define _dispatch_voucher_debug(msg, v, ...) 793#define _dispatch_kvoucher_debug(msg, kv, ...) 794 795DISPATCH_ALWAYS_INLINE 796static inline voucher_t 797_voucher_retain(voucher_t voucher) 798{ 799 return voucher; 800} 801 802DISPATCH_ALWAYS_INLINE 803static inline void 804_voucher_release(voucher_t voucher) 805{ 806 (void)voucher; 807} 808 809DISPATCH_ALWAYS_INLINE 810static inline voucher_t 811_voucher_get(void) 812{ 813 return NULL; 814} 815 816DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT 817static inline voucher_t 818_voucher_copy(void) 819{ 820 return NULL; 821} 822 823DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT 824static inline voucher_t 825_voucher_copy_without_importance(void) 826{ 827 return NULL; 828} 829 830DISPATCH_ALWAYS_INLINE 831static inline mach_voucher_t 832_voucher_swap_and_get_mach_voucher(voucher_t ov, voucher_t voucher) 833{ 834 (void)ov; (void)voucher; 835 return MACH_VOUCHER_NULL; 836} 837 838DISPATCH_ALWAYS_INLINE 839static inline voucher_t 840_voucher_adopt(voucher_t voucher) 841{ 842 return voucher; 843} 844 845DISPATCH_ALWAYS_INLINE 846static inline void 847_voucher_replace(voucher_t voucher) 848{ 849 (void)voucher; 850} 851 852DISPATCH_ALWAYS_INLINE 853static inline void 854_voucher_clear(void) 855{ 856} 857 858DISPATCH_ALWAYS_INLINE 859static inline pthread_priority_t 860_voucher_get_priority(voucher_t voucher) 861{ 862 (void)voucher; 863 return 0; 864} 865 866DISPATCH_ALWAYS_INLINE 867static inline bool 868_voucher_mach_msg_set_mach_voucher(mach_msg_header_t *msg, mach_voucher_t kv, 869 bool move_send) 870{ 871 (void)msg; (void)kv; (void)move_send; 872 return false; 873 874} 875 876DISPATCH_ALWAYS_INLINE 877static inline bool 878_voucher_mach_msg_set(mach_msg_header_t *msg, voucher_t voucher) 879{ 880 (void)msg; (void)voucher; 881 return false; 882} 883 884DISPATCH_ALWAYS_INLINE 885static inline mach_voucher_t 886_voucher_mach_msg_get(mach_msg_header_t *msg) 887{ 888 (void)msg; 889 return 0; 890} 891 892DISPATCH_ALWAYS_INLINE 893static inline mach_voucher_t 894_voucher_mach_msg_clear(mach_msg_header_t *msg, bool move_send) 895{ 896 (void)msg; (void)move_send; 897 return MACH_VOUCHER_NULL; 898} 899 900#define _dispatch_voucher_ktrace_dmsg_push(dmsg) 901#define _dispatch_voucher_ktrace_dmsg_pop(dmsg) 902 903DISPATCH_ALWAYS_INLINE 904static inline void 905_dispatch_continuation_voucher_set(dispatch_continuation_t dc, 906 dispatch_block_flags_t flags) 907{ 908 (void)dc; (void)flags; 909} 910 911DISPATCH_ALWAYS_INLINE 912static inline void 913_dispatch_continuation_voucher_adopt(dispatch_continuation_t dc) 914{ 915 (void)dc; 916} 917 918#define _voucher_activity_trace_msg(v, msg, type) 919 920DISPATCH_ALWAYS_INLINE 921static inline bool 922_voucher_activity_disabled(void) 923{ 924 return true; 925} 926 927#endif // VOUCHER_USE_MACH_VOUCHER 928 929#endif /* __DISPATCH_VOUCHER_INTERNAL__ */ 930