1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22/* 23 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27#ifndef _SYS_DTRACE_IMPL_H 28#define _SYS_DTRACE_IMPL_H 29 30/* #pragma ident "@(#)dtrace_impl.h 1.23 07/02/16 SMI" */ 31 32#ifdef __cplusplus 33extern "C" { 34#endif 35 36/* 37 * DTrace Dynamic Tracing Software: Kernel Implementation Interfaces 38 * 39 * Note: The contents of this file are private to the implementation of the 40 * Solaris system and DTrace subsystem and are subject to change at any time 41 * without notice. Applications and drivers using these interfaces will fail 42 * to run on future releases. These interfaces should not be used for any 43 * purpose except those expressly outlined in dtrace(7D) and libdtrace(3LIB). 44 * Please refer to the "Solaris Dynamic Tracing Guide" for more information. 45 */ 46 47#include <sys/dtrace.h> 48 49/* 50 * DTrace Implementation Constants and Typedefs 51 */ 52#define DTRACE_MAXPROPLEN 128 53#define DTRACE_DYNVAR_CHUNKSIZE 256 54 55struct dtrace_probe; 56struct dtrace_ecb; 57struct dtrace_predicate; 58struct dtrace_action; 59struct dtrace_provider; 60struct dtrace_state; 61 62typedef struct dtrace_probe dtrace_probe_t; 63typedef struct dtrace_ecb dtrace_ecb_t; 64typedef struct dtrace_predicate dtrace_predicate_t; 65typedef struct dtrace_action dtrace_action_t; 66typedef struct dtrace_provider dtrace_provider_t; 67typedef struct dtrace_meta dtrace_meta_t; 68typedef struct dtrace_state dtrace_state_t; 69typedef uint32_t dtrace_optid_t; 70typedef uint32_t dtrace_specid_t; 71typedef uint64_t dtrace_genid_t; 72 73/* 74 * DTrace Probes 75 * 76 * The probe is the fundamental unit of the DTrace architecture. Probes are 77 * created by DTrace providers, and managed by the DTrace framework. A probe 78 * is identified by a unique <provider, module, function, name> tuple, and has 79 * a unique probe identifier assigned to it. (Some probes are not associated 80 * with a specific point in text; these are called _unanchored probes_ and have 81 * no module or function associated with them.) Probes are represented as a 82 * dtrace_probe structure. To allow quick lookups based on each element of the 83 * probe tuple, probes are hashed by each of provider, module, function and 84 * name. (If a lookup is performed based on a regular expression, a 85 * dtrace_probekey is prepared, and a linear search is performed.) Each probe 86 * is additionally pointed to by a linear array indexed by its identifier. The 87 * identifier is the provider's mechanism for indicating to the DTrace 88 * framework that a probe has fired: the identifier is passed as the first 89 * argument to dtrace_probe(), where it is then mapped into the corresponding 90 * dtrace_probe structure. From the dtrace_probe structure, dtrace_probe() can 91 * iterate over the probe's list of enabling control blocks; see "DTrace 92 * Enabling Control Blocks", below.) 93 */ 94struct dtrace_probe { 95 dtrace_id_t dtpr_id; /* probe identifier */ 96 dtrace_ecb_t *dtpr_ecb; /* ECB list; see below */ 97 dtrace_ecb_t *dtpr_ecb_last; /* last ECB in list */ 98 void *dtpr_arg; /* provider argument */ 99 dtrace_cacheid_t dtpr_predcache; /* predicate cache ID */ 100 int dtpr_aframes; /* artificial frames */ 101 dtrace_provider_t *dtpr_provider; /* pointer to provider */ 102 char *dtpr_mod; /* probe's module name */ 103 char *dtpr_func; /* probe's function name */ 104 char *dtpr_name; /* probe's name */ 105 dtrace_probe_t *dtpr_nextmod; /* next in module hash */ 106 dtrace_probe_t *dtpr_prevmod; /* previous in module hash */ 107 dtrace_probe_t *dtpr_nextfunc; /* next in function hash */ 108 dtrace_probe_t *dtpr_prevfunc; /* previous in function hash */ 109 dtrace_probe_t *dtpr_nextname; /* next in name hash */ 110 dtrace_probe_t *dtpr_prevname; /* previous in name hash */ 111 dtrace_genid_t dtpr_gen; /* probe generation ID */ 112}; 113 114typedef int dtrace_probekey_f(const char *, const char *, int); 115 116typedef struct dtrace_probekey { 117 const char *dtpk_prov; /* provider name to match */ 118 dtrace_probekey_f *dtpk_pmatch; /* provider matching function */ 119 const char *dtpk_mod; /* module name to match */ 120 dtrace_probekey_f *dtpk_mmatch; /* module matching function */ 121 const char *dtpk_func; /* func name to match */ 122 dtrace_probekey_f *dtpk_fmatch; /* func matching function */ 123 const char *dtpk_name; /* name to match */ 124 dtrace_probekey_f *dtpk_nmatch; /* name matching function */ 125 dtrace_id_t dtpk_id; /* identifier to match */ 126} dtrace_probekey_t; 127 128typedef struct dtrace_hashbucket { 129 struct dtrace_hashbucket *dthb_next; /* next on hash chain */ 130 dtrace_probe_t *dthb_chain; /* chain of probes */ 131 int dthb_len; /* number of probes here */ 132} dtrace_hashbucket_t; 133 134typedef struct dtrace_hash { 135 dtrace_hashbucket_t **dth_tab; /* hash table */ 136 int dth_size; /* size of hash table */ 137 int dth_mask; /* mask to index into table */ 138 int dth_nbuckets; /* total number of buckets */ 139 uintptr_t dth_nextoffs; /* offset of next in probe */ 140 uintptr_t dth_prevoffs; /* offset of prev in probe */ 141 uintptr_t dth_stroffs; /* offset of str in probe */ 142} dtrace_hash_t; 143 144/* 145 * DTrace Enabling Control Blocks 146 * 147 * When a provider wishes to fire a probe, it calls into dtrace_probe(), 148 * passing the probe identifier as the first argument. As described above, 149 * dtrace_probe() maps the identifier into a pointer to a dtrace_probe_t 150 * structure. This structure contains information about the probe, and a 151 * pointer to the list of Enabling Control Blocks (ECBs). Each ECB points to 152 * DTrace consumer state, and contains an optional predicate, and a list of 153 * actions. (Shown schematically below.) The ECB abstraction allows a single 154 * probe to be multiplexed across disjoint consumers, or across disjoint 155 * enablings of a single probe within one consumer. 156 * 157 * Enabling Control Block 158 * dtrace_ecb_t 159 * +------------------------+ 160 * | dtrace_epid_t ---------+--------------> Enabled Probe ID (EPID) 161 * | dtrace_state_t * ------+--------------> State associated with this ECB 162 * | dtrace_predicate_t * --+---------+ 163 * | dtrace_action_t * -----+----+ | 164 * | dtrace_ecb_t * ---+ | | | Predicate (if any) 165 * +-------------------+----+ | | dtrace_predicate_t 166 * | | +---> +--------------------+ 167 * | | | dtrace_difo_t * ---+----> DIFO 168 * | | +--------------------+ 169 * | | 170 * Next ECB | | Action 171 * (if any) | | dtrace_action_t 172 * : +--> +-------------------+ 173 * : | dtrace_actkind_t -+------> kind 174 * v | dtrace_difo_t * --+------> DIFO (if any) 175 * | dtrace_recdesc_t -+------> record descr. 176 * | dtrace_action_t * +------+ 177 * +-------------------+ | 178 * | Next action 179 * +-------------------------------+ (if any) 180 * | 181 * | Action 182 * | dtrace_action_t 183 * +--> +-------------------+ 184 * | dtrace_actkind_t -+------> kind 185 * | dtrace_difo_t * --+------> DIFO (if any) 186 * | dtrace_action_t * +------+ 187 * +-------------------+ | 188 * | Next action 189 * +-------------------------------+ (if any) 190 * | 191 * : 192 * v 193 * 194 * 195 * dtrace_probe() iterates over the ECB list. If the ECB needs less space 196 * than is available in the principal buffer, the ECB is processed: if the 197 * predicate is non-NULL, the DIF object is executed. If the result is 198 * non-zero, the action list is processed, with each action being executed 199 * accordingly. When the action list has been completely executed, processing 200 * advances to the next ECB. processing advances to the next ECB. If the 201 * result is non-zero; For each ECB, it first determines the The ECB 202 * abstraction allows disjoint consumers to multiplex on single probes. 203 */ 204struct dtrace_ecb { 205 dtrace_epid_t dte_epid; /* enabled probe ID */ 206 uint32_t dte_alignment; /* required alignment */ 207 size_t dte_needed; /* bytes needed */ 208 size_t dte_size; /* total size of payload */ 209 dtrace_predicate_t *dte_predicate; /* predicate, if any */ 210 dtrace_action_t *dte_action; /* actions, if any */ 211 dtrace_ecb_t *dte_next; /* next ECB on probe */ 212 dtrace_state_t *dte_state; /* pointer to state */ 213 uint32_t dte_cond; /* security condition */ 214 dtrace_probe_t *dte_probe; /* pointer to probe */ 215 dtrace_action_t *dte_action_last; /* last action on ECB */ 216 uint64_t dte_uarg; /* library argument */ 217}; 218 219struct dtrace_predicate { 220 dtrace_difo_t *dtp_difo; /* DIF object */ 221 dtrace_cacheid_t dtp_cacheid; /* cache identifier */ 222 int dtp_refcnt; /* reference count */ 223}; 224 225struct dtrace_action { 226 dtrace_actkind_t dta_kind; /* kind of action */ 227 uint16_t dta_intuple; /* boolean: in aggregation */ 228 uint32_t dta_refcnt; /* reference count */ 229 dtrace_difo_t *dta_difo; /* pointer to DIFO */ 230 dtrace_recdesc_t dta_rec; /* record description */ 231 dtrace_action_t *dta_prev; /* previous action */ 232 dtrace_action_t *dta_next; /* next action */ 233}; 234 235typedef struct dtrace_aggregation { 236 dtrace_action_t dtag_action; /* action; must be first */ 237 dtrace_aggid_t dtag_id; /* identifier */ 238 dtrace_ecb_t *dtag_ecb; /* corresponding ECB */ 239 dtrace_action_t *dtag_first; /* first action in tuple */ 240 uint32_t dtag_base; /* base of aggregation */ 241 uint8_t dtag_hasarg; /* boolean: has argument */ 242 uint64_t dtag_initial; /* initial value */ 243 void (*dtag_aggregate)(uint64_t *, uint64_t, uint64_t); 244} dtrace_aggregation_t; 245 246/* 247 * DTrace Buffers 248 * 249 * Principal buffers, aggregation buffers, and speculative buffers are all 250 * managed with the dtrace_buffer structure. By default, this structure 251 * includes twin data buffers -- dtb_tomax and dtb_xamot -- that serve as the 252 * active and passive buffers, respectively. For speculative buffers, 253 * dtb_xamot will be NULL; for "ring" and "fill" buffers, dtb_xamot will point 254 * to a scratch buffer. For all buffer types, the dtrace_buffer structure is 255 * always allocated on a per-CPU basis; a single dtrace_buffer structure is 256 * never shared among CPUs. (That is, there is never true sharing of the 257 * dtrace_buffer structure; to prevent false sharing of the structure, it must 258 * always be aligned to the coherence granularity -- generally 64 bytes.) 259 * 260 * One of the critical design decisions of DTrace is that a given ECB always 261 * stores the same quantity and type of data. This is done to assure that the 262 * only metadata required for an ECB's traced data is the EPID. That is, from 263 * the EPID, the consumer can determine the data layout. (The data buffer 264 * layout is shown schematically below.) By assuring that one can determine 265 * data layout from the EPID, the metadata stream can be separated from the 266 * data stream -- simplifying the data stream enormously. 267 * 268 * base of data buffer ---> +------+--------------------+------+ 269 * | EPID | data | EPID | 270 * +------+--------+------+----+------+ 271 * | data | EPID | data | 272 * +---------------+------+-----------+ 273 * | data, cont. | 274 * +------+--------------------+------+ 275 * | EPID | data | | 276 * +------+--------------------+ | 277 * | || | 278 * | || | 279 * | \/ | 280 * : : 281 * . . 282 * . . 283 * . . 284 * : : 285 * | | 286 * limit of data buffer ---> +----------------------------------+ 287 * 288 * When evaluating an ECB, dtrace_probe() determines if the ECB's needs of the 289 * principal buffer (both scratch and payload) exceed the available space. If 290 * the ECB's needs exceed available space (and if the principal buffer policy 291 * is the default "switch" policy), the ECB is dropped, the buffer's drop count 292 * is incremented, and processing advances to the next ECB. If the ECB's needs 293 * can be met with the available space, the ECB is processed, but the offset in 294 * the principal buffer is only advanced if the ECB completes processing 295 * without error. 296 * 297 * When a buffer is to be switched (either because the buffer is the principal 298 * buffer with a "switch" policy or because it is an aggregation buffer), a 299 * cross call is issued to the CPU associated with the buffer. In the cross 300 * call context, interrupts are disabled, and the active and the inactive 301 * buffers are atomically switched. This involves switching the data pointers, 302 * copying the various state fields (offset, drops, errors, etc.) into their 303 * inactive equivalents, and clearing the state fields. Because interrupts are 304 * disabled during this procedure, the switch is guaranteed to appear atomic to 305 * dtrace_probe(). 306 * 307 * DTrace Ring Buffering 308 * 309 * To process a ring buffer correctly, one must know the oldest valid record. 310 * Processing starts at the oldest record in the buffer and continues until 311 * the end of the buffer is reached. Processing then resumes starting with 312 * the record stored at offset 0 in the buffer, and continues until the 313 * youngest record is processed. If trace records are of a fixed-length, 314 * determining the oldest record is trivial: 315 * 316 * - If the ring buffer has not wrapped, the oldest record is the record 317 * stored at offset 0. 318 * 319 * - If the ring buffer has wrapped, the oldest record is the record stored 320 * at the current offset. 321 * 322 * With variable length records, however, just knowing the current offset 323 * doesn't suffice for determining the oldest valid record: assuming that one 324 * allows for arbitrary data, one has no way of searching forward from the 325 * current offset to find the oldest valid record. (That is, one has no way 326 * of separating data from metadata.) It would be possible to simply refuse to 327 * process any data in the ring buffer between the current offset and the 328 * limit, but this leaves (potentially) an enormous amount of otherwise valid 329 * data unprocessed. 330 * 331 * To effect ring buffering, we track two offsets in the buffer: the current 332 * offset and the _wrapped_ offset. If a request is made to reserve some 333 * amount of data, and the buffer has wrapped, the wrapped offset is 334 * incremented until the wrapped offset minus the current offset is greater 335 * than or equal to the reserve request. This is done by repeatedly looking 336 * up the ECB corresponding to the EPID at the current wrapped offset, and 337 * incrementing the wrapped offset by the size of the data payload 338 * corresponding to that ECB. If this offset is greater than or equal to the 339 * limit of the data buffer, the wrapped offset is set to 0. Thus, the 340 * current offset effectively "chases" the wrapped offset around the buffer. 341 * Schematically: 342 * 343 * base of data buffer ---> +------+--------------------+------+ 344 * | EPID | data | EPID | 345 * +------+--------+------+----+------+ 346 * | data | EPID | data | 347 * +---------------+------+-----------+ 348 * | data, cont. | 349 * +------+---------------------------+ 350 * | EPID | data | 351 * current offset ---> +------+---------------------------+ 352 * | invalid data | 353 * wrapped offset ---> +------+--------------------+------+ 354 * | EPID | data | EPID | 355 * +------+--------+------+----+------+ 356 * | data | EPID | data | 357 * +---------------+------+-----------+ 358 * : : 359 * . . 360 * . ... valid data ... . 361 * . . 362 * : : 363 * +------+-------------+------+------+ 364 * | EPID | data | EPID | data | 365 * +------+------------++------+------+ 366 * | data, cont. | leftover | 367 * limit of data buffer ---> +-------------------+--------------+ 368 * 369 * If the amount of requested buffer space exceeds the amount of space 370 * available between the current offset and the end of the buffer: 371 * 372 * (1) all words in the data buffer between the current offset and the limit 373 * of the data buffer (marked "leftover", above) are set to 374 * DTRACE_EPIDNONE 375 * 376 * (2) the wrapped offset is set to zero 377 * 378 * (3) the iteration process described above occurs until the wrapped offset 379 * is greater than the amount of desired space. 380 * 381 * The wrapped offset is implemented by (re-)using the inactive offset. 382 * In a "switch" buffer policy, the inactive offset stores the offset in 383 * the inactive buffer; in a "ring" buffer policy, it stores the wrapped 384 * offset. 385 * 386 * DTrace Scratch Buffering 387 * 388 * Some ECBs may wish to allocate dynamically-sized temporary scratch memory. 389 * To accommodate such requests easily, scratch memory may be allocated in 390 * the buffer beyond the current offset plus the needed memory of the current 391 * ECB. If there isn't sufficient room in the buffer for the requested amount 392 * of scratch space, the allocation fails and an error is generated. Scratch 393 * memory is tracked in the dtrace_mstate_t and is automatically freed when 394 * the ECB ceases processing. Note that ring buffers cannot allocate their 395 * scratch from the principal buffer -- lest they needlessly overwrite older, 396 * valid data. Ring buffers therefore have their own dedicated scratch buffer 397 * from which scratch is allocated. 398 */ 399#define DTRACEBUF_RING 0x0001 /* bufpolicy set to "ring" */ 400#define DTRACEBUF_FILL 0x0002 /* bufpolicy set to "fill" */ 401#define DTRACEBUF_NOSWITCH 0x0004 /* do not switch buffer */ 402#define DTRACEBUF_WRAPPED 0x0008 /* ring buffer has wrapped */ 403#define DTRACEBUF_DROPPED 0x0010 /* drops occurred */ 404#define DTRACEBUF_ERROR 0x0020 /* errors occurred */ 405#define DTRACEBUF_FULL 0x0040 /* "fill" buffer is full */ 406#define DTRACEBUF_CONSUMED 0x0080 /* buffer has been consumed */ 407#define DTRACEBUF_INACTIVE 0x0100 /* buffer is not yet active */ 408 409typedef struct dtrace_buffer { 410 uint64_t dtb_offset; /* current offset in buffer */ 411 uint64_t dtb_size; /* size of buffer */ 412 uint32_t dtb_flags; /* flags */ 413 uint32_t dtb_drops; /* number of drops */ 414 caddr_t dtb_tomax; /* active buffer */ 415 caddr_t dtb_xamot; /* inactive buffer */ 416 uint32_t dtb_xamot_flags; /* inactive flags */ 417 uint32_t dtb_xamot_drops; /* drops in inactive buffer */ 418 uint64_t dtb_xamot_offset; /* offset in inactive buffer */ 419 uint32_t dtb_errors; /* number of errors */ 420 uint32_t dtb_xamot_errors; /* errors in inactive buffer */ 421#ifndef _LP64 422 uint64_t dtb_pad1; 423#endif 424} dtrace_buffer_t; 425 426/* 427 * DTrace Aggregation Buffers 428 * 429 * Aggregation buffers use much of the same mechanism as described above 430 * ("DTrace Buffers"). However, because an aggregation is fundamentally a 431 * hash, there exists dynamic metadata associated with an aggregation buffer 432 * that is not associated with other kinds of buffers. This aggregation 433 * metadata is _only_ relevant for the in-kernel implementation of 434 * aggregations; it is not actually relevant to user-level consumers. To do 435 * this, we allocate dynamic aggregation data (hash keys and hash buckets) 436 * starting below the _limit_ of the buffer, and we allocate data from the 437 * _base_ of the buffer. When the aggregation buffer is copied out, _only_ the 438 * data is copied out; the metadata is simply discarded. Schematically, 439 * aggregation buffers look like: 440 * 441 * base of data buffer ---> +-------+------+-----------+-------+ 442 * | aggid | key | value | aggid | 443 * +-------+------+-----------+-------+ 444 * | key | 445 * +-------+-------+-----+------------+ 446 * | value | aggid | key | value | 447 * +-------+------++-----+------+-----+ 448 * | aggid | key | value | | 449 * +-------+------+-------------+ | 450 * | || | 451 * | || | 452 * | \/ | 453 * : : 454 * . . 455 * . . 456 * . . 457 * : : 458 * | /\ | 459 * | || +------------+ 460 * | || | | 461 * +---------------------+ | 462 * | hash keys | 463 * | (dtrace_aggkey structures) | 464 * | | 465 * +----------------------------------+ 466 * | hash buckets | 467 * | (dtrace_aggbuffer structure) | 468 * | | 469 * limit of data buffer ---> +----------------------------------+ 470 * 471 * 472 * As implied above, just as we assure that ECBs always store a constant 473 * amount of data, we assure that a given aggregation -- identified by its 474 * aggregation ID -- always stores data of a constant quantity and type. 475 * As with EPIDs, this allows the aggregation ID to serve as the metadata for a 476 * given record. 477 * 478 * Note that the size of the dtrace_aggkey structure must be sizeof (uintptr_t) 479 * aligned. (If this the structure changes such that this becomes false, an 480 * assertion will fail in dtrace_aggregate().) 481 */ 482typedef struct dtrace_aggkey { 483 uint32_t dtak_hashval; /* hash value */ 484 uint32_t dtak_action:4; /* action -- 4 bits */ 485 uint32_t dtak_size:28; /* size -- 28 bits */ 486 caddr_t dtak_data; /* data pointer */ 487 struct dtrace_aggkey *dtak_next; /* next in hash chain */ 488} dtrace_aggkey_t; 489 490typedef struct dtrace_aggbuffer { 491 uintptr_t dtagb_hashsize; /* number of buckets */ 492 uintptr_t dtagb_free; /* free list of keys */ 493 dtrace_aggkey_t **dtagb_hash; /* hash table */ 494} dtrace_aggbuffer_t; 495 496/* 497 * DTrace Speculations 498 * 499 * Speculations have a per-CPU buffer and a global state. Once a speculation 500 * buffer has been comitted or discarded, it cannot be reused until all CPUs 501 * have taken the same action (commit or discard) on their respective 502 * speculative buffer. However, because DTrace probes may execute in arbitrary 503 * context, other CPUs cannot simply be cross-called at probe firing time to 504 * perform the necessary commit or discard. The speculation states thus 505 * optimize for the case that a speculative buffer is only active on one CPU at 506 * the time of a commit() or discard() -- for if this is the case, other CPUs 507 * need not take action, and the speculation is immediately available for 508 * reuse. If the speculation is active on multiple CPUs, it must be 509 * asynchronously cleaned -- potentially leading to a higher rate of dirty 510 * speculative drops. The speculation states are as follows: 511 * 512 * DTRACESPEC_INACTIVE <= Initial state; inactive speculation 513 * DTRACESPEC_ACTIVE <= Allocated, but not yet speculatively traced to 514 * DTRACESPEC_ACTIVEONE <= Speculatively traced to on one CPU 515 * DTRACESPEC_ACTIVEMANY <= Speculatively traced to on more than one CPU 516 * DTRACESPEC_COMMITTING <= Currently being commited on one CPU 517 * DTRACESPEC_COMMITTINGMANY <= Currently being commited on many CPUs 518 * DTRACESPEC_DISCARDING <= Currently being discarded on many CPUs 519 * 520 * The state transition diagram is as follows: 521 * 522 * +----------------------------------------------------------+ 523 * | | 524 * | +------------+ | 525 * | +-------------------| COMMITTING |<-----------------+ | 526 * | | +------------+ | | 527 * | | copied spec. ^ commit() on | | discard() on 528 * | | into principal | active CPU | | active CPU 529 * | | | commit() | | 530 * V V | | | 531 * +----------+ +--------+ +-----------+ 532 * | INACTIVE |---------------->| ACTIVE |--------------->| ACTIVEONE | 533 * +----------+ speculation() +--------+ speculate() +-----------+ 534 * ^ ^ | | | 535 * | | | discard() | | 536 * | | asynchronously | discard() on | | speculate() 537 * | | cleaned V inactive CPU | | on inactive 538 * | | +------------+ | | CPU 539 * | +-------------------| DISCARDING |<-----------------+ | 540 * | +------------+ | 541 * | asynchronously ^ | 542 * | copied spec. | discard() | 543 * | into principal +------------------------+ | 544 * | | V 545 * +----------------+ commit() +------------+ 546 * | COMMITTINGMANY |<----------------------------------| ACTIVEMANY | 547 * +----------------+ +------------+ 548 */ 549typedef enum dtrace_speculation_state { 550 DTRACESPEC_INACTIVE = 0, 551 DTRACESPEC_ACTIVE, 552 DTRACESPEC_ACTIVEONE, 553 DTRACESPEC_ACTIVEMANY, 554 DTRACESPEC_COMMITTING, 555 DTRACESPEC_COMMITTINGMANY, 556 DTRACESPEC_DISCARDING 557} dtrace_speculation_state_t; 558 559typedef struct dtrace_speculation { 560 dtrace_speculation_state_t dtsp_state; /* current speculation state */ 561 int dtsp_cleaning; /* non-zero if being cleaned */ 562 dtrace_buffer_t *dtsp_buffer; /* speculative buffer */ 563} dtrace_speculation_t; 564 565/* 566 * DTrace Dynamic Variables 567 * 568 * The dynamic variable problem is obviously decomposed into two subproblems: 569 * allocating new dynamic storage, and freeing old dynamic storage. The 570 * presence of the second problem makes the first much more complicated -- or 571 * rather, the absence of the second renders the first trivial. This is the 572 * case with aggregations, for which there is effectively no deallocation of 573 * dynamic storage. (Or more accurately, all dynamic storage is deallocated 574 * when a snapshot is taken of the aggregation.) As DTrace dynamic variables 575 * allow for both dynamic allocation and dynamic deallocation, the 576 * implementation of dynamic variables is quite a bit more complicated than 577 * that of their aggregation kin. 578 * 579 * We observe that allocating new dynamic storage is tricky only because the 580 * size can vary -- the allocation problem is much easier if allocation sizes 581 * are uniform. We further observe that in D, the size of dynamic variables is 582 * actually _not_ dynamic -- dynamic variable sizes may be determined by static 583 * analysis of DIF text. (This is true even of putatively dynamically-sized 584 * objects like strings and stacks, the sizes of which are dictated by the 585 * "stringsize" and "stackframes" variables, respectively.) We exploit this by 586 * performing this analysis on all DIF before enabling any probes. For each 587 * dynamic load or store, we calculate the dynamically-allocated size plus the 588 * size of the dtrace_dynvar structure plus the storage required to key the 589 * data. For all DIF, we take the largest value and dub it the _chunksize_. 590 * We then divide dynamic memory into two parts: a hash table that is wide 591 * enough to have every chunk in its own bucket, and a larger region of equal 592 * chunksize units. Whenever we wish to dynamically allocate a variable, we 593 * always allocate a single chunk of memory. Depending on the uniformity of 594 * allocation, this will waste some amount of memory -- but it eliminates the 595 * non-determinism inherent in traditional heap fragmentation. 596 * 597 * Dynamic objects are allocated by storing a non-zero value to them; they are 598 * deallocated by storing a zero value to them. Dynamic variables are 599 * complicated enormously by being shared between CPUs. In particular, 600 * consider the following scenario: 601 * 602 * CPU A CPU B 603 * +---------------------------------+ +---------------------------------+ 604 * | | | | 605 * | allocates dynamic object a[123] | | | 606 * | by storing the value 345 to it | | | 607 * | ---------> | 608 * | | | wishing to load from object | 609 * | | | a[123], performs lookup in | 610 * | | | dynamic variable space | 611 * | <--------- | 612 * | deallocates object a[123] by | | | 613 * | storing 0 to it | | | 614 * | | | | 615 * | allocates dynamic object b[567] | | performs load from a[123] | 616 * | by storing the value 789 to it | | | 617 * : : : : 618 * . . . . 619 * 620 * This is obviously a race in the D program, but there are nonetheless only 621 * two valid values for CPU B's load from a[123]: 345 or 0. Most importantly, 622 * CPU B may _not_ see the value 789 for a[123]. 623 * 624 * There are essentially two ways to deal with this: 625 * 626 * (1) Explicitly spin-lock variables. That is, if CPU B wishes to load 627 * from a[123], it needs to lock a[123] and hold the lock for the 628 * duration that it wishes to manipulate it. 629 * 630 * (2) Avoid reusing freed chunks until it is known that no CPU is referring 631 * to them. 632 * 633 * The implementation of (1) is rife with complexity, because it requires the 634 * user of a dynamic variable to explicitly decree when they are done using it. 635 * Were all variables by value, this perhaps wouldn't be debilitating -- but 636 * dynamic variables of non-scalar types are tracked by reference. That is, if 637 * a dynamic variable is, say, a string, and that variable is to be traced to, 638 * say, the principal buffer, the DIF emulation code returns to the main 639 * dtrace_probe() loop a pointer to the underlying storage, not the contents of 640 * the storage. Further, code calling on DIF emulation would have to be aware 641 * that the DIF emulation has returned a reference to a dynamic variable that 642 * has been potentially locked. The variable would have to be unlocked after 643 * the main dtrace_probe() loop is finished with the variable, and the main 644 * dtrace_probe() loop would have to be careful to not call any further DIF 645 * emulation while the variable is locked to avoid deadlock. More generally, 646 * if one were to implement (1), DIF emulation code dealing with dynamic 647 * variables could only deal with one dynamic variable at a time (lest deadlock 648 * result). To sum, (1) exports too much subtlety to the users of dynamic 649 * variables -- increasing maintenance burden and imposing serious constraints 650 * on future DTrace development. 651 * 652 * The implementation of (2) is also complex, but the complexity is more 653 * manageable. We need to be sure that when a variable is deallocated, it is 654 * not placed on a traditional free list, but rather on a _dirty_ list. Once a 655 * variable is on a dirty list, it cannot be found by CPUs performing a 656 * subsequent lookup of the variable -- but it may still be in use by other 657 * CPUs. To assure that all CPUs that may be seeing the old variable have 658 * cleared out of probe context, a dtrace_sync() can be issued. Once the 659 * dtrace_sync() has completed, it can be known that all CPUs are done 660 * manipulating the dynamic variable -- the dirty list can be atomically 661 * appended to the free list. Unfortunately, there's a slight hiccup in this 662 * mechanism: dtrace_sync() may not be issued from probe context. The 663 * dtrace_sync() must be therefore issued asynchronously from non-probe 664 * context. For this we rely on the DTrace cleaner, a cyclic that runs at the 665 * "cleanrate" frequency. To ease this implementation, we define several chunk 666 * lists: 667 * 668 * - Dirty. Deallocated chunks, not yet cleaned. Not available. 669 * 670 * - Rinsing. Formerly dirty chunks that are currently being asynchronously 671 * cleaned. Not available, but will be shortly. Dynamic variable 672 * allocation may not spin or block for availability, however. 673 * 674 * - Clean. Clean chunks, ready for allocation -- but not on the free list. 675 * 676 * - Free. Available for allocation. 677 * 678 * Moreover, to avoid absurd contention, _each_ of these lists is implemented 679 * on a per-CPU basis. This is only for performance, not correctness; chunks 680 * may be allocated from another CPU's free list. The algorithm for allocation 681 * then is this: 682 * 683 * (1) Attempt to atomically allocate from current CPU's free list. If list 684 * is non-empty and allocation is successful, allocation is complete. 685 * 686 * (2) If the clean list is non-empty, atomically move it to the free list, 687 * and reattempt (1). 688 * 689 * (3) If the dynamic variable space is in the CLEAN state, look for free 690 * and clean lists on other CPUs by setting the current CPU to the next 691 * CPU, and reattempting (1). If the next CPU is the current CPU (that 692 * is, if all CPUs have been checked), atomically switch the state of 693 * the dynamic variable space based on the following: 694 * 695 * - If no free chunks were found and no dirty chunks were found, 696 * atomically set the state to EMPTY. 697 * 698 * - If dirty chunks were found, atomically set the state to DIRTY. 699 * 700 * - If rinsing chunks were found, atomically set the state to RINSING. 701 * 702 * (4) Based on state of dynamic variable space state, increment appropriate 703 * counter to indicate dynamic drops (if in EMPTY state) vs. dynamic 704 * dirty drops (if in DIRTY state) vs. dynamic rinsing drops (if in 705 * RINSING state). Fail the allocation. 706 * 707 * The cleaning cyclic operates with the following algorithm: for all CPUs 708 * with a non-empty dirty list, atomically move the dirty list to the rinsing 709 * list. Perform a dtrace_sync(). For all CPUs with a non-empty rinsing list, 710 * atomically move the rinsing list to the clean list. Perform another 711 * dtrace_sync(). By this point, all CPUs have seen the new clean list; the 712 * state of the dynamic variable space can be restored to CLEAN. 713 * 714 * There exist two final races that merit explanation. The first is a simple 715 * allocation race: 716 * 717 * CPU A CPU B 718 * +---------------------------------+ +---------------------------------+ 719 * | | | | 720 * | allocates dynamic object a[123] | | allocates dynamic object a[123] | 721 * | by storing the value 345 to it | | by storing the value 567 to it | 722 * | | | | 723 * : : : : 724 * . . . . 725 * 726 * Again, this is a race in the D program. It can be resolved by having a[123] 727 * hold the value 345 or a[123] hold the value 567 -- but it must be true that 728 * a[123] have only _one_ of these values. (That is, the racing CPUs may not 729 * put the same element twice on the same hash chain.) This is resolved 730 * simply: before the allocation is undertaken, the start of the new chunk's 731 * hash chain is noted. Later, after the allocation is complete, the hash 732 * chain is atomically switched to point to the new element. If this fails 733 * (because of either concurrent allocations or an allocation concurrent with a 734 * deletion), the newly allocated chunk is deallocated to the dirty list, and 735 * the whole process of looking up (and potentially allocating) the dynamic 736 * variable is reattempted. 737 * 738 * The final race is a simple deallocation race: 739 * 740 * CPU A CPU B 741 * +---------------------------------+ +---------------------------------+ 742 * | | | | 743 * | deallocates dynamic object | | deallocates dynamic object | 744 * | a[123] by storing the value 0 | | a[123] by storing the value 0 | 745 * | to it | | to it | 746 * | | | | 747 * : : : : 748 * . . . . 749 * 750 * Once again, this is a race in the D program, but it is one that we must 751 * handle without corrupting the underlying data structures. Because 752 * deallocations require the deletion of a chunk from the middle of a hash 753 * chain, we cannot use a single-word atomic operation to remove it. For this, 754 * we add a spin lock to the hash buckets that is _only_ used for deallocations 755 * (allocation races are handled as above). Further, this spin lock is _only_ 756 * held for the duration of the delete; before control is returned to the DIF 757 * emulation code, the hash bucket is unlocked. 758 */ 759typedef struct dtrace_key { 760 uint64_t dttk_value; /* data value or data pointer */ 761 uint64_t dttk_size; /* 0 if by-val, >0 if by-ref */ 762} dtrace_key_t; 763 764typedef struct dtrace_tuple { 765 uint32_t dtt_nkeys; /* number of keys in tuple */ 766 uint32_t dtt_pad; /* padding */ 767 dtrace_key_t dtt_key[1]; /* array of tuple keys */ 768} dtrace_tuple_t; 769 770typedef struct dtrace_dynvar { 771 uint64_t dtdv_hashval; /* hash value -- 0 if free */ 772 struct dtrace_dynvar *dtdv_next; /* next on list or hash chain */ 773 void *dtdv_data; /* pointer to data */ 774 dtrace_tuple_t dtdv_tuple; /* tuple key */ 775} dtrace_dynvar_t; 776 777typedef enum dtrace_dynvar_op { 778 DTRACE_DYNVAR_ALLOC, 779 DTRACE_DYNVAR_NOALLOC, 780 DTRACE_DYNVAR_DEALLOC 781} dtrace_dynvar_op_t; 782 783typedef struct dtrace_dynhash { 784 dtrace_dynvar_t *dtdh_chain; /* hash chain for this bucket */ 785 uintptr_t dtdh_lock; /* deallocation lock */ 786#ifdef _LP64 787 uintptr_t dtdh_pad[6]; /* pad to avoid false sharing */ 788#else 789 uintptr_t dtdh_pad[14]; /* pad to avoid false sharing */ 790#endif 791} dtrace_dynhash_t; 792 793typedef struct dtrace_dstate_percpu { 794 dtrace_dynvar_t *dtdsc_free; /* free list for this CPU */ 795 dtrace_dynvar_t *dtdsc_dirty; /* dirty list for this CPU */ 796 dtrace_dynvar_t *dtdsc_rinsing; /* rinsing list for this CPU */ 797 dtrace_dynvar_t *dtdsc_clean; /* clean list for this CPU */ 798 uint64_t dtdsc_drops; /* number of capacity drops */ 799 uint64_t dtdsc_dirty_drops; /* number of dirty drops */ 800 uint64_t dtdsc_rinsing_drops; /* number of rinsing drops */ 801#ifdef _LP64 802 uint64_t dtdsc_pad; /* pad to avoid false sharing */ 803#else 804 uint64_t dtdsc_pad[2]; /* pad to avoid false sharing */ 805#endif 806} dtrace_dstate_percpu_t; 807 808typedef enum dtrace_dstate_state { 809 DTRACE_DSTATE_CLEAN = 0, 810 DTRACE_DSTATE_EMPTY, 811 DTRACE_DSTATE_DIRTY, 812 DTRACE_DSTATE_RINSING 813} dtrace_dstate_state_t; 814 815typedef struct dtrace_dstate { 816 void *dtds_base; /* base of dynamic var. space */ 817 size_t dtds_size; /* size of dynamic var. space */ 818 size_t dtds_hashsize; /* number of buckets in hash */ 819 size_t dtds_chunksize; /* size of each chunk */ 820 dtrace_dynhash_t *dtds_hash; /* pointer to hash table */ 821 dtrace_dstate_state_t dtds_state; /* current dynamic var. state */ 822 dtrace_dstate_percpu_t *dtds_percpu; /* per-CPU dyn. var. state */ 823} dtrace_dstate_t; 824 825/* 826 * DTrace Variable State 827 * 828 * The DTrace variable state tracks user-defined variables in its dtrace_vstate 829 * structure. Each DTrace consumer has exactly one dtrace_vstate structure, 830 * but some dtrace_vstate structures may exist without a corresponding DTrace 831 * consumer (see "DTrace Helpers", below). As described in <sys/dtrace.h>, 832 * user-defined variables can have one of three scopes: 833 * 834 * DIFV_SCOPE_GLOBAL => global scope 835 * DIFV_SCOPE_THREAD => thread-local scope (i.e. "self->" variables) 836 * DIFV_SCOPE_LOCAL => clause-local scope (i.e. "this->" variables) 837 * 838 * The variable state tracks variables by both their scope and their allocation 839 * type: 840 * 841 * - The dtvs_globals and dtvs_locals members each point to an array of 842 * dtrace_statvar structures. These structures contain both the variable 843 * metadata (dtrace_difv structures) and the underlying storage for all 844 * statically allocated variables, including statically allocated 845 * DIFV_SCOPE_GLOBAL variables and all DIFV_SCOPE_LOCAL variables. 846 * 847 * - The dtvs_tlocals member points to an array of dtrace_difv structures for 848 * DIFV_SCOPE_THREAD variables. As such, this array tracks _only_ the 849 * variable metadata for DIFV_SCOPE_THREAD variables; the underlying storage 850 * is allocated out of the dynamic variable space. 851 * 852 * - The dtvs_dynvars member is the dynamic variable state associated with the 853 * variable state. The dynamic variable state (described in "DTrace Dynamic 854 * Variables", above) tracks all DIFV_SCOPE_THREAD variables and all 855 * dynamically-allocated DIFV_SCOPE_GLOBAL variables. 856 */ 857typedef struct dtrace_statvar { 858 uint64_t dtsv_data; /* data or pointer to it */ 859 size_t dtsv_size; /* size of pointed-to data */ 860 int dtsv_refcnt; /* reference count */ 861 dtrace_difv_t dtsv_var; /* variable metadata */ 862} dtrace_statvar_t; 863 864typedef struct dtrace_vstate { 865 dtrace_state_t *dtvs_state; /* back pointer to state */ 866 dtrace_statvar_t **dtvs_globals; /* statically-allocated glbls */ 867 int dtvs_nglobals; /* number of globals */ 868 dtrace_difv_t *dtvs_tlocals; /* thread-local metadata */ 869 int dtvs_ntlocals; /* number of thread-locals */ 870 dtrace_statvar_t **dtvs_locals; /* clause-local data */ 871 int dtvs_nlocals; /* number of clause-locals */ 872 dtrace_dstate_t dtvs_dynvars; /* dynamic variable state */ 873} dtrace_vstate_t; 874 875/* 876 * DTrace Machine State 877 * 878 * In the process of processing a fired probe, DTrace needs to track and/or 879 * cache some per-CPU state associated with that particular firing. This is 880 * state that is always discarded after the probe firing has completed, and 881 * much of it is not specific to any DTrace consumer, remaining valid across 882 * all ECBs. This state is tracked in the dtrace_mstate structure. 883 */ 884#define DTRACE_MSTATE_ARGS 0x00000001 885#define DTRACE_MSTATE_PROBE 0x00000002 886#define DTRACE_MSTATE_EPID 0x00000004 887#define DTRACE_MSTATE_TIMESTAMP 0x00000008 888#define DTRACE_MSTATE_STACKDEPTH 0x00000010 889#define DTRACE_MSTATE_CALLER 0x00000020 890#define DTRACE_MSTATE_IPL 0x00000040 891#define DTRACE_MSTATE_FLTOFFS 0x00000080 892#define DTRACE_MSTATE_WALLTIMESTAMP 0x00000100 893#define DTRACE_MSTATE_USTACKDEPTH 0x00000200 894#define DTRACE_MSTATE_UCALLER 0x00000400 895 896typedef struct dtrace_mstate { 897 uintptr_t dtms_scratch_base; /* base of scratch space */ 898 uintptr_t dtms_scratch_ptr; /* current scratch pointer */ 899 size_t dtms_scratch_size; /* scratch size */ 900 uint32_t dtms_present; /* variables that are present */ 901 uint64_t dtms_arg[5]; /* cached arguments */ 902 dtrace_epid_t dtms_epid; /* current EPID */ 903 uint64_t dtms_timestamp; /* cached timestamp */ 904 hrtime_t dtms_walltimestamp; /* cached wall timestamp */ 905 int dtms_stackdepth; /* cached stackdepth */ 906 int dtms_ustackdepth; /* cached ustackdepth */ 907 struct dtrace_probe *dtms_probe; /* current probe */ 908 uintptr_t dtms_caller; /* cached caller */ 909 uint64_t dtms_ucaller; /* cached user-level caller */ 910 int dtms_ipl; /* cached interrupt pri lev */ 911 int dtms_fltoffs; /* faulting DIFO offset */ 912 uintptr_t dtms_strtok; /* saved strtok() pointer */ 913 uint32_t dtms_access; /* memory access rights */ 914 dtrace_difo_t *dtms_difo; /* current dif object */ 915} dtrace_mstate_t; 916 917#define DTRACE_COND_OWNER 0x1 918#define DTRACE_COND_USERMODE 0x2 919#define DTRACE_COND_ZONEOWNER 0x4 920 921#define DTRACE_PROBEKEY_MAXDEPTH 8 /* max glob recursion depth */ 922 923/* 924 * Access flag used by dtrace_mstate.dtms_access. 925 */ 926#define DTRACE_ACCESS_KERNEL 0x1 /* the priv to read kmem */ 927 928 929/* 930 * DTrace Activity 931 * 932 * Each DTrace consumer is in one of several states, which (for purposes of 933 * avoiding yet-another overloading of the noun "state") we call the current 934 * _activity_. The activity transitions on dtrace_go() (from DTRACIOCGO), on 935 * dtrace_stop() (from DTRACIOCSTOP) and on the exit() action. Activities may 936 * only transition in one direction; the activity transition diagram is a 937 * directed acyclic graph. The activity transition diagram is as follows: 938 * 939 * 940 * +----------+ +--------+ +--------+ 941 * | INACTIVE |------------------>| WARMUP |------------------>| ACTIVE | 942 * +----------+ dtrace_go(), +--------+ dtrace_go(), +--------+ 943 * before BEGIN | after BEGIN | | | 944 * | | | | 945 * exit() action | | | | 946 * from BEGIN ECB | | | | 947 * | | | | 948 * v | | | 949 * +----------+ exit() action | | | 950 * +-----------------------------| DRAINING |<-------------------+ | | 951 * | +----------+ | | 952 * | | | | 953 * | dtrace_stop(), | | | 954 * | before END | | | 955 * | | | | 956 * | v | | 957 * | +---------+ +----------+ | | 958 * | | STOPPED |<----------------| COOLDOWN |<----------------------+ | 959 * | +---------+ dtrace_stop(), +----------+ dtrace_stop(), | 960 * | after END before END | 961 * | | 962 * | +--------+ | 963 * +----------------------------->| KILLED |<--------------------------+ 964 * deadman timeout or +--------+ deadman timeout or 965 * killed consumer killed consumer 966 * 967 * Note that once a DTrace consumer has stopped tracing, there is no way to 968 * restart it; if a DTrace consumer wishes to restart tracing, it must reopen 969 * the DTrace pseudodevice. 970 */ 971typedef enum dtrace_activity { 972 DTRACE_ACTIVITY_INACTIVE = 0, /* not yet running */ 973 DTRACE_ACTIVITY_WARMUP, /* while starting */ 974 DTRACE_ACTIVITY_ACTIVE, /* running */ 975 DTRACE_ACTIVITY_DRAINING, /* before stopping */ 976 DTRACE_ACTIVITY_COOLDOWN, /* while stopping */ 977 DTRACE_ACTIVITY_STOPPED, /* after stopping */ 978 DTRACE_ACTIVITY_KILLED /* killed */ 979} dtrace_activity_t; 980 981#if defined(__APPLE__) 982/* 983 * DTrace dof modes 984 * 985 * DTrace has four "dof modes". They are: 986 * 987 * DTRACE_DOF_MODE_NEVER Never load any dof, period. 988 * DTRACE_DOF_MODE_LAZY_ON Defer loading dof until later 989 * DTRACE_DOF_MODE_LAZY_OFF Load all deferred dof now, and any new dof 990 * DTRACE_DOF_MODE_NON_LAZY Load all dof immediately. 991 * 992 * It is legal to transition between the two lazy modes. The NEVER and 993 * NON_LAZY modes are permanent, and must not change once set. 994 * 995 * The current dof mode is kept in dtrace_dof_mode, which is protected by the 996 * dtrace_dof_mode_lock. This is a RW lock, reads require shared access, writes 997 * require exclusive access. Because NEVER and NON_LAZY are permanent states, 998 * it is legal to test for those modes without holding the dof mode lock. 999 * 1000 * Lock ordering is dof mode lock before any dtrace lock, and before the 1001 * process p_dtrace_sprlock. In general, other locks should not be held when 1002 * taking the dof mode lock. Acquiring the dof mode lock in exclusive mode 1003 * will block process fork, exec, and exit, so it should be held exclusive 1004 * for as short a time as possible. 1005 */ 1006 1007#define DTRACE_DOF_MODE_NEVER 0 1008#define DTRACE_DOF_MODE_LAZY_ON 1 1009#define DTRACE_DOF_MODE_LAZY_OFF 2 1010#define DTRACE_DOF_MODE_NON_LAZY 3 1011 1012/* 1013 * dtrace kernel symbol modes are used to control when the kernel may dispose of 1014 * symbol information used by the fbt/sdt provider. The kernel itself, as well as 1015 * every kext, has symbol table/nlist info that has historically been preserved 1016 * for dtrace's use. This allowed dtrace to be lazy about allocating fbt/sdt probes, 1017 * at the expense of keeping the symbol info in the kernel permanently. 1018 * 1019 * Starting in 10.7+, fbt probes may be created from userspace, in the same 1020 * fashion as pid probes. The kernel allows dtrace "first right of refusal" 1021 * whenever symbol data becomes available (such as a kext load). If dtrace is 1022 * active, it will immediately read/copy the needed data, and then the kernel 1023 * may free it. If dtrace is not active, it returns immediately, having done 1024 * no work or allocations, and the symbol data is freed. Should dtrace need 1025 * this data later, it is expected that the userspace client will push the 1026 * data into the kernel via ioctl calls. 1027 * 1028 * The kernel symbol modes are used to control what dtrace does with symbol data: 1029 * 1030 * DTRACE_KERNEL_SYMBOLS_NEVER Effectively disables fbt/sdt 1031 * DTRACE_KERNEL_SYMBOLS_FROM_KERNEL Immediately read/copy symbol data 1032 * DTRACE_KERNEL_SYMBOLS_FROM_USERSPACE Wait for symbols from userspace 1033 * DTRACE_KERNEL_SYMBOLS_ALWAYS_FROM_KERNEL Immediately read/copy symbol data 1034 * 1035 * It is legal to transition between DTRACE_KERNEL_SYMBOLS_FROM_KERNEL and 1036 * DTRACE_KERNEL_SYMBOLS_FROM_USERSPACE. The DTRACE_KERNEL_SYMBOLS_NEVER and 1037 * DTRACE_KERNEL_SYMBOLS_ALWAYS_FROM_KERNEL are permanent modes, intended to 1038 * disable fbt probes entirely, or prevent any symbols being loaded from 1039 * userspace. 1040* 1041 * The kernel symbol mode is kept in dtrace_kernel_symbol_mode, which is protected 1042 * by the dtrace_lock. 1043 */ 1044 1045#define DTRACE_KERNEL_SYMBOLS_NEVER 0 1046#define DTRACE_KERNEL_SYMBOLS_FROM_KERNEL 1 1047#define DTRACE_KERNEL_SYMBOLS_FROM_USERSPACE 2 1048#define DTRACE_KERNEL_SYMBOLS_ALWAYS_FROM_KERNEL 3 1049 1050#endif /* __APPLE__ */ 1051 1052/* 1053 * DTrace Helper Implementation 1054 * 1055 * A description of the helper architecture may be found in <sys/dtrace.h>. 1056 * Each process contains a pointer to its helpers in its p_dtrace_helpers 1057 * member. This is a pointer to a dtrace_helpers structure, which contains an 1058 * array of pointers to dtrace_helper structures, helper variable state (shared 1059 * among a process's helpers) and a generation count. (The generation count is 1060 * used to provide an identifier when a helper is added so that it may be 1061 * subsequently removed.) The dtrace_helper structure is self-explanatory, 1062 * containing pointers to the objects needed to execute the helper. Note that 1063 * helpers are _duplicated_ across fork(2), and destroyed on exec(2). No more 1064 * than dtrace_helpers_max are allowed per-process. 1065 */ 1066#define DTRACE_HELPER_ACTION_USTACK 0 1067#define DTRACE_NHELPER_ACTIONS 1 1068 1069typedef struct dtrace_helper_action { 1070 int dtha_generation; /* helper action generation */ 1071 int dtha_nactions; /* number of actions */ 1072 dtrace_difo_t *dtha_predicate; /* helper action predicate */ 1073 dtrace_difo_t **dtha_actions; /* array of actions */ 1074 struct dtrace_helper_action *dtha_next; /* next helper action */ 1075} dtrace_helper_action_t; 1076 1077typedef struct dtrace_helper_provider { 1078 int dthp_generation; /* helper provider generation */ 1079 uint32_t dthp_ref; /* reference count */ 1080 dof_helper_t dthp_prov; /* DOF w/ provider and probes */ 1081} dtrace_helper_provider_t; 1082 1083typedef struct dtrace_helpers { 1084 dtrace_helper_action_t **dthps_actions; /* array of helper actions */ 1085 dtrace_vstate_t dthps_vstate; /* helper action var. state */ 1086 dtrace_helper_provider_t **dthps_provs; /* array of providers */ 1087 uint_t dthps_nprovs; /* count of providers */ 1088 uint_t dthps_maxprovs; /* provider array size */ 1089 int dthps_generation; /* current generation */ 1090 pid_t dthps_pid; /* pid of associated proc */ 1091 int dthps_deferred; /* helper in deferred list */ 1092 struct dtrace_helpers *dthps_next; /* next pointer */ 1093 struct dtrace_helpers *dthps_prev; /* prev pointer */ 1094} dtrace_helpers_t; 1095 1096/* 1097 * DTrace Helper Action Tracing 1098 * 1099 * Debugging helper actions can be arduous. To ease the development and 1100 * debugging of helpers, DTrace contains a tracing-framework-within-a-tracing- 1101 * framework: helper tracing. If dtrace_helptrace_enabled is non-zero (which 1102 * it is by default on DEBUG kernels), all helper activity will be traced to a 1103 * global, in-kernel ring buffer. Each entry includes a pointer to the specific 1104 * helper, the location within the helper, and a trace of all local variables. 1105 * The ring buffer may be displayed in a human-readable format with the 1106 * ::dtrace_helptrace mdb(1) dcmd. 1107 */ 1108#define DTRACE_HELPTRACE_NEXT (-1) 1109#define DTRACE_HELPTRACE_DONE (-2) 1110#define DTRACE_HELPTRACE_ERR (-3) 1111 1112typedef struct dtrace_helptrace { 1113 dtrace_helper_action_t *dtht_helper; /* helper action */ 1114 int dtht_where; /* where in helper action */ 1115 int dtht_nlocals; /* number of locals */ 1116 int dtht_fault; /* type of fault (if any) */ 1117 int dtht_fltoffs; /* DIF offset */ 1118 uint64_t dtht_illval; /* faulting value */ 1119 uint64_t dtht_locals[1]; /* local variables */ 1120} dtrace_helptrace_t; 1121 1122/* 1123 * DTrace Credentials 1124 * 1125 * In probe context, we have limited flexibility to examine the credentials 1126 * of the DTrace consumer that created a particular enabling. We use 1127 * the Least Privilege interfaces to cache the consumer's cred pointer and 1128 * some facts about that credential in a dtrace_cred_t structure. These 1129 * can limit the consumer's breadth of visibility and what actions the 1130 * consumer may take. 1131 */ 1132#define DTRACE_CRV_ALLPROC 0x01 1133#define DTRACE_CRV_KERNEL 0x02 1134#define DTRACE_CRV_ALLZONE 0x04 1135 1136#define DTRACE_CRV_ALL (DTRACE_CRV_ALLPROC | DTRACE_CRV_KERNEL | \ 1137 DTRACE_CRV_ALLZONE) 1138 1139#define DTRACE_CRA_PROC 0x0001 1140#define DTRACE_CRA_PROC_CONTROL 0x0002 1141#define DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER 0x0004 1142#define DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE 0x0008 1143#define DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG 0x0010 1144#define DTRACE_CRA_KERNEL 0x0020 1145#define DTRACE_CRA_KERNEL_DESTRUCTIVE 0x0040 1146 1147#define DTRACE_CRA_ALL (DTRACE_CRA_PROC | \ 1148 DTRACE_CRA_PROC_CONTROL | \ 1149 DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER | \ 1150 DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE | \ 1151 DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG | \ 1152 DTRACE_CRA_KERNEL | \ 1153 DTRACE_CRA_KERNEL_DESTRUCTIVE) 1154 1155typedef struct dtrace_cred { 1156 cred_t *dcr_cred; 1157 uint8_t dcr_destructive; 1158 uint8_t dcr_visible; 1159 uint16_t dcr_action; 1160} dtrace_cred_t; 1161 1162/* 1163 * DTrace Consumer State 1164 * 1165 * Each DTrace consumer has an associated dtrace_state structure that contains 1166 * its in-kernel DTrace state -- including options, credentials, statistics and 1167 * pointers to ECBs, buffers, speculations and formats. A dtrace_state 1168 * structure is also allocated for anonymous enablings. When anonymous state 1169 * is grabbed, the grabbing consumers dts_anon pointer is set to the grabbed 1170 * dtrace_state structure. 1171 */ 1172struct dtrace_state { 1173 dev_t dts_dev; /* device */ 1174 int dts_necbs; /* total number of ECBs */ 1175 dtrace_ecb_t **dts_ecbs; /* array of ECBs */ 1176 dtrace_epid_t dts_epid; /* next EPID to allocate */ 1177 size_t dts_needed; /* greatest needed space */ 1178 struct dtrace_state *dts_anon; /* anon. state, if grabbed */ 1179 dtrace_activity_t dts_activity; /* current activity */ 1180 dtrace_vstate_t dts_vstate; /* variable state */ 1181 dtrace_buffer_t *dts_buffer; /* principal buffer */ 1182 dtrace_buffer_t *dts_aggbuffer; /* aggregation buffer */ 1183 dtrace_speculation_t *dts_speculations; /* speculation array */ 1184 int dts_nspeculations; /* number of speculations */ 1185 int dts_naggregations; /* number of aggregations */ 1186 dtrace_aggregation_t **dts_aggregations; /* aggregation array */ 1187 vmem_t *dts_aggid_arena; /* arena for aggregation IDs */ 1188 uint64_t dts_errors; /* total number of errors */ 1189 uint32_t dts_speculations_busy; /* number of spec. busy */ 1190 uint32_t dts_speculations_unavail; /* number of spec unavail */ 1191 uint32_t dts_stkstroverflows; /* stack string tab overflows */ 1192 uint32_t dts_dblerrors; /* errors in ERROR probes */ 1193 uint32_t dts_reserve; /* space reserved for END */ 1194 hrtime_t dts_laststatus; /* time of last status */ 1195 cyclic_id_t dts_cleaner; /* cleaning cyclic */ 1196 cyclic_id_t dts_deadman; /* deadman cyclic */ 1197 hrtime_t dts_alive; /* time last alive */ 1198 char dts_speculates; /* boolean: has speculations */ 1199 char dts_destructive; /* boolean: has dest. actions */ 1200 int dts_nformats; /* number of formats */ 1201 char **dts_formats; /* format string array */ 1202 dtrace_optval_t dts_options[DTRACEOPT_MAX]; /* options */ 1203 dtrace_cred_t dts_cred; /* credentials */ 1204 size_t dts_nretained; /* number of retained enabs */ 1205#if defined(__APPLE__) 1206 uint64_t dts_arg_error_illval; 1207#endif /* __APPLE__ */ 1208}; 1209 1210struct dtrace_provider { 1211 dtrace_pattr_t dtpv_attr; /* provider attributes */ 1212 dtrace_ppriv_t dtpv_priv; /* provider privileges */ 1213 dtrace_pops_t dtpv_pops; /* provider operations */ 1214 char *dtpv_name; /* provider name */ 1215 void *dtpv_arg; /* provider argument */ 1216 uint_t dtpv_defunct; /* boolean: defunct provider */ 1217 struct dtrace_provider *dtpv_next; /* next provider */ 1218}; 1219 1220struct dtrace_meta { 1221 dtrace_mops_t dtm_mops; /* meta provider operations */ 1222 char *dtm_name; /* meta provider name */ 1223 void *dtm_arg; /* meta provider user arg */ 1224 uint64_t dtm_count; /* no. of associated provs. */ 1225}; 1226 1227/* 1228 * DTrace Enablings 1229 * 1230 * A dtrace_enabling structure is used to track a collection of ECB 1231 * descriptions -- before they have been turned into actual ECBs. This is 1232 * created as a result of DOF processing, and is generally used to generate 1233 * ECBs immediately thereafter. However, enablings are also generally 1234 * retained should the probes they describe be created at a later time; as 1235 * each new module or provider registers with the framework, the retained 1236 * enablings are reevaluated, with any new match resulting in new ECBs. To 1237 * prevent probes from being matched more than once, the enabling tracks the 1238 * last probe generation matched, and only matches probes from subsequent 1239 * generations. 1240 */ 1241typedef struct dtrace_enabling { 1242 dtrace_ecbdesc_t **dten_desc; /* all ECB descriptions */ 1243 int dten_ndesc; /* number of ECB descriptions */ 1244 int dten_maxdesc; /* size of ECB array */ 1245 dtrace_vstate_t *dten_vstate; /* associated variable state */ 1246 dtrace_genid_t dten_probegen; /* matched probe generation */ 1247 dtrace_ecbdesc_t *dten_current; /* current ECB description */ 1248 int dten_error; /* current error value */ 1249 int dten_primed; /* boolean: set if primed */ 1250 struct dtrace_enabling *dten_prev; /* previous enabling */ 1251 struct dtrace_enabling *dten_next; /* next enabling */ 1252} dtrace_enabling_t; 1253 1254/* 1255 * DTrace Anonymous Enablings 1256 * 1257 * Anonymous enablings are DTrace enablings that are not associated with a 1258 * controlling process, but rather derive their enabling from DOF stored as 1259 * properties in the dtrace.conf file. If there is an anonymous enabling, a 1260 * DTrace consumer state and enabling are created on attach. The state may be 1261 * subsequently grabbed by the first consumer specifying the "grabanon" 1262 * option. As long as an anonymous DTrace enabling exists, dtrace(7D) will 1263 * refuse to unload. 1264 */ 1265typedef struct dtrace_anon { 1266 dtrace_state_t *dta_state; /* DTrace consumer state */ 1267 dtrace_enabling_t *dta_enabling; /* pointer to enabling */ 1268 processorid_t dta_beganon; /* which CPU BEGIN ran on */ 1269} dtrace_anon_t; 1270 1271/* 1272 * DTrace Error Debugging 1273 */ 1274#if DEBUG 1275#define DTRACE_ERRDEBUG 1276#endif 1277 1278#ifdef DTRACE_ERRDEBUG 1279 1280typedef struct dtrace_errhash { 1281 const char *dter_msg; /* error message */ 1282 int dter_count; /* number of times seen */ 1283} dtrace_errhash_t; 1284 1285#define DTRACE_ERRHASHSZ 256 /* must be > number of err msgs */ 1286 1287#endif /* DTRACE_ERRDEBUG */ 1288 1289/* 1290 * DTrace Toxic Ranges 1291 * 1292 * DTrace supports safe loads from probe context; if the address turns out to 1293 * be invalid, a bit will be set by the kernel indicating that DTrace 1294 * encountered a memory error, and DTrace will propagate the error to the user 1295 * accordingly. However, there may exist some regions of memory in which an 1296 * arbitrary load can change system state, and from which it is impossible to 1297 * recover from such a load after it has been attempted. Examples of this may 1298 * include memory in which programmable I/O registers are mapped (for which a 1299 * read may have some implications for the device) or (in the specific case of 1300 * UltraSPARC-I and -II) the virtual address hole. The platform is required 1301 * to make DTrace aware of these toxic ranges; DTrace will then check that 1302 * target addresses are not in a toxic range before attempting to issue a 1303 * safe load. 1304 */ 1305typedef struct dtrace_toxrange { 1306 uintptr_t dtt_base; /* base of toxic range */ 1307 uintptr_t dtt_limit; /* limit of toxic range */ 1308} dtrace_toxrange_t; 1309 1310extern uint64_t dtrace_getarg(int, int); 1311extern int dtrace_getipl(void); 1312extern uintptr_t dtrace_caller(int); 1313extern uint32_t dtrace_cas32(uint32_t *, uint32_t, uint32_t); 1314extern void *dtrace_casptr(void *, void *, void *); 1315#if !defined(__APPLE__) 1316extern void dtrace_copyin(uintptr_t, uintptr_t, size_t, volatile uint16_t *); 1317extern void dtrace_copyinstr(uintptr_t, uintptr_t, size_t, volatile uint16_t *); 1318extern void dtrace_copyout(uintptr_t, uintptr_t, size_t, volatile uint16_t *); 1319extern void dtrace_copyoutstr(uintptr_t, uintptr_t, size_t, 1320 volatile uint16_t *); 1321#else 1322extern void dtrace_copyin(user_addr_t, uintptr_t, size_t, volatile uint16_t *); 1323extern void dtrace_copyinstr(user_addr_t, uintptr_t, size_t, volatile uint16_t *); 1324extern void dtrace_copyout(uintptr_t, user_addr_t, size_t, volatile uint16_t *); 1325extern void dtrace_copyoutstr(uintptr_t, user_addr_t, size_t, volatile uint16_t *); 1326#endif /* __APPLE__ */ 1327extern void dtrace_getpcstack(pc_t *, int, int, uint32_t *); 1328#if !defined(__APPLE__) 1329extern ulong_t dtrace_getreg(struct regs *, uint_t); 1330#else 1331extern uint64_t dtrace_getreg(struct regs *, uint_t); 1332#endif /* __APPLE__ */ 1333extern int dtrace_getstackdepth(int); 1334extern void dtrace_getupcstack(uint64_t *, int); 1335extern void dtrace_getufpstack(uint64_t *, uint64_t *, int); 1336extern int dtrace_getustackdepth(void); 1337extern uintptr_t dtrace_fulword(void *); 1338#if !defined(__APPLE__) 1339extern uint8_t dtrace_fuword8(void *); 1340extern uint16_t dtrace_fuword16(void *); 1341extern uint32_t dtrace_fuword32(void *); 1342extern uint64_t dtrace_fuword64(void *); 1343extern void dtrace_probe_error(dtrace_state_t *, dtrace_epid_t, int, int, 1344 int, uintptr_t); 1345#else 1346extern uint8_t dtrace_fuword8(user_addr_t); 1347extern uint16_t dtrace_fuword16(user_addr_t); 1348extern uint32_t dtrace_fuword32(user_addr_t); 1349extern uint64_t dtrace_fuword64(user_addr_t); 1350extern void dtrace_probe_error(dtrace_state_t *, dtrace_epid_t, int, int, 1351 int, uint64_t); 1352#endif /* __APPLE__ */ 1353extern int dtrace_assfail(const char *, const char *, int); 1354extern int dtrace_attached(void); 1355extern hrtime_t dtrace_gethrestime(void); 1356extern void dtrace_isa_init(void); 1357 1358#ifdef __sparc 1359extern void dtrace_flush_windows(void); 1360extern void dtrace_flush_user_windows(void); 1361extern uint_t dtrace_getotherwin(void); 1362extern uint_t dtrace_getfprs(void); 1363#else 1364extern void dtrace_copy(uintptr_t, uintptr_t, size_t); 1365extern void dtrace_copystr(uintptr_t, uintptr_t, size_t, volatile uint16_t *); 1366#endif 1367 1368/* 1369 * DTrace Assertions 1370 * 1371 * DTrace calls ASSERT from probe context. To assure that a failed ASSERT 1372 * does not induce a markedly more catastrophic failure (e.g., one from which 1373 * a dump cannot be gleaned), DTrace must define its own ASSERT to be one that 1374 * may safely be called from probe context. This header file must thus be 1375 * included by any DTrace component that calls ASSERT from probe context, and 1376 * _only_ by those components. (The only exception to this is kernel 1377 * debugging infrastructure at user-level that doesn't depend on calling 1378 * ASSERT.) 1379 */ 1380#undef ASSERT 1381#if DEBUG 1382#define ASSERT(EX) ((void)((EX) || \ 1383 dtrace_assfail(#EX, __FILE__, __LINE__))) 1384#else 1385#define ASSERT(X) ((void)0) 1386#endif 1387 1388#ifdef __cplusplus 1389} 1390#endif 1391 1392#endif /* _SYS_DTRACE_IMPL_H */ 1393 1394