fasttrap.c revision 268734
184699Smike/* 284699Smike * CDDL HEADER START 384699Smike * 484699Smike * The contents of this file are subject to the terms of the 584699Smike * Common Development and Distribution License (the "License"). 684699Smike * You may not use this file except in compliance with the License. 784699Smike * 884699Smike * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 984699Smike * or http://www.opensolaris.org/os/licensing. 1084699Smike * See the License for the specific language governing permissions 1184699Smike * and limitations under the License. 1284699Smike * 1384699Smike * When distributing Covered Code, include this CDDL HEADER in each 1484699Smike * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 1584699Smike * If applicable, add the following below this CDDL HEADER, with the 1684699Smike * fields enclosed by brackets "[]" replaced with your own identifying 17251069Semaste * information: Portions Copyright [yyyy] [name of copyright owner] 1884699Smike * 1984699Smike * CDDL HEADER END 2084699Smike * 2184699Smike * Portions Copyright 2010 The FreeBSD Foundation 2284699Smike * 2384699Smike * $FreeBSD: stable/10/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c 268734 2014-07-16 02:02:46Z pfg $ 2484699Smike */ 2584699Smike 2684699Smike/* 2784699Smike * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 2884699Smike * Use is subject to license terms. 2984699Smike */ 3084699Smike 3184699Smike/* 3284699Smike * Copyright (c) 2013, Joyent, Inc. All rights reserved. 3384699Smike */ 3486170Sobrien 3586170Sobrien#include <sys/atomic.h> 3686170Sobrien#include <sys/errno.h> 3784699Smike#include <sys/stat.h> 3884699Smike#include <sys/modctl.h> 3984699Smike#include <sys/conf.h> 4084699Smike#include <sys/systm.h> 4184699Smike#if defined(sun) 4284699Smike#include <sys/ddi.h> 4384699Smike#endif 4484699Smike#include <sys/sunddi.h> 4584699Smike#include <sys/cpuvar.h> 4684699Smike#include <sys/kmem.h> 47188080Sdanger#if defined(sun) 4884699Smike#include <sys/strsubr.h> 4984699Smike#endif 5084699Smike#include <sys/fasttrap.h> 5184699Smike#include <sys/fasttrap_impl.h> 5284699Smike#include <sys/fasttrap_isa.h> 5384699Smike#include <sys/dtrace.h> 5484699Smike#include <sys/dtrace_impl.h> 5584699Smike#include <sys/sysmacros.h> 56141701Spjd#include <sys/proc.h> 5784699Smike#include <sys/policy.h> 5884699Smike#if defined(sun) 5984699Smike#include <util/qsort.h> 6084699Smike#endif 6184699Smike#include <sys/mutex.h> 6284699Smike#include <sys/kernel.h> 6384699Smike#if !defined(sun) 6484699Smike#include <sys/user.h> 6584699Smike#include <sys/dtrace_bsd.h> 66#include <sys/u8_textprep.h> 67#include <cddl/dev/dtrace/dtrace_cddl.h> 68#endif 69 70/* 71 * User-Land Trap-Based Tracing 72 * ---------------------------- 73 * 74 * The fasttrap provider allows DTrace consumers to instrument any user-level 75 * instruction to gather data; this includes probes with semantic 76 * signifigance like entry and return as well as simple offsets into the 77 * function. While the specific techniques used are very ISA specific, the 78 * methodology is generalizable to any architecture. 79 * 80 * 81 * The General Methodology 82 * ----------------------- 83 * 84 * With the primary goal of tracing every user-land instruction and the 85 * limitation that we can't trust user space so don't want to rely on much 86 * information there, we begin by replacing the instructions we want to trace 87 * with trap instructions. Each instruction we overwrite is saved into a hash 88 * table keyed by process ID and pc address. When we enter the kernel due to 89 * this trap instruction, we need the effects of the replaced instruction to 90 * appear to have occurred before we proceed with the user thread's 91 * execution. 92 * 93 * Each user level thread is represented by a ulwp_t structure which is 94 * always easily accessible through a register. The most basic way to produce 95 * the effects of the instruction we replaced is to copy that instruction out 96 * to a bit of scratch space reserved in the user thread's ulwp_t structure 97 * (a sort of kernel-private thread local storage), set the PC to that 98 * scratch space and single step. When we reenter the kernel after single 99 * stepping the instruction we must then adjust the PC to point to what would 100 * normally be the next instruction. Of course, special care must be taken 101 * for branches and jumps, but these represent such a small fraction of any 102 * instruction set that writing the code to emulate these in the kernel is 103 * not too difficult. 104 * 105 * Return probes may require several tracepoints to trace every return site, 106 * and, conversely, each tracepoint may activate several probes (the entry 107 * and offset 0 probes, for example). To solve this muliplexing problem, 108 * tracepoints contain lists of probes to activate and probes contain lists 109 * of tracepoints to enable. If a probe is activated, it adds its ID to 110 * existing tracepoints or creates new ones as necessary. 111 * 112 * Most probes are activated _before_ the instruction is executed, but return 113 * probes are activated _after_ the effects of the last instruction of the 114 * function are visible. Return probes must be fired _after_ we have 115 * single-stepped the instruction whereas all other probes are fired 116 * beforehand. 117 * 118 * 119 * Lock Ordering 120 * ------------- 121 * 122 * The lock ordering below -- both internally and with respect to the DTrace 123 * framework -- is a little tricky and bears some explanation. Each provider 124 * has a lock (ftp_mtx) that protects its members including reference counts 125 * for enabled probes (ftp_rcount), consumers actively creating probes 126 * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider 127 * from being freed. A provider is looked up by taking the bucket lock for the 128 * provider hash table, and is returned with its lock held. The provider lock 129 * may be taken in functions invoked by the DTrace framework, but may not be 130 * held while calling functions in the DTrace framework. 131 * 132 * To ensure consistency over multiple calls to the DTrace framework, the 133 * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may 134 * not be taken when holding the provider lock as that would create a cyclic 135 * lock ordering. In situations where one would naturally take the provider 136 * lock and then the creation lock, we instead up a reference count to prevent 137 * the provider from disappearing, drop the provider lock, and acquire the 138 * creation lock. 139 * 140 * Briefly: 141 * bucket lock before provider lock 142 * DTrace before provider lock 143 * creation lock before DTrace 144 * never hold the provider lock and creation lock simultaneously 145 */ 146 147static d_open_t fasttrap_open; 148static d_ioctl_t fasttrap_ioctl; 149 150static struct cdevsw fasttrap_cdevsw = { 151 .d_version = D_VERSION, 152 .d_open = fasttrap_open, 153 .d_ioctl = fasttrap_ioctl, 154 .d_name = "fasttrap", 155}; 156static struct cdev *fasttrap_cdev; 157static dtrace_meta_provider_id_t fasttrap_meta_id; 158 159static struct proc *fasttrap_cleanup_proc; 160static struct mtx fasttrap_cleanup_mtx; 161static uint_t fasttrap_cleanup_work, fasttrap_cleanup_drain, fasttrap_cleanup_cv; 162 163/* 164 * Generation count on modifications to the global tracepoint lookup table. 165 */ 166static volatile uint64_t fasttrap_mod_gen; 167 168/* 169 * When the fasttrap provider is loaded, fasttrap_max is set to either 170 * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the 171 * fasttrap.conf file. Each time a probe is created, fasttrap_total is 172 * incremented by the number of tracepoints that may be associated with that 173 * probe; fasttrap_total is capped at fasttrap_max. 174 */ 175#define FASTTRAP_MAX_DEFAULT 250000 176static uint32_t fasttrap_max; 177static uint32_t fasttrap_total; 178 179/* 180 * Copyright (c) 2011, Joyent, Inc. All rights reserved. 181 */ 182 183#define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000 184#define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100 185#define FASTTRAP_PROCS_DEFAULT_SIZE 0x100 186 187#define FASTTRAP_PID_NAME "pid" 188 189fasttrap_hash_t fasttrap_tpoints; 190static fasttrap_hash_t fasttrap_provs; 191static fasttrap_hash_t fasttrap_procs; 192 193static uint64_t fasttrap_pid_count; /* pid ref count */ 194static kmutex_t fasttrap_count_mtx; /* lock on ref count */ 195 196#define FASTTRAP_ENABLE_FAIL 1 197#define FASTTRAP_ENABLE_PARTIAL 2 198 199static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t); 200static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t); 201 202static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *, 203 const dtrace_pattr_t *); 204static void fasttrap_provider_retire(pid_t, const char *, int); 205static void fasttrap_provider_free(fasttrap_provider_t *); 206 207static fasttrap_proc_t *fasttrap_proc_lookup(pid_t); 208static void fasttrap_proc_release(fasttrap_proc_t *); 209 210#define FASTTRAP_PROVS_INDEX(pid, name) \ 211 ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask) 212 213#define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask) 214 215#if !defined(sun) 216static kmutex_t fasttrap_cpuc_pid_lock[MAXCPU]; 217#endif 218 219static int 220fasttrap_highbit(ulong_t i) 221{ 222 int h = 1; 223 224 if (i == 0) 225 return (0); 226#ifdef _LP64 227 if (i & 0xffffffff00000000ul) { 228 h += 32; i >>= 32; 229 } 230#endif 231 if (i & 0xffff0000) { 232 h += 16; i >>= 16; 233 } 234 if (i & 0xff00) { 235 h += 8; i >>= 8; 236 } 237 if (i & 0xf0) { 238 h += 4; i >>= 4; 239 } 240 if (i & 0xc) { 241 h += 2; i >>= 2; 242 } 243 if (i & 0x2) { 244 h += 1; 245 } 246 return (h); 247} 248 249static uint_t 250fasttrap_hash_str(const char *p) 251{ 252 unsigned int g; 253 uint_t hval = 0; 254 255 while (*p) { 256 hval = (hval << 4) + *p++; 257 if ((g = (hval & 0xf0000000)) != 0) 258 hval ^= g >> 24; 259 hval &= ~g; 260 } 261 return (hval); 262} 263 264void 265fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc) 266{ 267#if defined(sun) 268 sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP); 269 270 sqp->sq_info.si_signo = SIGTRAP; 271 sqp->sq_info.si_code = TRAP_DTRACE; 272 sqp->sq_info.si_addr = (caddr_t)pc; 273 274 mutex_enter(&p->p_lock); 275 sigaddqa(p, t, sqp); 276 mutex_exit(&p->p_lock); 277 278 if (t != NULL) 279 aston(t); 280#else 281 ksiginfo_t *ksi = kmem_zalloc(sizeof (ksiginfo_t), KM_SLEEP); 282 283 ksiginfo_init(ksi); 284 ksi->ksi_signo = SIGTRAP; 285 ksi->ksi_code = TRAP_DTRACE; 286 ksi->ksi_addr = (caddr_t)pc; 287 PROC_LOCK(p); 288 (void) tdksignal(t, SIGTRAP, ksi); 289 PROC_UNLOCK(p); 290#endif 291} 292 293/* 294 * This function ensures that no threads are actively using the memory 295 * associated with probes that were formerly live. 296 */ 297static void 298fasttrap_mod_barrier(uint64_t gen) 299{ 300 int i; 301 302 if (gen < fasttrap_mod_gen) 303 return; 304 305 fasttrap_mod_gen++; 306 307 CPU_FOREACH(i) { 308 mutex_enter(&fasttrap_cpuc_pid_lock[i]); 309 mutex_exit(&fasttrap_cpuc_pid_lock[i]); 310 } 311} 312 313/* 314 * This function performs asynchronous cleanup of fasttrap providers. The 315 * Solaris implementation of this mechanism use a timeout that's activated in 316 * fasttrap_pid_cleanup(), but this doesn't work in FreeBSD: one may sleep while 317 * holding the DTrace mutexes, but it is unsafe to sleep in a callout handler. 318 * Thus we use a dedicated process to perform the cleanup when requested. 319 */ 320/*ARGSUSED*/ 321static void 322fasttrap_pid_cleanup_cb(void *data) 323{ 324 fasttrap_provider_t **fpp, *fp; 325 fasttrap_bucket_t *bucket; 326 dtrace_provider_id_t provid; 327 int i, later = 0, rval; 328 329 mtx_lock(&fasttrap_cleanup_mtx); 330 while (!fasttrap_cleanup_drain || later > 0) { 331 fasttrap_cleanup_work = 0; 332 mtx_unlock(&fasttrap_cleanup_mtx); 333 334 later = 0; 335 336 /* 337 * Iterate over all the providers trying to remove the marked 338 * ones. If a provider is marked but not retired, we just 339 * have to take a crack at removing it -- it's no big deal if 340 * we can't. 341 */ 342 for (i = 0; i < fasttrap_provs.fth_nent; i++) { 343 bucket = &fasttrap_provs.fth_table[i]; 344 mutex_enter(&bucket->ftb_mtx); 345 fpp = (fasttrap_provider_t **)&bucket->ftb_data; 346 347 while ((fp = *fpp) != NULL) { 348 if (!fp->ftp_marked) { 349 fpp = &fp->ftp_next; 350 continue; 351 } 352 353 mutex_enter(&fp->ftp_mtx); 354 355 /* 356 * If this provider has consumers actively 357 * creating probes (ftp_ccount) or is a USDT 358 * provider (ftp_mcount), we can't unregister 359 * or even condense. 360 */ 361 if (fp->ftp_ccount != 0 || 362 fp->ftp_mcount != 0) { 363 mutex_exit(&fp->ftp_mtx); 364 fp->ftp_marked = 0; 365 continue; 366 } 367 368 if (!fp->ftp_retired || fp->ftp_rcount != 0) 369 fp->ftp_marked = 0; 370 371 mutex_exit(&fp->ftp_mtx); 372 373 /* 374 * If we successfully unregister this 375 * provider we can remove it from the hash 376 * chain and free the memory. If our attempt 377 * to unregister fails and this is a retired 378 * provider, increment our flag to try again 379 * pretty soon. If we've consumed more than 380 * half of our total permitted number of 381 * probes call dtrace_condense() to try to 382 * clean out the unenabled probes. 383 */ 384 provid = fp->ftp_provid; 385 if ((rval = dtrace_unregister(provid)) != 0) { 386 if (fasttrap_total > fasttrap_max / 2) 387 (void) dtrace_condense(provid); 388 389 if (rval == EAGAIN) 390 fp->ftp_marked = 1; 391 392 later += fp->ftp_marked; 393 fpp = &fp->ftp_next; 394 } else { 395 *fpp = fp->ftp_next; 396 fasttrap_provider_free(fp); 397 } 398 } 399 mutex_exit(&bucket->ftb_mtx); 400 } 401 mtx_lock(&fasttrap_cleanup_mtx); 402 403 /* 404 * If we were unable to retire a provider, try again after a 405 * second. This situation can occur in certain circumstances 406 * where providers cannot be unregistered even though they have 407 * no probes enabled because of an execution of dtrace -l or 408 * something similar. 409 */ 410 if (later > 0 || fasttrap_cleanup_work || 411 fasttrap_cleanup_drain) { 412 mtx_unlock(&fasttrap_cleanup_mtx); 413 pause("ftclean", hz); 414 mtx_lock(&fasttrap_cleanup_mtx); 415 } else 416 mtx_sleep(&fasttrap_cleanup_cv, &fasttrap_cleanup_mtx, 417 0, "ftcl", 0); 418 } 419 420 /* 421 * Wake up the thread in fasttrap_unload() now that we're done. 422 */ 423 wakeup(&fasttrap_cleanup_drain); 424 mtx_unlock(&fasttrap_cleanup_mtx); 425 426 kthread_exit(); 427} 428 429/* 430 * Activates the asynchronous cleanup mechanism. 431 */ 432static void 433fasttrap_pid_cleanup(void) 434{ 435 436 mtx_lock(&fasttrap_cleanup_mtx); 437 if (!fasttrap_cleanup_work) { 438 fasttrap_cleanup_work = 1; 439 wakeup(&fasttrap_cleanup_cv); 440 } 441 mtx_unlock(&fasttrap_cleanup_mtx); 442} 443 444/* 445 * This is called from cfork() via dtrace_fasttrap_fork(). The child 446 * process's address space is (roughly) a copy of the parent process's so 447 * we have to remove all the instrumentation we had previously enabled in the 448 * parent. 449 */ 450static void 451fasttrap_fork(proc_t *p, proc_t *cp) 452{ 453 pid_t ppid = p->p_pid; 454 int i; 455 456#if defined(sun) 457 ASSERT(curproc == p); 458 ASSERT(p->p_proc_flag & P_PR_LOCK); 459#else 460 PROC_LOCK_ASSERT(p, MA_OWNED); 461#endif 462#if defined(sun) 463 ASSERT(p->p_dtrace_count > 0); 464#else 465 if (p->p_dtrace_helpers) { 466 /* 467 * dtrace_helpers_duplicate() allocates memory. 468 */ 469 _PHOLD(cp); 470 PROC_UNLOCK(p); 471 PROC_UNLOCK(cp); 472 dtrace_helpers_duplicate(p, cp); 473 PROC_LOCK(cp); 474 PROC_LOCK(p); 475 _PRELE(cp); 476 } 477 /* 478 * This check is purposely here instead of in kern_fork.c because, 479 * for legal resons, we cannot include the dtrace_cddl.h header 480 * inside kern_fork.c and insert if-clause there. 481 */ 482 if (p->p_dtrace_count == 0) 483 return; 484#endif 485 ASSERT(cp->p_dtrace_count == 0); 486 487 /* 488 * This would be simpler and faster if we maintained per-process 489 * hash tables of enabled tracepoints. It could, however, potentially 490 * slow down execution of a tracepoint since we'd need to go 491 * through two levels of indirection. In the future, we should 492 * consider either maintaining per-process ancillary lists of 493 * enabled tracepoints or hanging a pointer to a per-process hash 494 * table of enabled tracepoints off the proc structure. 495 */ 496 497 /* 498 * We don't have to worry about the child process disappearing 499 * because we're in fork(). 500 */ 501#if defined(sun) 502 mtx_lock_spin(&cp->p_slock); 503 sprlock_proc(cp); 504 mtx_unlock_spin(&cp->p_slock); 505#else 506 /* 507 * fasttrap_tracepoint_remove() expects the child process to be 508 * unlocked and the VM then expects curproc to be unlocked. 509 */ 510 _PHOLD(cp); 511 PROC_UNLOCK(cp); 512 PROC_UNLOCK(p); 513#endif 514 515 /* 516 * Iterate over every tracepoint looking for ones that belong to the 517 * parent process, and remove each from the child process. 518 */ 519 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) { 520 fasttrap_tracepoint_t *tp; 521 fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i]; 522 523 mutex_enter(&bucket->ftb_mtx); 524 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 525 if (tp->ftt_pid == ppid && 526 tp->ftt_proc->ftpc_acount != 0) { 527 int ret = fasttrap_tracepoint_remove(cp, tp); 528 ASSERT(ret == 0); 529 530 /* 531 * The count of active providers can only be 532 * decremented (i.e. to zero) during exec, 533 * exit, and removal of a meta provider so it 534 * should be impossible to drop the count 535 * mid-fork. 536 */ 537 ASSERT(tp->ftt_proc->ftpc_acount != 0); 538 } 539 } 540 mutex_exit(&bucket->ftb_mtx); 541 } 542 543#if defined(sun) 544 mutex_enter(&cp->p_lock); 545 sprunlock(cp); 546#else 547 PROC_LOCK(p); 548 PROC_LOCK(cp); 549 _PRELE(cp); 550#endif 551} 552 553/* 554 * This is called from proc_exit() or from exec_common() if p_dtrace_probes 555 * is set on the proc structure to indicate that there is a pid provider 556 * associated with this process. 557 */ 558static void 559fasttrap_exec_exit(proc_t *p) 560{ 561#if defined(sun) 562 ASSERT(p == curproc); 563#endif 564 PROC_LOCK_ASSERT(p, MA_OWNED); 565 _PHOLD(p); 566 PROC_UNLOCK(p); 567 568 /* 569 * We clean up the pid provider for this process here; user-land 570 * static probes are handled by the meta-provider remove entry point. 571 */ 572 fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0); 573#if !defined(sun) 574 if (p->p_dtrace_helpers) 575 dtrace_helpers_destroy(p); 576#endif 577 PROC_LOCK(p); 578 _PRELE(p); 579} 580 581 582/*ARGSUSED*/ 583static void 584fasttrap_pid_provide(void *arg, dtrace_probedesc_t *desc) 585{ 586 /* 587 * There are no "default" pid probes. 588 */ 589} 590 591static int 592fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 593{ 594 fasttrap_tracepoint_t *tp, *new_tp = NULL; 595 fasttrap_bucket_t *bucket; 596 fasttrap_id_t *id; 597 pid_t pid; 598 uintptr_t pc; 599 600 ASSERT(index < probe->ftp_ntps); 601 602 pid = probe->ftp_pid; 603 pc = probe->ftp_tps[index].fit_tp->ftt_pc; 604 id = &probe->ftp_tps[index].fit_id; 605 606 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 607 608#if defined(sun) 609 ASSERT(!(p->p_flag & SVFORK)); 610#endif 611 612 /* 613 * Before we make any modifications, make sure we've imposed a barrier 614 * on the generation in which this probe was last modified. 615 */ 616 fasttrap_mod_barrier(probe->ftp_gen); 617 618 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 619 620 /* 621 * If the tracepoint has already been enabled, just add our id to the 622 * list of interested probes. This may be our second time through 623 * this path in which case we'll have constructed the tracepoint we'd 624 * like to install. If we can't find a match, and have an allocated 625 * tracepoint ready to go, enable that one now. 626 * 627 * A tracepoint whose process is defunct is also considered defunct. 628 */ 629again: 630 mutex_enter(&bucket->ftb_mtx); 631 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 632 /* 633 * Note that it's safe to access the active count on the 634 * associated proc structure because we know that at least one 635 * provider (this one) will still be around throughout this 636 * operation. 637 */ 638 if (tp->ftt_pid != pid || tp->ftt_pc != pc || 639 tp->ftt_proc->ftpc_acount == 0) 640 continue; 641 642 /* 643 * Now that we've found a matching tracepoint, it would be 644 * a decent idea to confirm that the tracepoint is still 645 * enabled and the trap instruction hasn't been overwritten. 646 * Since this is a little hairy, we'll punt for now. 647 */ 648 649 /* 650 * This can't be the first interested probe. We don't have 651 * to worry about another thread being in the midst of 652 * deleting this tracepoint (which would be the only valid 653 * reason for a tracepoint to have no interested probes) 654 * since we're holding P_PR_LOCK for this process. 655 */ 656 ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL); 657 658 switch (id->fti_ptype) { 659 case DTFTP_ENTRY: 660 case DTFTP_OFFSETS: 661 case DTFTP_IS_ENABLED: 662 id->fti_next = tp->ftt_ids; 663 membar_producer(); 664 tp->ftt_ids = id; 665 membar_producer(); 666 break; 667 668 case DTFTP_RETURN: 669 case DTFTP_POST_OFFSETS: 670 id->fti_next = tp->ftt_retids; 671 membar_producer(); 672 tp->ftt_retids = id; 673 membar_producer(); 674 break; 675 676 default: 677 ASSERT(0); 678 } 679 680 mutex_exit(&bucket->ftb_mtx); 681 682 if (new_tp != NULL) { 683 new_tp->ftt_ids = NULL; 684 new_tp->ftt_retids = NULL; 685 } 686 687 return (0); 688 } 689 690 /* 691 * If we have a good tracepoint ready to go, install it now while 692 * we have the lock held and no one can screw with us. 693 */ 694 if (new_tp != NULL) { 695 int rc = 0; 696 697 new_tp->ftt_next = bucket->ftb_data; 698 membar_producer(); 699 bucket->ftb_data = new_tp; 700 membar_producer(); 701 mutex_exit(&bucket->ftb_mtx); 702 703 /* 704 * Activate the tracepoint in the ISA-specific manner. 705 * If this fails, we need to report the failure, but 706 * indicate that this tracepoint must still be disabled 707 * by calling fasttrap_tracepoint_disable(). 708 */ 709 if (fasttrap_tracepoint_install(p, new_tp) != 0) 710 rc = FASTTRAP_ENABLE_PARTIAL; 711 712 /* 713 * Increment the count of the number of tracepoints active in 714 * the victim process. 715 */ 716#if defined(sun) 717 ASSERT(p->p_proc_flag & P_PR_LOCK); 718#endif 719 p->p_dtrace_count++; 720 721 return (rc); 722 } 723 724 mutex_exit(&bucket->ftb_mtx); 725 726 /* 727 * Initialize the tracepoint that's been preallocated with the probe. 728 */ 729 new_tp = probe->ftp_tps[index].fit_tp; 730 731 ASSERT(new_tp->ftt_pid == pid); 732 ASSERT(new_tp->ftt_pc == pc); 733 ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc); 734 ASSERT(new_tp->ftt_ids == NULL); 735 ASSERT(new_tp->ftt_retids == NULL); 736 737 switch (id->fti_ptype) { 738 case DTFTP_ENTRY: 739 case DTFTP_OFFSETS: 740 case DTFTP_IS_ENABLED: 741 id->fti_next = NULL; 742 new_tp->ftt_ids = id; 743 break; 744 745 case DTFTP_RETURN: 746 case DTFTP_POST_OFFSETS: 747 id->fti_next = NULL; 748 new_tp->ftt_retids = id; 749 break; 750 751 default: 752 ASSERT(0); 753 } 754 755 /* 756 * If the ISA-dependent initialization goes to plan, go back to the 757 * beginning and try to install this freshly made tracepoint. 758 */ 759 if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0) 760 goto again; 761 762 new_tp->ftt_ids = NULL; 763 new_tp->ftt_retids = NULL; 764 765 return (FASTTRAP_ENABLE_FAIL); 766} 767 768static void 769fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 770{ 771 fasttrap_bucket_t *bucket; 772 fasttrap_provider_t *provider = probe->ftp_prov; 773 fasttrap_tracepoint_t **pp, *tp; 774 fasttrap_id_t *id, **idp = NULL; 775 pid_t pid; 776 uintptr_t pc; 777 778 ASSERT(index < probe->ftp_ntps); 779 780 pid = probe->ftp_pid; 781 pc = probe->ftp_tps[index].fit_tp->ftt_pc; 782 id = &probe->ftp_tps[index].fit_id; 783 784 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 785 786 /* 787 * Find the tracepoint and make sure that our id is one of the 788 * ones registered with it. 789 */ 790 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 791 mutex_enter(&bucket->ftb_mtx); 792 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 793 if (tp->ftt_pid == pid && tp->ftt_pc == pc && 794 tp->ftt_proc == provider->ftp_proc) 795 break; 796 } 797 798 /* 799 * If we somehow lost this tracepoint, we're in a world of hurt. 800 */ 801 ASSERT(tp != NULL); 802 803 switch (id->fti_ptype) { 804 case DTFTP_ENTRY: 805 case DTFTP_OFFSETS: 806 case DTFTP_IS_ENABLED: 807 ASSERT(tp->ftt_ids != NULL); 808 idp = &tp->ftt_ids; 809 break; 810 811 case DTFTP_RETURN: 812 case DTFTP_POST_OFFSETS: 813 ASSERT(tp->ftt_retids != NULL); 814 idp = &tp->ftt_retids; 815 break; 816 817 default: 818 ASSERT(0); 819 } 820 821 while ((*idp)->fti_probe != probe) { 822 idp = &(*idp)->fti_next; 823 ASSERT(*idp != NULL); 824 } 825 826 id = *idp; 827 *idp = id->fti_next; 828 membar_producer(); 829 830 ASSERT(id->fti_probe == probe); 831 832 /* 833 * If there are other registered enablings of this tracepoint, we're 834 * all done, but if this was the last probe assocated with this 835 * this tracepoint, we need to remove and free it. 836 */ 837 if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) { 838 839 /* 840 * If the current probe's tracepoint is in use, swap it 841 * for an unused tracepoint. 842 */ 843 if (tp == probe->ftp_tps[index].fit_tp) { 844 fasttrap_probe_t *tmp_probe; 845 fasttrap_tracepoint_t **tmp_tp; 846 uint_t tmp_index; 847 848 if (tp->ftt_ids != NULL) { 849 tmp_probe = tp->ftt_ids->fti_probe; 850 /* LINTED - alignment */ 851 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids); 852 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 853 } else { 854 tmp_probe = tp->ftt_retids->fti_probe; 855 /* LINTED - alignment */ 856 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids); 857 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 858 } 859 860 ASSERT(*tmp_tp != NULL); 861 ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp); 862 ASSERT((*tmp_tp)->ftt_ids == NULL); 863 ASSERT((*tmp_tp)->ftt_retids == NULL); 864 865 probe->ftp_tps[index].fit_tp = *tmp_tp; 866 *tmp_tp = tp; 867 } 868 869 mutex_exit(&bucket->ftb_mtx); 870 871 /* 872 * Tag the modified probe with the generation in which it was 873 * changed. 874 */ 875 probe->ftp_gen = fasttrap_mod_gen; 876 return; 877 } 878 879 mutex_exit(&bucket->ftb_mtx); 880 881 /* 882 * We can't safely remove the tracepoint from the set of active 883 * tracepoints until we've actually removed the fasttrap instruction 884 * from the process's text. We can, however, operate on this 885 * tracepoint secure in the knowledge that no other thread is going to 886 * be looking at it since we hold P_PR_LOCK on the process if it's 887 * live or we hold the provider lock on the process if it's dead and 888 * gone. 889 */ 890 891 /* 892 * We only need to remove the actual instruction if we're looking 893 * at an existing process 894 */ 895 if (p != NULL) { 896 /* 897 * If we fail to restore the instruction we need to kill 898 * this process since it's in a completely unrecoverable 899 * state. 900 */ 901 if (fasttrap_tracepoint_remove(p, tp) != 0) 902 fasttrap_sigtrap(p, NULL, pc); 903 904 /* 905 * Decrement the count of the number of tracepoints active 906 * in the victim process. 907 */ 908#if defined(sun) 909 ASSERT(p->p_proc_flag & P_PR_LOCK); 910#endif 911 p->p_dtrace_count--; 912 } 913 914 /* 915 * Remove the probe from the hash table of active tracepoints. 916 */ 917 mutex_enter(&bucket->ftb_mtx); 918 pp = (fasttrap_tracepoint_t **)&bucket->ftb_data; 919 ASSERT(*pp != NULL); 920 while (*pp != tp) { 921 pp = &(*pp)->ftt_next; 922 ASSERT(*pp != NULL); 923 } 924 925 *pp = tp->ftt_next; 926 membar_producer(); 927 928 mutex_exit(&bucket->ftb_mtx); 929 930 /* 931 * Tag the modified probe with the generation in which it was changed. 932 */ 933 probe->ftp_gen = fasttrap_mod_gen; 934} 935 936static void 937fasttrap_enable_callbacks(void) 938{ 939 /* 940 * We don't have to play the rw lock game here because we're 941 * providing something rather than taking something away -- 942 * we can be sure that no threads have tried to follow this 943 * function pointer yet. 944 */ 945 mutex_enter(&fasttrap_count_mtx); 946 if (fasttrap_pid_count == 0) { 947 ASSERT(dtrace_pid_probe_ptr == NULL); 948 ASSERT(dtrace_return_probe_ptr == NULL); 949 dtrace_pid_probe_ptr = &fasttrap_pid_probe; 950 dtrace_return_probe_ptr = &fasttrap_return_probe; 951 } 952 ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe); 953 ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe); 954 fasttrap_pid_count++; 955 mutex_exit(&fasttrap_count_mtx); 956} 957 958static void 959fasttrap_disable_callbacks(void) 960{ 961#if defined(sun) 962 ASSERT(MUTEX_HELD(&cpu_lock)); 963#endif 964 965 966 mutex_enter(&fasttrap_count_mtx); 967 ASSERT(fasttrap_pid_count > 0); 968 fasttrap_pid_count--; 969 if (fasttrap_pid_count == 0) { 970#if defined(sun) 971 cpu_t *cur, *cpu = CPU; 972 973 for (cur = cpu->cpu_next_onln; cur != cpu; 974 cur = cur->cpu_next_onln) { 975 rw_enter(&cur->cpu_ft_lock, RW_WRITER); 976 } 977#endif 978 dtrace_pid_probe_ptr = NULL; 979 dtrace_return_probe_ptr = NULL; 980#if defined(sun) 981 for (cur = cpu->cpu_next_onln; cur != cpu; 982 cur = cur->cpu_next_onln) { 983 rw_exit(&cur->cpu_ft_lock); 984 } 985#endif 986 } 987 mutex_exit(&fasttrap_count_mtx); 988} 989 990/*ARGSUSED*/ 991static void 992fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg) 993{ 994 fasttrap_probe_t *probe = parg; 995 proc_t *p = NULL; 996 int i, rc; 997 998 ASSERT(probe != NULL); 999 ASSERT(!probe->ftp_enabled); 1000 ASSERT(id == probe->ftp_id); 1001#if defined(sun) 1002 ASSERT(MUTEX_HELD(&cpu_lock)); 1003#endif 1004 1005 /* 1006 * Increment the count of enabled probes on this probe's provider; 1007 * the provider can't go away while the probe still exists. We 1008 * must increment this even if we aren't able to properly enable 1009 * this probe. 1010 */ 1011 mutex_enter(&probe->ftp_prov->ftp_mtx); 1012 probe->ftp_prov->ftp_rcount++; 1013 mutex_exit(&probe->ftp_prov->ftp_mtx); 1014 1015 /* 1016 * If this probe's provider is retired (meaning it was valid in a 1017 * previously exec'ed incarnation of this address space), bail out. The 1018 * provider can't go away while we're in this code path. 1019 */ 1020 if (probe->ftp_prov->ftp_retired) 1021 return; 1022 1023 /* 1024 * If we can't find the process, it may be that we're in the context of 1025 * a fork in which the traced process is being born and we're copying 1026 * USDT probes. Otherwise, the process is gone so bail. 1027 */ 1028#if defined(sun) 1029 if ((p = sprlock(probe->ftp_pid)) == NULL) { 1030 if ((curproc->p_flag & SFORKING) == 0) 1031 return; 1032 1033 mutex_enter(&pidlock); 1034 p = prfind(probe->ftp_pid); 1035 1036 /* 1037 * Confirm that curproc is indeed forking the process in which 1038 * we're trying to enable probes. 1039 */ 1040 ASSERT(p != NULL); 1041 ASSERT(p->p_parent == curproc); 1042 ASSERT(p->p_stat == SIDL); 1043 1044 mutex_enter(&p->p_lock); 1045 mutex_exit(&pidlock); 1046 1047 sprlock_proc(p); 1048 } 1049 1050 ASSERT(!(p->p_flag & SVFORK)); 1051 mutex_exit(&p->p_lock); 1052#else 1053 if ((p = pfind(probe->ftp_pid)) == NULL) 1054 return; 1055#endif 1056 1057 /* 1058 * We have to enable the trap entry point before any user threads have 1059 * the chance to execute the trap instruction we're about to place 1060 * in their process's text. 1061 */ 1062#ifdef __FreeBSD__ 1063 /* 1064 * pfind() returns a locked process. 1065 */ 1066 _PHOLD(p); 1067 PROC_UNLOCK(p); 1068#endif 1069 fasttrap_enable_callbacks(); 1070 1071 /* 1072 * Enable all the tracepoints and add this probe's id to each 1073 * tracepoint's list of active probes. 1074 */ 1075 for (i = 0; i < probe->ftp_ntps; i++) { 1076 if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) { 1077 /* 1078 * If enabling the tracepoint failed completely, 1079 * we don't have to disable it; if the failure 1080 * was only partial we must disable it. 1081 */ 1082 if (rc == FASTTRAP_ENABLE_FAIL) 1083 i--; 1084 else 1085 ASSERT(rc == FASTTRAP_ENABLE_PARTIAL); 1086 1087 /* 1088 * Back up and pull out all the tracepoints we've 1089 * created so far for this probe. 1090 */ 1091 while (i >= 0) { 1092 fasttrap_tracepoint_disable(p, probe, i); 1093 i--; 1094 } 1095 1096#if defined(sun) 1097 mutex_enter(&p->p_lock); 1098 sprunlock(p); 1099#else 1100 PRELE(p); 1101#endif 1102 1103 /* 1104 * Since we're not actually enabling this probe, 1105 * drop our reference on the trap table entry. 1106 */ 1107 fasttrap_disable_callbacks(); 1108 return; 1109 } 1110 } 1111#if defined(sun) 1112 mutex_enter(&p->p_lock); 1113 sprunlock(p); 1114#else 1115 PRELE(p); 1116#endif 1117 1118 probe->ftp_enabled = 1; 1119} 1120 1121/*ARGSUSED*/ 1122static void 1123fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg) 1124{ 1125 fasttrap_probe_t *probe = parg; 1126 fasttrap_provider_t *provider = probe->ftp_prov; 1127 proc_t *p; 1128 int i, whack = 0; 1129 1130 ASSERT(id == probe->ftp_id); 1131 1132 mutex_enter(&provider->ftp_mtx); 1133 1134 /* 1135 * We won't be able to acquire a /proc-esque lock on the process 1136 * iff the process is dead and gone. In this case, we rely on the 1137 * provider lock as a point of mutual exclusion to prevent other 1138 * DTrace consumers from disabling this probe. 1139 */ 1140 if ((p = pfind(probe->ftp_pid)) != NULL) { 1141#ifdef __FreeBSD__ 1142 _PHOLD(p); 1143 PROC_UNLOCK(p); 1144#endif 1145 } 1146 1147 /* 1148 * Disable all the associated tracepoints (for fully enabled probes). 1149 */ 1150 if (probe->ftp_enabled) { 1151 for (i = 0; i < probe->ftp_ntps; i++) { 1152 fasttrap_tracepoint_disable(p, probe, i); 1153 } 1154 } 1155 1156 ASSERT(provider->ftp_rcount > 0); 1157 provider->ftp_rcount--; 1158 1159 if (p != NULL) { 1160 /* 1161 * Even though we may not be able to remove it entirely, we 1162 * mark this retired provider to get a chance to remove some 1163 * of the associated probes. 1164 */ 1165 if (provider->ftp_retired && !provider->ftp_marked) 1166 whack = provider->ftp_marked = 1; 1167 mutex_exit(&provider->ftp_mtx); 1168 } else { 1169 /* 1170 * If the process is dead, we're just waiting for the 1171 * last probe to be disabled to be able to free it. 1172 */ 1173 if (provider->ftp_rcount == 0 && !provider->ftp_marked) 1174 whack = provider->ftp_marked = 1; 1175 mutex_exit(&provider->ftp_mtx); 1176 } 1177 1178 if (whack) 1179 fasttrap_pid_cleanup(); 1180 1181#ifdef __FreeBSD__ 1182 if (p != NULL) 1183 PRELE(p); 1184#endif 1185 if (!probe->ftp_enabled) 1186 return; 1187 1188 probe->ftp_enabled = 0; 1189 1190#if defined(sun) 1191 ASSERT(MUTEX_HELD(&cpu_lock)); 1192#endif 1193 fasttrap_disable_callbacks(); 1194} 1195 1196/*ARGSUSED*/ 1197static void 1198fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg, 1199 dtrace_argdesc_t *desc) 1200{ 1201 fasttrap_probe_t *probe = parg; 1202 char *str; 1203 int i, ndx; 1204 1205 desc->dtargd_native[0] = '\0'; 1206 desc->dtargd_xlate[0] = '\0'; 1207 1208 if (probe->ftp_prov->ftp_retired != 0 || 1209 desc->dtargd_ndx >= probe->ftp_nargs) { 1210 desc->dtargd_ndx = DTRACE_ARGNONE; 1211 return; 1212 } 1213 1214 ndx = (probe->ftp_argmap != NULL) ? 1215 probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx; 1216 1217 str = probe->ftp_ntypes; 1218 for (i = 0; i < ndx; i++) { 1219 str += strlen(str) + 1; 1220 } 1221 1222 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native)); 1223 (void) strcpy(desc->dtargd_native, str); 1224 1225 if (probe->ftp_xtypes == NULL) 1226 return; 1227 1228 str = probe->ftp_xtypes; 1229 for (i = 0; i < desc->dtargd_ndx; i++) { 1230 str += strlen(str) + 1; 1231 } 1232 1233 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate)); 1234 (void) strcpy(desc->dtargd_xlate, str); 1235} 1236 1237/*ARGSUSED*/ 1238static void 1239fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg) 1240{ 1241 fasttrap_probe_t *probe = parg; 1242 int i; 1243 size_t size; 1244 1245 ASSERT(probe != NULL); 1246 ASSERT(!probe->ftp_enabled); 1247 ASSERT(fasttrap_total >= probe->ftp_ntps); 1248 1249 atomic_add_32(&fasttrap_total, -probe->ftp_ntps); 1250 size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]); 1251 1252 if (probe->ftp_gen + 1 >= fasttrap_mod_gen) 1253 fasttrap_mod_barrier(probe->ftp_gen); 1254 1255 for (i = 0; i < probe->ftp_ntps; i++) { 1256 kmem_free(probe->ftp_tps[i].fit_tp, 1257 sizeof (fasttrap_tracepoint_t)); 1258 } 1259 1260 kmem_free(probe, size); 1261} 1262 1263 1264static const dtrace_pattr_t pid_attr = { 1265{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1266{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1267{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1268{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1269{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1270}; 1271 1272static dtrace_pops_t pid_pops = { 1273 fasttrap_pid_provide, 1274 NULL, 1275 fasttrap_pid_enable, 1276 fasttrap_pid_disable, 1277 NULL, 1278 NULL, 1279 fasttrap_pid_getargdesc, 1280 fasttrap_pid_getarg, 1281 NULL, 1282 fasttrap_pid_destroy 1283}; 1284 1285static dtrace_pops_t usdt_pops = { 1286 fasttrap_pid_provide, 1287 NULL, 1288 fasttrap_pid_enable, 1289 fasttrap_pid_disable, 1290 NULL, 1291 NULL, 1292 fasttrap_pid_getargdesc, 1293 fasttrap_usdt_getarg, 1294 NULL, 1295 fasttrap_pid_destroy 1296}; 1297 1298static fasttrap_proc_t * 1299fasttrap_proc_lookup(pid_t pid) 1300{ 1301 fasttrap_bucket_t *bucket; 1302 fasttrap_proc_t *fprc, *new_fprc; 1303 1304 1305 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1306 mutex_enter(&bucket->ftb_mtx); 1307 1308 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1309 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1310 mutex_enter(&fprc->ftpc_mtx); 1311 mutex_exit(&bucket->ftb_mtx); 1312 fprc->ftpc_rcount++; 1313 atomic_add_64(&fprc->ftpc_acount, 1); 1314 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1315 mutex_exit(&fprc->ftpc_mtx); 1316 1317 return (fprc); 1318 } 1319 } 1320 1321 /* 1322 * Drop the bucket lock so we don't try to perform a sleeping 1323 * allocation under it. 1324 */ 1325 mutex_exit(&bucket->ftb_mtx); 1326 1327 new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP); 1328 new_fprc->ftpc_pid = pid; 1329 new_fprc->ftpc_rcount = 1; 1330 new_fprc->ftpc_acount = 1; 1331#if !defined(sun) 1332 mutex_init(&new_fprc->ftpc_mtx, "fasttrap proc mtx", MUTEX_DEFAULT, 1333 NULL); 1334#endif 1335 1336 mutex_enter(&bucket->ftb_mtx); 1337 1338 /* 1339 * Take another lap through the list to make sure a proc hasn't 1340 * been created for this pid while we weren't under the bucket lock. 1341 */ 1342 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1343 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1344 mutex_enter(&fprc->ftpc_mtx); 1345 mutex_exit(&bucket->ftb_mtx); 1346 fprc->ftpc_rcount++; 1347 atomic_add_64(&fprc->ftpc_acount, 1); 1348 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1349 mutex_exit(&fprc->ftpc_mtx); 1350 1351 kmem_free(new_fprc, sizeof (fasttrap_proc_t)); 1352 1353 return (fprc); 1354 } 1355 } 1356 1357 new_fprc->ftpc_next = bucket->ftb_data; 1358 bucket->ftb_data = new_fprc; 1359 1360 mutex_exit(&bucket->ftb_mtx); 1361 1362 return (new_fprc); 1363} 1364 1365static void 1366fasttrap_proc_release(fasttrap_proc_t *proc) 1367{ 1368 fasttrap_bucket_t *bucket; 1369 fasttrap_proc_t *fprc, **fprcp; 1370 pid_t pid = proc->ftpc_pid; 1371 1372 mutex_enter(&proc->ftpc_mtx); 1373 1374 ASSERT(proc->ftpc_rcount != 0); 1375 ASSERT(proc->ftpc_acount <= proc->ftpc_rcount); 1376 1377 if (--proc->ftpc_rcount != 0) { 1378 mutex_exit(&proc->ftpc_mtx); 1379 return; 1380 } 1381 1382 mutex_exit(&proc->ftpc_mtx); 1383 1384 /* 1385 * There should definitely be no live providers associated with this 1386 * process at this point. 1387 */ 1388 ASSERT(proc->ftpc_acount == 0); 1389 1390 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1391 mutex_enter(&bucket->ftb_mtx); 1392 1393 fprcp = (fasttrap_proc_t **)&bucket->ftb_data; 1394 while ((fprc = *fprcp) != NULL) { 1395 if (fprc == proc) 1396 break; 1397 1398 fprcp = &fprc->ftpc_next; 1399 } 1400 1401 /* 1402 * Something strange has happened if we can't find the proc. 1403 */ 1404 ASSERT(fprc != NULL); 1405 1406 *fprcp = fprc->ftpc_next; 1407 1408 mutex_exit(&bucket->ftb_mtx); 1409 1410 kmem_free(fprc, sizeof (fasttrap_proc_t)); 1411} 1412 1413/* 1414 * Lookup a fasttrap-managed provider based on its name and associated pid. 1415 * If the pattr argument is non-NULL, this function instantiates the provider 1416 * if it doesn't exist otherwise it returns NULL. The provider is returned 1417 * with its lock held. 1418 */ 1419static fasttrap_provider_t * 1420fasttrap_provider_lookup(pid_t pid, const char *name, 1421 const dtrace_pattr_t *pattr) 1422{ 1423 fasttrap_provider_t *fp, *new_fp = NULL; 1424 fasttrap_bucket_t *bucket; 1425 char provname[DTRACE_PROVNAMELEN]; 1426 proc_t *p; 1427 cred_t *cred; 1428 1429 ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1430 ASSERT(pattr != NULL); 1431 1432 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1433 mutex_enter(&bucket->ftb_mtx); 1434 1435 /* 1436 * Take a lap through the list and return the match if we find it. 1437 */ 1438 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1439 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1440 !fp->ftp_retired) { 1441 mutex_enter(&fp->ftp_mtx); 1442 mutex_exit(&bucket->ftb_mtx); 1443 return (fp); 1444 } 1445 } 1446 1447 /* 1448 * Drop the bucket lock so we don't try to perform a sleeping 1449 * allocation under it. 1450 */ 1451 mutex_exit(&bucket->ftb_mtx); 1452 1453 /* 1454 * Make sure the process exists, isn't a child created as the result 1455 * of a vfork(2), and isn't a zombie (but may be in fork). 1456 */ 1457 if ((p = pfind(pid)) == NULL) 1458 return (NULL); 1459 1460 /* 1461 * Increment p_dtrace_probes so that the process knows to inform us 1462 * when it exits or execs. fasttrap_provider_free() decrements this 1463 * when we're done with this provider. 1464 */ 1465 p->p_dtrace_probes++; 1466 1467 /* 1468 * Grab the credentials for this process so we have 1469 * something to pass to dtrace_register(). 1470 */ 1471 PROC_LOCK_ASSERT(p, MA_OWNED); 1472 crhold(p->p_ucred); 1473 cred = p->p_ucred; 1474 PROC_UNLOCK(p); 1475 1476 new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP); 1477 new_fp->ftp_pid = pid; 1478 new_fp->ftp_proc = fasttrap_proc_lookup(pid); 1479#if !defined(sun) 1480 mutex_init(&new_fp->ftp_mtx, "provider mtx", MUTEX_DEFAULT, NULL); 1481 mutex_init(&new_fp->ftp_cmtx, "lock on creating", MUTEX_DEFAULT, NULL); 1482#endif 1483 1484 ASSERT(new_fp->ftp_proc != NULL); 1485 1486 mutex_enter(&bucket->ftb_mtx); 1487 1488 /* 1489 * Take another lap through the list to make sure a provider hasn't 1490 * been created for this pid while we weren't under the bucket lock. 1491 */ 1492 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1493 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1494 !fp->ftp_retired) { 1495 mutex_enter(&fp->ftp_mtx); 1496 mutex_exit(&bucket->ftb_mtx); 1497 fasttrap_provider_free(new_fp); 1498 crfree(cred); 1499 return (fp); 1500 } 1501 } 1502 1503 (void) strcpy(new_fp->ftp_name, name); 1504 1505 /* 1506 * Fail and return NULL if either the provider name is too long 1507 * or we fail to register this new provider with the DTrace 1508 * framework. Note that this is the only place we ever construct 1509 * the full provider name -- we keep it in pieces in the provider 1510 * structure. 1511 */ 1512 if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >= 1513 sizeof (provname) || 1514 dtrace_register(provname, pattr, 1515 DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred, 1516 pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp, 1517 &new_fp->ftp_provid) != 0) { 1518 mutex_exit(&bucket->ftb_mtx); 1519 fasttrap_provider_free(new_fp); 1520 crfree(cred); 1521 return (NULL); 1522 } 1523 1524 new_fp->ftp_next = bucket->ftb_data; 1525 bucket->ftb_data = new_fp; 1526 1527 mutex_enter(&new_fp->ftp_mtx); 1528 mutex_exit(&bucket->ftb_mtx); 1529 1530 crfree(cred); 1531 return (new_fp); 1532} 1533 1534static void 1535fasttrap_provider_free(fasttrap_provider_t *provider) 1536{ 1537 pid_t pid = provider->ftp_pid; 1538 proc_t *p; 1539 1540 /* 1541 * There need to be no associated enabled probes, no consumers 1542 * creating probes, and no meta providers referencing this provider. 1543 */ 1544 ASSERT(provider->ftp_rcount == 0); 1545 ASSERT(provider->ftp_ccount == 0); 1546 ASSERT(provider->ftp_mcount == 0); 1547 1548 /* 1549 * If this provider hasn't been retired, we need to explicitly drop the 1550 * count of active providers on the associated process structure. 1551 */ 1552 if (!provider->ftp_retired) { 1553 atomic_add_64(&provider->ftp_proc->ftpc_acount, -1); 1554 ASSERT(provider->ftp_proc->ftpc_acount < 1555 provider->ftp_proc->ftpc_rcount); 1556 } 1557 1558 fasttrap_proc_release(provider->ftp_proc); 1559 1560#if !defined(sun) 1561 mutex_destroy(&provider->ftp_mtx); 1562 mutex_destroy(&provider->ftp_cmtx); 1563#endif 1564 kmem_free(provider, sizeof (fasttrap_provider_t)); 1565 1566 /* 1567 * Decrement p_dtrace_probes on the process whose provider we're 1568 * freeing. We don't have to worry about clobbering somone else's 1569 * modifications to it because we have locked the bucket that 1570 * corresponds to this process's hash chain in the provider hash 1571 * table. Don't sweat it if we can't find the process. 1572 */ 1573 if ((p = pfind(pid)) == NULL) { 1574 return; 1575 } 1576 1577 p->p_dtrace_probes--; 1578#if !defined(sun) 1579 PROC_UNLOCK(p); 1580#endif 1581} 1582 1583static void 1584fasttrap_provider_retire(pid_t pid, const char *name, int mprov) 1585{ 1586 fasttrap_provider_t *fp; 1587 fasttrap_bucket_t *bucket; 1588 dtrace_provider_id_t provid; 1589 1590 ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1591 1592 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1593 mutex_enter(&bucket->ftb_mtx); 1594 1595 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1596 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1597 !fp->ftp_retired) 1598 break; 1599 } 1600 1601 if (fp == NULL) { 1602 mutex_exit(&bucket->ftb_mtx); 1603 return; 1604 } 1605 1606 mutex_enter(&fp->ftp_mtx); 1607 ASSERT(!mprov || fp->ftp_mcount > 0); 1608 if (mprov && --fp->ftp_mcount != 0) { 1609 mutex_exit(&fp->ftp_mtx); 1610 mutex_exit(&bucket->ftb_mtx); 1611 return; 1612 } 1613 1614 /* 1615 * Mark the provider to be removed in our post-processing step, mark it 1616 * retired, and drop the active count on its proc. Marking it indicates 1617 * that we should try to remove it; setting the retired flag indicates 1618 * that we're done with this provider; dropping the active the proc 1619 * releases our hold, and when this reaches zero (as it will during 1620 * exit or exec) the proc and associated providers become defunct. 1621 * 1622 * We obviously need to take the bucket lock before the provider lock 1623 * to perform the lookup, but we need to drop the provider lock 1624 * before calling into the DTrace framework since we acquire the 1625 * provider lock in callbacks invoked from the DTrace framework. The 1626 * bucket lock therefore protects the integrity of the provider hash 1627 * table. 1628 */ 1629 atomic_add_64(&fp->ftp_proc->ftpc_acount, -1); 1630 ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount); 1631 1632 fp->ftp_retired = 1; 1633 fp->ftp_marked = 1; 1634 provid = fp->ftp_provid; 1635 mutex_exit(&fp->ftp_mtx); 1636 1637 /* 1638 * We don't have to worry about invalidating the same provider twice 1639 * since fasttrap_provider_lookup() will ignore provider that have 1640 * been marked as retired. 1641 */ 1642 dtrace_invalidate(provid); 1643 1644 mutex_exit(&bucket->ftb_mtx); 1645 1646 fasttrap_pid_cleanup(); 1647} 1648 1649static int 1650fasttrap_uint32_cmp(const void *ap, const void *bp) 1651{ 1652 return (*(const uint32_t *)ap - *(const uint32_t *)bp); 1653} 1654 1655static int 1656fasttrap_uint64_cmp(const void *ap, const void *bp) 1657{ 1658 return (*(const uint64_t *)ap - *(const uint64_t *)bp); 1659} 1660 1661static int 1662fasttrap_add_probe(fasttrap_probe_spec_t *pdata) 1663{ 1664 fasttrap_provider_t *provider; 1665 fasttrap_probe_t *pp; 1666 fasttrap_tracepoint_t *tp; 1667 char *name; 1668 int i, aframes = 0, whack; 1669 1670 /* 1671 * There needs to be at least one desired trace point. 1672 */ 1673 if (pdata->ftps_noffs == 0) 1674 return (EINVAL); 1675 1676 switch (pdata->ftps_type) { 1677 case DTFTP_ENTRY: 1678 name = "entry"; 1679 aframes = FASTTRAP_ENTRY_AFRAMES; 1680 break; 1681 case DTFTP_RETURN: 1682 name = "return"; 1683 aframes = FASTTRAP_RETURN_AFRAMES; 1684 break; 1685 case DTFTP_OFFSETS: 1686 name = NULL; 1687 break; 1688 default: 1689 return (EINVAL); 1690 } 1691 1692 if ((provider = fasttrap_provider_lookup(pdata->ftps_pid, 1693 FASTTRAP_PID_NAME, &pid_attr)) == NULL) 1694 return (ESRCH); 1695 1696 /* 1697 * Increment this reference count to indicate that a consumer is 1698 * actively adding a new probe associated with this provider. This 1699 * prevents the provider from being deleted -- we'll need to check 1700 * for pending deletions when we drop this reference count. 1701 */ 1702 provider->ftp_ccount++; 1703 mutex_exit(&provider->ftp_mtx); 1704 1705 /* 1706 * Grab the creation lock to ensure consistency between calls to 1707 * dtrace_probe_lookup() and dtrace_probe_create() in the face of 1708 * other threads creating probes. We must drop the provider lock 1709 * before taking this lock to avoid a three-way deadlock with the 1710 * DTrace framework. 1711 */ 1712 mutex_enter(&provider->ftp_cmtx); 1713 1714 if (name == NULL) { 1715 for (i = 0; i < pdata->ftps_noffs; i++) { 1716 char name_str[17]; 1717 1718 (void) sprintf(name_str, "%llx", 1719 (unsigned long long)pdata->ftps_offs[i]); 1720 1721 if (dtrace_probe_lookup(provider->ftp_provid, 1722 pdata->ftps_mod, pdata->ftps_func, name_str) != 0) 1723 continue; 1724 1725 atomic_add_32(&fasttrap_total, 1); 1726 1727 if (fasttrap_total > fasttrap_max) { 1728 atomic_add_32(&fasttrap_total, -1); 1729 goto no_mem; 1730 } 1731 1732 pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP); 1733 1734 pp->ftp_prov = provider; 1735 pp->ftp_faddr = pdata->ftps_pc; 1736 pp->ftp_fsize = pdata->ftps_size; 1737 pp->ftp_pid = pdata->ftps_pid; 1738 pp->ftp_ntps = 1; 1739 1740 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1741 KM_SLEEP); 1742 1743 tp->ftt_proc = provider->ftp_proc; 1744 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 1745 tp->ftt_pid = pdata->ftps_pid; 1746 1747 pp->ftp_tps[0].fit_tp = tp; 1748 pp->ftp_tps[0].fit_id.fti_probe = pp; 1749 pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type; 1750 1751 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 1752 pdata->ftps_mod, pdata->ftps_func, name_str, 1753 FASTTRAP_OFFSET_AFRAMES, pp); 1754 } 1755 1756 } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod, 1757 pdata->ftps_func, name) == 0) { 1758 atomic_add_32(&fasttrap_total, pdata->ftps_noffs); 1759 1760 if (fasttrap_total > fasttrap_max) { 1761 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1762 goto no_mem; 1763 } 1764 1765 /* 1766 * Make sure all tracepoint program counter values are unique. 1767 * We later assume that each probe has exactly one tracepoint 1768 * for a given pc. 1769 */ 1770 qsort(pdata->ftps_offs, pdata->ftps_noffs, 1771 sizeof (uint64_t), fasttrap_uint64_cmp); 1772 for (i = 1; i < pdata->ftps_noffs; i++) { 1773 if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1]) 1774 continue; 1775 1776 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1777 goto no_mem; 1778 } 1779 1780 ASSERT(pdata->ftps_noffs > 0); 1781 pp = kmem_zalloc(offsetof(fasttrap_probe_t, 1782 ftp_tps[pdata->ftps_noffs]), KM_SLEEP); 1783 1784 pp->ftp_prov = provider; 1785 pp->ftp_faddr = pdata->ftps_pc; 1786 pp->ftp_fsize = pdata->ftps_size; 1787 pp->ftp_pid = pdata->ftps_pid; 1788 pp->ftp_ntps = pdata->ftps_noffs; 1789 1790 for (i = 0; i < pdata->ftps_noffs; i++) { 1791 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1792 KM_SLEEP); 1793 1794 tp->ftt_proc = provider->ftp_proc; 1795 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 1796 tp->ftt_pid = pdata->ftps_pid; 1797 1798 pp->ftp_tps[i].fit_tp = tp; 1799 pp->ftp_tps[i].fit_id.fti_probe = pp; 1800 pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type; 1801 } 1802 1803 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 1804 pdata->ftps_mod, pdata->ftps_func, name, aframes, pp); 1805 } 1806 1807 mutex_exit(&provider->ftp_cmtx); 1808 1809 /* 1810 * We know that the provider is still valid since we incremented the 1811 * creation reference count. If someone tried to clean up this provider 1812 * while we were using it (e.g. because the process called exec(2) or 1813 * exit(2)), take note of that and try to clean it up now. 1814 */ 1815 mutex_enter(&provider->ftp_mtx); 1816 provider->ftp_ccount--; 1817 whack = provider->ftp_retired; 1818 mutex_exit(&provider->ftp_mtx); 1819 1820 if (whack) 1821 fasttrap_pid_cleanup(); 1822 1823 return (0); 1824 1825no_mem: 1826 /* 1827 * If we've exhausted the allowable resources, we'll try to remove 1828 * this provider to free some up. This is to cover the case where 1829 * the user has accidentally created many more probes than was 1830 * intended (e.g. pid123:::). 1831 */ 1832 mutex_exit(&provider->ftp_cmtx); 1833 mutex_enter(&provider->ftp_mtx); 1834 provider->ftp_ccount--; 1835 provider->ftp_marked = 1; 1836 mutex_exit(&provider->ftp_mtx); 1837 1838 fasttrap_pid_cleanup(); 1839 1840 return (ENOMEM); 1841} 1842 1843/*ARGSUSED*/ 1844static void * 1845fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 1846{ 1847 fasttrap_provider_t *provider; 1848 1849 /* 1850 * A 32-bit unsigned integer (like a pid for example) can be 1851 * expressed in 10 or fewer decimal digits. Make sure that we'll 1852 * have enough space for the provider name. 1853 */ 1854 if (strlen(dhpv->dthpv_provname) + 10 >= 1855 sizeof (provider->ftp_name)) { 1856 printf("failed to instantiate provider %s: " 1857 "name too long to accomodate pid", dhpv->dthpv_provname); 1858 return (NULL); 1859 } 1860 1861 /* 1862 * Don't let folks spoof the true pid provider. 1863 */ 1864 if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) { 1865 printf("failed to instantiate provider %s: " 1866 "%s is an invalid name", dhpv->dthpv_provname, 1867 FASTTRAP_PID_NAME); 1868 return (NULL); 1869 } 1870 1871 /* 1872 * The highest stability class that fasttrap supports is ISA; cap 1873 * the stability of the new provider accordingly. 1874 */ 1875 if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA) 1876 dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA; 1877 if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA) 1878 dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA; 1879 if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA) 1880 dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA; 1881 if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA) 1882 dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA; 1883 if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA) 1884 dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA; 1885 1886 if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname, 1887 &dhpv->dthpv_pattr)) == NULL) { 1888 printf("failed to instantiate provider %s for " 1889 "process %u", dhpv->dthpv_provname, (uint_t)pid); 1890 return (NULL); 1891 } 1892 1893 /* 1894 * Up the meta provider count so this provider isn't removed until 1895 * the meta provider has been told to remove it. 1896 */ 1897 provider->ftp_mcount++; 1898 1899 mutex_exit(&provider->ftp_mtx); 1900 1901 return (provider); 1902} 1903 1904/*ARGSUSED*/ 1905static void 1906fasttrap_meta_create_probe(void *arg, void *parg, 1907 dtrace_helper_probedesc_t *dhpb) 1908{ 1909 fasttrap_provider_t *provider = parg; 1910 fasttrap_probe_t *pp; 1911 fasttrap_tracepoint_t *tp; 1912 int i, j; 1913 uint32_t ntps; 1914 1915 /* 1916 * Since the meta provider count is non-zero we don't have to worry 1917 * about this provider disappearing. 1918 */ 1919 ASSERT(provider->ftp_mcount > 0); 1920 1921 /* 1922 * The offsets must be unique. 1923 */ 1924 qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t), 1925 fasttrap_uint32_cmp); 1926 for (i = 1; i < dhpb->dthpb_noffs; i++) { 1927 if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <= 1928 dhpb->dthpb_base + dhpb->dthpb_offs[i - 1]) 1929 return; 1930 } 1931 1932 qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t), 1933 fasttrap_uint32_cmp); 1934 for (i = 1; i < dhpb->dthpb_nenoffs; i++) { 1935 if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <= 1936 dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1]) 1937 return; 1938 } 1939 1940 /* 1941 * Grab the creation lock to ensure consistency between calls to 1942 * dtrace_probe_lookup() and dtrace_probe_create() in the face of 1943 * other threads creating probes. 1944 */ 1945 mutex_enter(&provider->ftp_cmtx); 1946 1947 if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod, 1948 dhpb->dthpb_func, dhpb->dthpb_name) != 0) { 1949 mutex_exit(&provider->ftp_cmtx); 1950 return; 1951 } 1952 1953 ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs; 1954 ASSERT(ntps > 0); 1955 1956 atomic_add_32(&fasttrap_total, ntps); 1957 1958 if (fasttrap_total > fasttrap_max) { 1959 atomic_add_32(&fasttrap_total, -ntps); 1960 mutex_exit(&provider->ftp_cmtx); 1961 return; 1962 } 1963 1964 pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP); 1965 1966 pp->ftp_prov = provider; 1967 pp->ftp_pid = provider->ftp_pid; 1968 pp->ftp_ntps = ntps; 1969 pp->ftp_nargs = dhpb->dthpb_xargc; 1970 pp->ftp_xtypes = dhpb->dthpb_xtypes; 1971 pp->ftp_ntypes = dhpb->dthpb_ntypes; 1972 1973 /* 1974 * First create a tracepoint for each actual point of interest. 1975 */ 1976 for (i = 0; i < dhpb->dthpb_noffs; i++) { 1977 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 1978 1979 tp->ftt_proc = provider->ftp_proc; 1980 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i]; 1981 tp->ftt_pid = provider->ftp_pid; 1982 1983 pp->ftp_tps[i].fit_tp = tp; 1984 pp->ftp_tps[i].fit_id.fti_probe = pp; 1985#ifdef __sparc 1986 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS; 1987#else 1988 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS; 1989#endif 1990 } 1991 1992 /* 1993 * Then create a tracepoint for each is-enabled point. 1994 */ 1995 for (j = 0; i < ntps; i++, j++) { 1996 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 1997 1998 tp->ftt_proc = provider->ftp_proc; 1999 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j]; 2000 tp->ftt_pid = provider->ftp_pid; 2001 2002 pp->ftp_tps[i].fit_tp = tp; 2003 pp->ftp_tps[i].fit_id.fti_probe = pp; 2004 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED; 2005 } 2006 2007 /* 2008 * If the arguments are shuffled around we set the argument remapping 2009 * table. Later, when the probe fires, we only remap the arguments 2010 * if the table is non-NULL. 2011 */ 2012 for (i = 0; i < dhpb->dthpb_xargc; i++) { 2013 if (dhpb->dthpb_args[i] != i) { 2014 pp->ftp_argmap = dhpb->dthpb_args; 2015 break; 2016 } 2017 } 2018 2019 /* 2020 * The probe is fully constructed -- register it with DTrace. 2021 */ 2022 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod, 2023 dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp); 2024 2025 mutex_exit(&provider->ftp_cmtx); 2026} 2027 2028/*ARGSUSED*/ 2029static void 2030fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 2031{ 2032 /* 2033 * Clean up the USDT provider. There may be active consumers of the 2034 * provider busy adding probes, no damage will actually befall the 2035 * provider until that count has dropped to zero. This just puts 2036 * the provider on death row. 2037 */ 2038 fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1); 2039} 2040 2041static dtrace_mops_t fasttrap_mops = { 2042 fasttrap_meta_create_probe, 2043 fasttrap_meta_provide, 2044 fasttrap_meta_remove 2045}; 2046 2047/*ARGSUSED*/ 2048static int 2049fasttrap_open(struct cdev *dev __unused, int oflags __unused, 2050 int devtype __unused, struct thread *td __unused) 2051{ 2052 return (0); 2053} 2054 2055/*ARGSUSED*/ 2056static int 2057fasttrap_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int fflag, 2058 struct thread *td) 2059{ 2060#ifdef notyet 2061 struct kinfo_proc kp; 2062 const cred_t *cr = td->td_ucred; 2063#endif 2064 if (!dtrace_attached()) 2065 return (EAGAIN); 2066 2067 if (cmd == FASTTRAPIOC_MAKEPROBE) { 2068 fasttrap_probe_spec_t *uprobe = *(fasttrap_probe_spec_t **)arg; 2069 fasttrap_probe_spec_t *probe; 2070 uint64_t noffs; 2071 size_t size; 2072 int ret, err; 2073 2074 if (copyin(&uprobe->ftps_noffs, &noffs, 2075 sizeof (uprobe->ftps_noffs))) 2076 return (EFAULT); 2077 2078 /* 2079 * Probes must have at least one tracepoint. 2080 */ 2081 if (noffs == 0) 2082 return (EINVAL); 2083 2084 size = sizeof (fasttrap_probe_spec_t) + 2085 sizeof (probe->ftps_offs[0]) * (noffs - 1); 2086 2087 if (size > 1024 * 1024) 2088 return (ENOMEM); 2089 2090 probe = kmem_alloc(size, KM_SLEEP); 2091 2092 if (copyin(uprobe, probe, size) != 0 || 2093 probe->ftps_noffs != noffs) { 2094 kmem_free(probe, size); 2095 return (EFAULT); 2096 } 2097 2098 /* 2099 * Verify that the function and module strings contain no 2100 * funny characters. 2101 */ 2102 if (u8_validate(probe->ftps_func, strlen(probe->ftps_func), 2103 NULL, U8_VALIDATE_ENTIRE, &err) < 0) { 2104 ret = EINVAL; 2105 goto err; 2106 } 2107 2108 if (u8_validate(probe->ftps_mod, strlen(probe->ftps_mod), 2109 NULL, U8_VALIDATE_ENTIRE, &err) < 0) { 2110 ret = EINVAL; 2111 goto err; 2112 } 2113 2114#ifdef notyet 2115 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 2116 proc_t *p; 2117 pid_t pid = probe->ftps_pid; 2118 2119#if defined(sun) 2120 mutex_enter(&pidlock); 2121#endif 2122 /* 2123 * Report an error if the process doesn't exist 2124 * or is actively being birthed. 2125 */ 2126 p = pfind(pid); 2127 if (p) 2128 fill_kinfo_proc(p, &kp); 2129 if (p == NULL || kp.ki_stat == SIDL) { 2130#if defined(sun) 2131 mutex_exit(&pidlock); 2132#endif 2133 return (ESRCH); 2134 } 2135#if defined(sun) 2136 mutex_enter(&p->p_lock); 2137 mutex_exit(&pidlock); 2138#else 2139 PROC_LOCK_ASSERT(p, MA_OWNED); 2140#endif 2141 2142#ifdef notyet 2143 if ((ret = priv_proc_cred_perm(cr, p, NULL, 2144 VREAD | VWRITE)) != 0) { 2145#if defined(sun) 2146 mutex_exit(&p->p_lock); 2147#else 2148 PROC_UNLOCK(p); 2149#endif 2150 return (ret); 2151 } 2152#endif /* notyet */ 2153#if defined(sun) 2154 mutex_exit(&p->p_lock); 2155#else 2156 PROC_UNLOCK(p); 2157#endif 2158 } 2159#endif /* notyet */ 2160 2161 ret = fasttrap_add_probe(probe); 2162err: 2163 kmem_free(probe, size); 2164 2165 return (ret); 2166 2167 } else if (cmd == FASTTRAPIOC_GETINSTR) { 2168 fasttrap_instr_query_t instr; 2169 fasttrap_tracepoint_t *tp; 2170 uint_t index; 2171#if defined(sun) 2172 int ret; 2173#endif 2174 2175#if defined(sun) 2176 if (copyin((void *)arg, &instr, sizeof (instr)) != 0) 2177 return (EFAULT); 2178#endif 2179 2180#ifdef notyet 2181 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 2182 proc_t *p; 2183 pid_t pid = instr.ftiq_pid; 2184 2185#if defined(sun) 2186 mutex_enter(&pidlock); 2187#endif 2188 /* 2189 * Report an error if the process doesn't exist 2190 * or is actively being birthed. 2191 */ 2192 p = pfind(pid); 2193 if (p) 2194 fill_kinfo_proc(p, &kp); 2195 if (p == NULL || kp.ki_stat == SIDL) { 2196#if defined(sun) 2197 mutex_exit(&pidlock); 2198#endif 2199 return (ESRCH); 2200 } 2201#if defined(sun) 2202 mutex_enter(&p->p_lock); 2203 mutex_exit(&pidlock); 2204#else 2205 PROC_LOCK_ASSERT(p, MA_OWNED); 2206#endif 2207 2208#ifdef notyet 2209 if ((ret = priv_proc_cred_perm(cr, p, NULL, 2210 VREAD)) != 0) { 2211#if defined(sun) 2212 mutex_exit(&p->p_lock); 2213#else 2214 PROC_UNLOCK(p); 2215#endif 2216 return (ret); 2217 } 2218#endif /* notyet */ 2219 2220#if defined(sun) 2221 mutex_exit(&p->p_lock); 2222#else 2223 PROC_UNLOCK(p); 2224#endif 2225 } 2226#endif /* notyet */ 2227 2228 index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc); 2229 2230 mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2231 tp = fasttrap_tpoints.fth_table[index].ftb_data; 2232 while (tp != NULL) { 2233 if (instr.ftiq_pid == tp->ftt_pid && 2234 instr.ftiq_pc == tp->ftt_pc && 2235 tp->ftt_proc->ftpc_acount != 0) 2236 break; 2237 2238 tp = tp->ftt_next; 2239 } 2240 2241 if (tp == NULL) { 2242 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2243 return (ENOENT); 2244 } 2245 2246 bcopy(&tp->ftt_instr, &instr.ftiq_instr, 2247 sizeof (instr.ftiq_instr)); 2248 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2249 2250 if (copyout(&instr, (void *)arg, sizeof (instr)) != 0) 2251 return (EFAULT); 2252 2253 return (0); 2254 } 2255 2256 return (EINVAL); 2257} 2258 2259static int 2260fasttrap_load(void) 2261{ 2262 ulong_t nent; 2263 int i, ret; 2264 2265 /* Create the /dev/dtrace/fasttrap entry. */ 2266 fasttrap_cdev = make_dev(&fasttrap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, 2267 "dtrace/fasttrap"); 2268 2269 mtx_init(&fasttrap_cleanup_mtx, "fasttrap clean", "dtrace", MTX_DEF); 2270 mutex_init(&fasttrap_count_mtx, "fasttrap count mtx", MUTEX_DEFAULT, 2271 NULL); 2272 2273#if defined(sun) 2274 fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2275 "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT); 2276#else 2277 fasttrap_max = FASTTRAP_MAX_DEFAULT; 2278#endif 2279 fasttrap_total = 0; 2280 2281 /* 2282 * Conjure up the tracepoints hashtable... 2283 */ 2284#if defined(sun) 2285 nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2286 "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE); 2287#else 2288 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE; 2289#endif 2290 2291 if (nent == 0 || nent > 0x1000000) 2292 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE; 2293 2294 if ((nent & (nent - 1)) == 0) 2295 fasttrap_tpoints.fth_nent = nent; 2296 else 2297 fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent); 2298 ASSERT(fasttrap_tpoints.fth_nent > 0); 2299 fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1; 2300 fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent * 2301 sizeof (fasttrap_bucket_t), KM_SLEEP); 2302#if !defined(sun) 2303 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2304 mutex_init(&fasttrap_tpoints.fth_table[i].ftb_mtx, 2305 "tracepoints bucket mtx", MUTEX_DEFAULT, NULL); 2306#endif 2307 2308 /* 2309 * ... and the providers hash table... 2310 */ 2311 nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE; 2312 if ((nent & (nent - 1)) == 0) 2313 fasttrap_provs.fth_nent = nent; 2314 else 2315 fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent); 2316 ASSERT(fasttrap_provs.fth_nent > 0); 2317 fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1; 2318 fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent * 2319 sizeof (fasttrap_bucket_t), KM_SLEEP); 2320#if !defined(sun) 2321 for (i = 0; i < fasttrap_provs.fth_nent; i++) 2322 mutex_init(&fasttrap_provs.fth_table[i].ftb_mtx, 2323 "providers bucket mtx", MUTEX_DEFAULT, NULL); 2324#endif 2325 2326 ret = kproc_create(fasttrap_pid_cleanup_cb, NULL, 2327 &fasttrap_cleanup_proc, 0, 0, "ftcleanup"); 2328 if (ret != 0) { 2329 destroy_dev(fasttrap_cdev); 2330#if !defined(sun) 2331 for (i = 0; i < fasttrap_provs.fth_nent; i++) 2332 mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx); 2333 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2334 mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx); 2335#endif 2336 kmem_free(fasttrap_provs.fth_table, fasttrap_provs.fth_nent * 2337 sizeof (fasttrap_bucket_t)); 2338 mtx_destroy(&fasttrap_cleanup_mtx); 2339 mutex_destroy(&fasttrap_count_mtx); 2340 return (ret); 2341 } 2342 2343 2344 /* 2345 * ... and the procs hash table. 2346 */ 2347 nent = FASTTRAP_PROCS_DEFAULT_SIZE; 2348 if ((nent & (nent - 1)) == 0) 2349 fasttrap_procs.fth_nent = nent; 2350 else 2351 fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent); 2352 ASSERT(fasttrap_procs.fth_nent > 0); 2353 fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1; 2354 fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent * 2355 sizeof (fasttrap_bucket_t), KM_SLEEP); 2356#if !defined(sun) 2357 for (i = 0; i < fasttrap_procs.fth_nent; i++) 2358 mutex_init(&fasttrap_procs.fth_table[i].ftb_mtx, 2359 "processes bucket mtx", MUTEX_DEFAULT, NULL); 2360 2361 CPU_FOREACH(i) { 2362 mutex_init(&fasttrap_cpuc_pid_lock[i], "fasttrap barrier", 2363 MUTEX_DEFAULT, NULL); 2364 } 2365#endif 2366 2367 /* 2368 * Install our hooks into fork(2), exec(2), and exit(2). 2369 */ 2370 dtrace_fasttrap_fork = &fasttrap_fork; 2371 dtrace_fasttrap_exit = &fasttrap_exec_exit; 2372 dtrace_fasttrap_exec = &fasttrap_exec_exit; 2373 2374 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2375 &fasttrap_meta_id); 2376 2377 return (0); 2378} 2379 2380static int 2381fasttrap_unload(void) 2382{ 2383 int i, fail = 0; 2384 2385 /* 2386 * Unregister the meta-provider to make sure no new fasttrap- 2387 * managed providers come along while we're trying to close up 2388 * shop. If we fail to detach, we'll need to re-register as a 2389 * meta-provider. We can fail to unregister as a meta-provider 2390 * if providers we manage still exist. 2391 */ 2392 if (fasttrap_meta_id != DTRACE_METAPROVNONE && 2393 dtrace_meta_unregister(fasttrap_meta_id) != 0) 2394 return (-1); 2395 2396 /* 2397 * Iterate over all of our providers. If there's still a process 2398 * that corresponds to that pid, fail to detach. 2399 */ 2400 for (i = 0; i < fasttrap_provs.fth_nent; i++) { 2401 fasttrap_provider_t **fpp, *fp; 2402 fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i]; 2403 2404 mutex_enter(&bucket->ftb_mtx); 2405 fpp = (fasttrap_provider_t **)&bucket->ftb_data; 2406 while ((fp = *fpp) != NULL) { 2407 /* 2408 * Acquire and release the lock as a simple way of 2409 * waiting for any other consumer to finish with 2410 * this provider. A thread must first acquire the 2411 * bucket lock so there's no chance of another thread 2412 * blocking on the provider's lock. 2413 */ 2414 mutex_enter(&fp->ftp_mtx); 2415 mutex_exit(&fp->ftp_mtx); 2416 2417 if (dtrace_unregister(fp->ftp_provid) != 0) { 2418 fail = 1; 2419 fpp = &fp->ftp_next; 2420 } else { 2421 *fpp = fp->ftp_next; 2422 fasttrap_provider_free(fp); 2423 } 2424 } 2425 2426 mutex_exit(&bucket->ftb_mtx); 2427 } 2428 2429 if (fail) { 2430 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2431 &fasttrap_meta_id); 2432 2433 return (-1); 2434 } 2435 2436 /* 2437 * Stop new processes from entering these hooks now, before the 2438 * fasttrap_cleanup thread runs. That way all processes will hopefully 2439 * be out of these hooks before we free fasttrap_provs.fth_table 2440 */ 2441 ASSERT(dtrace_fasttrap_fork == &fasttrap_fork); 2442 dtrace_fasttrap_fork = NULL; 2443 2444 ASSERT(dtrace_fasttrap_exec == &fasttrap_exec_exit); 2445 dtrace_fasttrap_exec = NULL; 2446 2447 ASSERT(dtrace_fasttrap_exit == &fasttrap_exec_exit); 2448 dtrace_fasttrap_exit = NULL; 2449 2450 mtx_lock(&fasttrap_cleanup_mtx); 2451 fasttrap_cleanup_drain = 1; 2452 /* Wait for the cleanup thread to finish up and signal us. */ 2453 wakeup(&fasttrap_cleanup_cv); 2454 mtx_sleep(&fasttrap_cleanup_drain, &fasttrap_cleanup_mtx, 0, "ftcld", 2455 0); 2456 fasttrap_cleanup_proc = NULL; 2457 mtx_destroy(&fasttrap_cleanup_mtx); 2458 2459#ifdef DEBUG 2460 mutex_enter(&fasttrap_count_mtx); 2461 ASSERT(fasttrap_pid_count == 0); 2462 mutex_exit(&fasttrap_count_mtx); 2463#endif 2464 2465#if !defined(sun) 2466 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2467 mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx); 2468 for (i = 0; i < fasttrap_provs.fth_nent; i++) 2469 mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx); 2470 for (i = 0; i < fasttrap_procs.fth_nent; i++) 2471 mutex_destroy(&fasttrap_procs.fth_table[i].ftb_mtx); 2472#endif 2473 kmem_free(fasttrap_tpoints.fth_table, 2474 fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t)); 2475 fasttrap_tpoints.fth_nent = 0; 2476 2477 kmem_free(fasttrap_provs.fth_table, 2478 fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t)); 2479 fasttrap_provs.fth_nent = 0; 2480 2481 kmem_free(fasttrap_procs.fth_table, 2482 fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t)); 2483 fasttrap_procs.fth_nent = 0; 2484 2485#if !defined(sun) 2486 destroy_dev(fasttrap_cdev); 2487 mutex_destroy(&fasttrap_count_mtx); 2488 CPU_FOREACH(i) { 2489 mutex_destroy(&fasttrap_cpuc_pid_lock[i]); 2490 } 2491#endif 2492 2493 return (0); 2494} 2495 2496/* ARGSUSED */ 2497static int 2498fasttrap_modevent(module_t mod __unused, int type, void *data __unused) 2499{ 2500 int error = 0; 2501 2502 switch (type) { 2503 case MOD_LOAD: 2504 break; 2505 2506 case MOD_UNLOAD: 2507 break; 2508 2509 case MOD_SHUTDOWN: 2510 break; 2511 2512 default: 2513 error = EOPNOTSUPP; 2514 break; 2515 } 2516 return (error); 2517} 2518 2519SYSINIT(fasttrap_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fasttrap_load, 2520 NULL); 2521SYSUNINIT(fasttrap_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, 2522 fasttrap_unload, NULL); 2523 2524DEV_MODULE(fasttrap, fasttrap_modevent, NULL); 2525MODULE_VERSION(fasttrap, 1); 2526MODULE_DEPEND(fasttrap, dtrace, 1, 1, 1); 2527MODULE_DEPEND(fasttrap, opensolaris, 1, 1, 1); 2528