fasttrap.c revision 297077
1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 * 21 * Portions Copyright 2010 The FreeBSD Foundation 22 * 23 * $FreeBSD: stable/10/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c 297077 2016-03-20 20:00:25Z mav $ 24 */ 25 26/* 27 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 28 * Use is subject to license terms. 29 */ 30 31/* 32 * Copyright (c) 2013, Joyent, Inc. All rights reserved. 33 */ 34 35#include <sys/atomic.h> 36#include <sys/errno.h> 37#include <sys/stat.h> 38#include <sys/modctl.h> 39#include <sys/conf.h> 40#include <sys/systm.h> 41#ifdef illumos 42#include <sys/ddi.h> 43#endif 44#include <sys/sunddi.h> 45#include <sys/cpuvar.h> 46#include <sys/kmem.h> 47#ifdef illumos 48#include <sys/strsubr.h> 49#endif 50#include <sys/fasttrap.h> 51#include <sys/fasttrap_impl.h> 52#include <sys/fasttrap_isa.h> 53#include <sys/dtrace.h> 54#include <sys/dtrace_impl.h> 55#include <sys/sysmacros.h> 56#include <sys/proc.h> 57#include <sys/policy.h> 58#ifdef illumos 59#include <util/qsort.h> 60#endif 61#include <sys/mutex.h> 62#include <sys/kernel.h> 63#ifndef illumos 64#include <sys/dtrace_bsd.h> 65#include <sys/eventhandler.h> 66#include <sys/sysctl.h> 67#include <sys/u8_textprep.h> 68#include <sys/user.h> 69#include <vm/vm.h> 70#include <vm/pmap.h> 71#include <vm/vm_map.h> 72#include <vm/vm_param.h> 73#include <sys/u8_textprep.h> 74#include <cddl/dev/dtrace/dtrace_cddl.h> 75#endif 76 77/* 78 * User-Land Trap-Based Tracing 79 * ---------------------------- 80 * 81 * The fasttrap provider allows DTrace consumers to instrument any user-level 82 * instruction to gather data; this includes probes with semantic 83 * signifigance like entry and return as well as simple offsets into the 84 * function. While the specific techniques used are very ISA specific, the 85 * methodology is generalizable to any architecture. 86 * 87 * 88 * The General Methodology 89 * ----------------------- 90 * 91 * With the primary goal of tracing every user-land instruction and the 92 * limitation that we can't trust user space so don't want to rely on much 93 * information there, we begin by replacing the instructions we want to trace 94 * with trap instructions. Each instruction we overwrite is saved into a hash 95 * table keyed by process ID and pc address. When we enter the kernel due to 96 * this trap instruction, we need the effects of the replaced instruction to 97 * appear to have occurred before we proceed with the user thread's 98 * execution. 99 * 100 * Each user level thread is represented by a ulwp_t structure which is 101 * always easily accessible through a register. The most basic way to produce 102 * the effects of the instruction we replaced is to copy that instruction out 103 * to a bit of scratch space reserved in the user thread's ulwp_t structure 104 * (a sort of kernel-private thread local storage), set the PC to that 105 * scratch space and single step. When we reenter the kernel after single 106 * stepping the instruction we must then adjust the PC to point to what would 107 * normally be the next instruction. Of course, special care must be taken 108 * for branches and jumps, but these represent such a small fraction of any 109 * instruction set that writing the code to emulate these in the kernel is 110 * not too difficult. 111 * 112 * Return probes may require several tracepoints to trace every return site, 113 * and, conversely, each tracepoint may activate several probes (the entry 114 * and offset 0 probes, for example). To solve this muliplexing problem, 115 * tracepoints contain lists of probes to activate and probes contain lists 116 * of tracepoints to enable. If a probe is activated, it adds its ID to 117 * existing tracepoints or creates new ones as necessary. 118 * 119 * Most probes are activated _before_ the instruction is executed, but return 120 * probes are activated _after_ the effects of the last instruction of the 121 * function are visible. Return probes must be fired _after_ we have 122 * single-stepped the instruction whereas all other probes are fired 123 * beforehand. 124 * 125 * 126 * Lock Ordering 127 * ------------- 128 * 129 * The lock ordering below -- both internally and with respect to the DTrace 130 * framework -- is a little tricky and bears some explanation. Each provider 131 * has a lock (ftp_mtx) that protects its members including reference counts 132 * for enabled probes (ftp_rcount), consumers actively creating probes 133 * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider 134 * from being freed. A provider is looked up by taking the bucket lock for the 135 * provider hash table, and is returned with its lock held. The provider lock 136 * may be taken in functions invoked by the DTrace framework, but may not be 137 * held while calling functions in the DTrace framework. 138 * 139 * To ensure consistency over multiple calls to the DTrace framework, the 140 * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may 141 * not be taken when holding the provider lock as that would create a cyclic 142 * lock ordering. In situations where one would naturally take the provider 143 * lock and then the creation lock, we instead up a reference count to prevent 144 * the provider from disappearing, drop the provider lock, and acquire the 145 * creation lock. 146 * 147 * Briefly: 148 * bucket lock before provider lock 149 * DTrace before provider lock 150 * creation lock before DTrace 151 * never hold the provider lock and creation lock simultaneously 152 */ 153 154static d_open_t fasttrap_open; 155static d_ioctl_t fasttrap_ioctl; 156 157static struct cdevsw fasttrap_cdevsw = { 158 .d_version = D_VERSION, 159 .d_open = fasttrap_open, 160 .d_ioctl = fasttrap_ioctl, 161 .d_name = "fasttrap", 162}; 163static struct cdev *fasttrap_cdev; 164static dtrace_meta_provider_id_t fasttrap_meta_id; 165 166static struct proc *fasttrap_cleanup_proc; 167static struct mtx fasttrap_cleanup_mtx; 168static uint_t fasttrap_cleanup_work, fasttrap_cleanup_drain, fasttrap_cleanup_cv; 169 170/* 171 * Generation count on modifications to the global tracepoint lookup table. 172 */ 173static volatile uint64_t fasttrap_mod_gen; 174 175/* 176 * When the fasttrap provider is loaded, fasttrap_max is set to either 177 * FASTTRAP_MAX_DEFAULT, or the value for fasttrap-max-probes in the 178 * fasttrap.conf file (Illumos), or the value provied in the loader.conf (FreeBSD). 179 * Each time a probe is created, fasttrap_total is incremented by the number 180 * of tracepoints that may be associated with that probe; fasttrap_total is capped 181 * at fasttrap_max. 182 */ 183#define FASTTRAP_MAX_DEFAULT 250000 184static uint32_t fasttrap_max = FASTTRAP_MAX_DEFAULT; 185static uint32_t fasttrap_total; 186 187/* 188 * Copyright (c) 2011, Joyent, Inc. All rights reserved. 189 */ 190 191#define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000 192#define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100 193#define FASTTRAP_PROCS_DEFAULT_SIZE 0x100 194 195#define FASTTRAP_PID_NAME "pid" 196 197fasttrap_hash_t fasttrap_tpoints; 198static fasttrap_hash_t fasttrap_provs; 199static fasttrap_hash_t fasttrap_procs; 200 201static uint64_t fasttrap_pid_count; /* pid ref count */ 202static kmutex_t fasttrap_count_mtx; /* lock on ref count */ 203 204#define FASTTRAP_ENABLE_FAIL 1 205#define FASTTRAP_ENABLE_PARTIAL 2 206 207static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t); 208static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t); 209 210static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *, 211 const dtrace_pattr_t *); 212static void fasttrap_provider_retire(pid_t, const char *, int); 213static void fasttrap_provider_free(fasttrap_provider_t *); 214 215static fasttrap_proc_t *fasttrap_proc_lookup(pid_t); 216static void fasttrap_proc_release(fasttrap_proc_t *); 217 218#ifndef illumos 219static void fasttrap_thread_dtor(void *, struct thread *); 220#endif 221 222#define FASTTRAP_PROVS_INDEX(pid, name) \ 223 ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask) 224 225#define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask) 226 227#ifndef illumos 228static kmutex_t fasttrap_cpuc_pid_lock[MAXCPU]; 229static eventhandler_tag fasttrap_thread_dtor_tag; 230#endif 231 232static unsigned long tpoints_hash_size = FASTTRAP_TPOINTS_DEFAULT_SIZE; 233 234#ifdef __FreeBSD__ 235SYSCTL_DECL(_kern_dtrace); 236SYSCTL_NODE(_kern_dtrace, OID_AUTO, fasttrap, CTLFLAG_RD, 0, "DTrace fasttrap parameters"); 237SYSCTL_UINT(_kern_dtrace_fasttrap, OID_AUTO, max_probes, CTLFLAG_RWTUN, &fasttrap_max, 238 FASTTRAP_MAX_DEFAULT, "Maximum number of fasttrap probes"); 239SYSCTL_ULONG(_kern_dtrace_fasttrap, OID_AUTO, tpoints_hash_size, CTLFLAG_RDTUN, &tpoints_hash_size, 240 FASTTRAP_TPOINTS_DEFAULT_SIZE, "Size of the tracepoint hash table"); 241#endif 242 243static int 244fasttrap_highbit(ulong_t i) 245{ 246 int h = 1; 247 248 if (i == 0) 249 return (0); 250#ifdef _LP64 251 if (i & 0xffffffff00000000ul) { 252 h += 32; i >>= 32; 253 } 254#endif 255 if (i & 0xffff0000) { 256 h += 16; i >>= 16; 257 } 258 if (i & 0xff00) { 259 h += 8; i >>= 8; 260 } 261 if (i & 0xf0) { 262 h += 4; i >>= 4; 263 } 264 if (i & 0xc) { 265 h += 2; i >>= 2; 266 } 267 if (i & 0x2) { 268 h += 1; 269 } 270 return (h); 271} 272 273static uint_t 274fasttrap_hash_str(const char *p) 275{ 276 unsigned int g; 277 uint_t hval = 0; 278 279 while (*p) { 280 hval = (hval << 4) + *p++; 281 if ((g = (hval & 0xf0000000)) != 0) 282 hval ^= g >> 24; 283 hval &= ~g; 284 } 285 return (hval); 286} 287 288void 289fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc) 290{ 291#ifdef illumos 292 sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP); 293 294 sqp->sq_info.si_signo = SIGTRAP; 295 sqp->sq_info.si_code = TRAP_DTRACE; 296 sqp->sq_info.si_addr = (caddr_t)pc; 297 298 mutex_enter(&p->p_lock); 299 sigaddqa(p, t, sqp); 300 mutex_exit(&p->p_lock); 301 302 if (t != NULL) 303 aston(t); 304#else 305 ksiginfo_t *ksi = kmem_zalloc(sizeof (ksiginfo_t), KM_SLEEP); 306 307 ksiginfo_init(ksi); 308 ksi->ksi_signo = SIGTRAP; 309 ksi->ksi_code = TRAP_DTRACE; 310 ksi->ksi_addr = (caddr_t)pc; 311 PROC_LOCK(p); 312 (void) tdsendsignal(p, t, SIGTRAP, ksi); 313 PROC_UNLOCK(p); 314#endif 315} 316 317#ifndef illumos 318/* 319 * Obtain a chunk of scratch space in the address space of the target process. 320 */ 321fasttrap_scrspace_t * 322fasttrap_scraddr(struct thread *td, fasttrap_proc_t *fprc) 323{ 324 fasttrap_scrblock_t *scrblk; 325 fasttrap_scrspace_t *scrspc; 326 struct proc *p; 327 vm_offset_t addr; 328 int error, i; 329 330 scrspc = NULL; 331 if (td->t_dtrace_sscr != NULL) { 332 /* If the thread already has scratch space, we're done. */ 333 scrspc = (fasttrap_scrspace_t *)td->t_dtrace_sscr; 334 return (scrspc); 335 } 336 337 p = td->td_proc; 338 339 mutex_enter(&fprc->ftpc_mtx); 340 if (LIST_EMPTY(&fprc->ftpc_fscr)) { 341 /* 342 * No scratch space is available, so we'll map a new scratch 343 * space block into the traced process' address space. 344 */ 345 addr = 0; 346 error = vm_map_find(&p->p_vmspace->vm_map, NULL, 0, &addr, 347 FASTTRAP_SCRBLOCK_SIZE, 0, VMFS_ANY_SPACE, VM_PROT_ALL, 348 VM_PROT_ALL, 0); 349 if (error != KERN_SUCCESS) 350 goto done; 351 352 scrblk = malloc(sizeof(*scrblk), M_SOLARIS, M_WAITOK); 353 scrblk->ftsb_addr = addr; 354 LIST_INSERT_HEAD(&fprc->ftpc_scrblks, scrblk, ftsb_next); 355 356 /* 357 * Carve the block up into chunks and put them on the free list. 358 */ 359 for (i = 0; 360 i < FASTTRAP_SCRBLOCK_SIZE / FASTTRAP_SCRSPACE_SIZE; i++) { 361 scrspc = malloc(sizeof(*scrspc), M_SOLARIS, M_WAITOK); 362 scrspc->ftss_addr = addr + 363 i * FASTTRAP_SCRSPACE_SIZE; 364 LIST_INSERT_HEAD(&fprc->ftpc_fscr, scrspc, 365 ftss_next); 366 } 367 } 368 369 /* 370 * Take the first scratch chunk off the free list, put it on the 371 * allocated list, and return its address. 372 */ 373 scrspc = LIST_FIRST(&fprc->ftpc_fscr); 374 LIST_REMOVE(scrspc, ftss_next); 375 LIST_INSERT_HEAD(&fprc->ftpc_ascr, scrspc, ftss_next); 376 377 /* 378 * This scratch space is reserved for use by td until the thread exits. 379 */ 380 td->t_dtrace_sscr = scrspc; 381 382done: 383 mutex_exit(&fprc->ftpc_mtx); 384 385 return (scrspc); 386} 387 388/* 389 * Return any allocated per-thread scratch space chunks back to the process' 390 * free list. 391 */ 392static void 393fasttrap_thread_dtor(void *arg __unused, struct thread *td) 394{ 395 fasttrap_bucket_t *bucket; 396 fasttrap_proc_t *fprc; 397 fasttrap_scrspace_t *scrspc; 398 pid_t pid; 399 400 if (td->t_dtrace_sscr == NULL) 401 return; 402 403 pid = td->td_proc->p_pid; 404 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 405 fprc = NULL; 406 407 /* Look up the fasttrap process handle for this process. */ 408 mutex_enter(&bucket->ftb_mtx); 409 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 410 if (fprc->ftpc_pid == pid) { 411 mutex_enter(&fprc->ftpc_mtx); 412 mutex_exit(&bucket->ftb_mtx); 413 break; 414 } 415 } 416 if (fprc == NULL) { 417 mutex_exit(&bucket->ftb_mtx); 418 return; 419 } 420 421 scrspc = (fasttrap_scrspace_t *)td->t_dtrace_sscr; 422 LIST_REMOVE(scrspc, ftss_next); 423 LIST_INSERT_HEAD(&fprc->ftpc_fscr, scrspc, ftss_next); 424 425 mutex_exit(&fprc->ftpc_mtx); 426} 427#endif 428 429/* 430 * This function ensures that no threads are actively using the memory 431 * associated with probes that were formerly live. 432 */ 433static void 434fasttrap_mod_barrier(uint64_t gen) 435{ 436 int i; 437 438 if (gen < fasttrap_mod_gen) 439 return; 440 441 fasttrap_mod_gen++; 442 443 CPU_FOREACH(i) { 444 mutex_enter(&fasttrap_cpuc_pid_lock[i]); 445 mutex_exit(&fasttrap_cpuc_pid_lock[i]); 446 } 447} 448 449/* 450 * This function performs asynchronous cleanup of fasttrap providers. The 451 * Solaris implementation of this mechanism use a timeout that's activated in 452 * fasttrap_pid_cleanup(), but this doesn't work in FreeBSD: one may sleep while 453 * holding the DTrace mutexes, but it is unsafe to sleep in a callout handler. 454 * Thus we use a dedicated process to perform the cleanup when requested. 455 */ 456/*ARGSUSED*/ 457static void 458fasttrap_pid_cleanup_cb(void *data) 459{ 460 fasttrap_provider_t **fpp, *fp; 461 fasttrap_bucket_t *bucket; 462 dtrace_provider_id_t provid; 463 int i, later = 0, rval; 464 465 mtx_lock(&fasttrap_cleanup_mtx); 466 while (!fasttrap_cleanup_drain || later > 0) { 467 fasttrap_cleanup_work = 0; 468 mtx_unlock(&fasttrap_cleanup_mtx); 469 470 later = 0; 471 472 /* 473 * Iterate over all the providers trying to remove the marked 474 * ones. If a provider is marked but not retired, we just 475 * have to take a crack at removing it -- it's no big deal if 476 * we can't. 477 */ 478 for (i = 0; i < fasttrap_provs.fth_nent; i++) { 479 bucket = &fasttrap_provs.fth_table[i]; 480 mutex_enter(&bucket->ftb_mtx); 481 fpp = (fasttrap_provider_t **)&bucket->ftb_data; 482 483 while ((fp = *fpp) != NULL) { 484 if (!fp->ftp_marked) { 485 fpp = &fp->ftp_next; 486 continue; 487 } 488 489 mutex_enter(&fp->ftp_mtx); 490 491 /* 492 * If this provider has consumers actively 493 * creating probes (ftp_ccount) or is a USDT 494 * provider (ftp_mcount), we can't unregister 495 * or even condense. 496 */ 497 if (fp->ftp_ccount != 0 || 498 fp->ftp_mcount != 0) { 499 mutex_exit(&fp->ftp_mtx); 500 fp->ftp_marked = 0; 501 continue; 502 } 503 504 if (!fp->ftp_retired || fp->ftp_rcount != 0) 505 fp->ftp_marked = 0; 506 507 mutex_exit(&fp->ftp_mtx); 508 509 /* 510 * If we successfully unregister this 511 * provider we can remove it from the hash 512 * chain and free the memory. If our attempt 513 * to unregister fails and this is a retired 514 * provider, increment our flag to try again 515 * pretty soon. If we've consumed more than 516 * half of our total permitted number of 517 * probes call dtrace_condense() to try to 518 * clean out the unenabled probes. 519 */ 520 provid = fp->ftp_provid; 521 if ((rval = dtrace_unregister(provid)) != 0) { 522 if (fasttrap_total > fasttrap_max / 2) 523 (void) dtrace_condense(provid); 524 525 if (rval == EAGAIN) 526 fp->ftp_marked = 1; 527 528 later += fp->ftp_marked; 529 fpp = &fp->ftp_next; 530 } else { 531 *fpp = fp->ftp_next; 532 fasttrap_provider_free(fp); 533 } 534 } 535 mutex_exit(&bucket->ftb_mtx); 536 } 537 mtx_lock(&fasttrap_cleanup_mtx); 538 539 /* 540 * If we were unable to retire a provider, try again after a 541 * second. This situation can occur in certain circumstances 542 * where providers cannot be unregistered even though they have 543 * no probes enabled because of an execution of dtrace -l or 544 * something similar. 545 */ 546 if (later > 0 || fasttrap_cleanup_work || 547 fasttrap_cleanup_drain) { 548 mtx_unlock(&fasttrap_cleanup_mtx); 549 pause("ftclean", hz); 550 mtx_lock(&fasttrap_cleanup_mtx); 551 } else 552 mtx_sleep(&fasttrap_cleanup_cv, &fasttrap_cleanup_mtx, 553 0, "ftcl", 0); 554 } 555 556 /* 557 * Wake up the thread in fasttrap_unload() now that we're done. 558 */ 559 wakeup(&fasttrap_cleanup_drain); 560 mtx_unlock(&fasttrap_cleanup_mtx); 561 562 kthread_exit(); 563} 564 565/* 566 * Activates the asynchronous cleanup mechanism. 567 */ 568static void 569fasttrap_pid_cleanup(void) 570{ 571 572 mtx_lock(&fasttrap_cleanup_mtx); 573 if (!fasttrap_cleanup_work) { 574 fasttrap_cleanup_work = 1; 575 wakeup(&fasttrap_cleanup_cv); 576 } 577 mtx_unlock(&fasttrap_cleanup_mtx); 578} 579 580/* 581 * This is called from cfork() via dtrace_fasttrap_fork(). The child 582 * process's address space is (roughly) a copy of the parent process's so 583 * we have to remove all the instrumentation we had previously enabled in the 584 * parent. 585 */ 586static void 587fasttrap_fork(proc_t *p, proc_t *cp) 588{ 589#ifndef illumos 590 fasttrap_scrblock_t *scrblk; 591 fasttrap_proc_t *fprc = NULL; 592#endif 593 pid_t ppid = p->p_pid; 594 int i; 595 596#ifdef illumos 597 ASSERT(curproc == p); 598 ASSERT(p->p_proc_flag & P_PR_LOCK); 599#else 600 PROC_LOCK_ASSERT(p, MA_OWNED); 601#endif 602#ifdef illumos 603 ASSERT(p->p_dtrace_count > 0); 604#else 605 if (p->p_dtrace_helpers) { 606 /* 607 * dtrace_helpers_duplicate() allocates memory. 608 */ 609 _PHOLD(cp); 610 PROC_UNLOCK(p); 611 PROC_UNLOCK(cp); 612 dtrace_helpers_duplicate(p, cp); 613 PROC_LOCK(cp); 614 PROC_LOCK(p); 615 _PRELE(cp); 616 } 617 /* 618 * This check is purposely here instead of in kern_fork.c because, 619 * for legal resons, we cannot include the dtrace_cddl.h header 620 * inside kern_fork.c and insert if-clause there. 621 */ 622 if (p->p_dtrace_count == 0) 623 return; 624#endif 625 ASSERT(cp->p_dtrace_count == 0); 626 627 /* 628 * This would be simpler and faster if we maintained per-process 629 * hash tables of enabled tracepoints. It could, however, potentially 630 * slow down execution of a tracepoint since we'd need to go 631 * through two levels of indirection. In the future, we should 632 * consider either maintaining per-process ancillary lists of 633 * enabled tracepoints or hanging a pointer to a per-process hash 634 * table of enabled tracepoints off the proc structure. 635 */ 636 637 /* 638 * We don't have to worry about the child process disappearing 639 * because we're in fork(). 640 */ 641#ifdef illumos 642 mtx_lock_spin(&cp->p_slock); 643 sprlock_proc(cp); 644 mtx_unlock_spin(&cp->p_slock); 645#else 646 /* 647 * fasttrap_tracepoint_remove() expects the child process to be 648 * unlocked and the VM then expects curproc to be unlocked. 649 */ 650 _PHOLD(cp); 651 PROC_UNLOCK(cp); 652 PROC_UNLOCK(p); 653#endif 654 655 /* 656 * Iterate over every tracepoint looking for ones that belong to the 657 * parent process, and remove each from the child process. 658 */ 659 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) { 660 fasttrap_tracepoint_t *tp; 661 fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i]; 662 663 mutex_enter(&bucket->ftb_mtx); 664 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 665 if (tp->ftt_pid == ppid && 666 tp->ftt_proc->ftpc_acount != 0) { 667 int ret = fasttrap_tracepoint_remove(cp, tp); 668 ASSERT(ret == 0); 669 670 /* 671 * The count of active providers can only be 672 * decremented (i.e. to zero) during exec, 673 * exit, and removal of a meta provider so it 674 * should be impossible to drop the count 675 * mid-fork. 676 */ 677 ASSERT(tp->ftt_proc->ftpc_acount != 0); 678#ifndef illumos 679 fprc = tp->ftt_proc; 680#endif 681 } 682 } 683 mutex_exit(&bucket->ftb_mtx); 684 685#ifndef illumos 686 /* 687 * Unmap any scratch space inherited from the parent's address 688 * space. 689 */ 690 if (fprc != NULL) { 691 mutex_enter(&fprc->ftpc_mtx); 692 LIST_FOREACH(scrblk, &fprc->ftpc_scrblks, ftsb_next) { 693 vm_map_remove(&cp->p_vmspace->vm_map, 694 scrblk->ftsb_addr, 695 scrblk->ftsb_addr + FASTTRAP_SCRBLOCK_SIZE); 696 } 697 mutex_exit(&fprc->ftpc_mtx); 698 } 699#endif 700 } 701 702#ifdef illumos 703 mutex_enter(&cp->p_lock); 704 sprunlock(cp); 705#else 706 PROC_LOCK(p); 707 PROC_LOCK(cp); 708 _PRELE(cp); 709#endif 710} 711 712/* 713 * This is called from proc_exit() or from exec_common() if p_dtrace_probes 714 * is set on the proc structure to indicate that there is a pid provider 715 * associated with this process. 716 */ 717static void 718fasttrap_exec_exit(proc_t *p) 719{ 720#ifndef illumos 721 struct thread *td; 722#endif 723 724#ifdef illumos 725 ASSERT(p == curproc); 726#else 727 PROC_LOCK_ASSERT(p, MA_OWNED); 728 _PHOLD(p); 729 /* 730 * Since struct threads may be recycled, we cannot rely on t_dtrace_sscr 731 * fields to be zeroed by kdtrace_thread_ctor. Thus we must zero it 732 * ourselves when a process exits. 733 */ 734 FOREACH_THREAD_IN_PROC(p, td) 735 td->t_dtrace_sscr = NULL; 736 PROC_UNLOCK(p); 737#endif 738 739 /* 740 * We clean up the pid provider for this process here; user-land 741 * static probes are handled by the meta-provider remove entry point. 742 */ 743 fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0); 744#ifndef illumos 745 if (p->p_dtrace_helpers) 746 dtrace_helpers_destroy(p); 747 PROC_LOCK(p); 748 _PRELE(p); 749#endif 750} 751 752 753/*ARGSUSED*/ 754static void 755fasttrap_pid_provide(void *arg, dtrace_probedesc_t *desc) 756{ 757 /* 758 * There are no "default" pid probes. 759 */ 760} 761 762static int 763fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 764{ 765 fasttrap_tracepoint_t *tp, *new_tp = NULL; 766 fasttrap_bucket_t *bucket; 767 fasttrap_id_t *id; 768 pid_t pid; 769 uintptr_t pc; 770 771 ASSERT(index < probe->ftp_ntps); 772 773 pid = probe->ftp_pid; 774 pc = probe->ftp_tps[index].fit_tp->ftt_pc; 775 id = &probe->ftp_tps[index].fit_id; 776 777 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 778 779#ifdef illumos 780 ASSERT(!(p->p_flag & SVFORK)); 781#endif 782 783 /* 784 * Before we make any modifications, make sure we've imposed a barrier 785 * on the generation in which this probe was last modified. 786 */ 787 fasttrap_mod_barrier(probe->ftp_gen); 788 789 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 790 791 /* 792 * If the tracepoint has already been enabled, just add our id to the 793 * list of interested probes. This may be our second time through 794 * this path in which case we'll have constructed the tracepoint we'd 795 * like to install. If we can't find a match, and have an allocated 796 * tracepoint ready to go, enable that one now. 797 * 798 * A tracepoint whose process is defunct is also considered defunct. 799 */ 800again: 801 mutex_enter(&bucket->ftb_mtx); 802 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 803 /* 804 * Note that it's safe to access the active count on the 805 * associated proc structure because we know that at least one 806 * provider (this one) will still be around throughout this 807 * operation. 808 */ 809 if (tp->ftt_pid != pid || tp->ftt_pc != pc || 810 tp->ftt_proc->ftpc_acount == 0) 811 continue; 812 813 /* 814 * Now that we've found a matching tracepoint, it would be 815 * a decent idea to confirm that the tracepoint is still 816 * enabled and the trap instruction hasn't been overwritten. 817 * Since this is a little hairy, we'll punt for now. 818 */ 819 820 /* 821 * This can't be the first interested probe. We don't have 822 * to worry about another thread being in the midst of 823 * deleting this tracepoint (which would be the only valid 824 * reason for a tracepoint to have no interested probes) 825 * since we're holding P_PR_LOCK for this process. 826 */ 827 ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL); 828 829 switch (id->fti_ptype) { 830 case DTFTP_ENTRY: 831 case DTFTP_OFFSETS: 832 case DTFTP_IS_ENABLED: 833 id->fti_next = tp->ftt_ids; 834 membar_producer(); 835 tp->ftt_ids = id; 836 membar_producer(); 837 break; 838 839 case DTFTP_RETURN: 840 case DTFTP_POST_OFFSETS: 841 id->fti_next = tp->ftt_retids; 842 membar_producer(); 843 tp->ftt_retids = id; 844 membar_producer(); 845 break; 846 847 default: 848 ASSERT(0); 849 } 850 851 mutex_exit(&bucket->ftb_mtx); 852 853 if (new_tp != NULL) { 854 new_tp->ftt_ids = NULL; 855 new_tp->ftt_retids = NULL; 856 } 857 858 return (0); 859 } 860 861 /* 862 * If we have a good tracepoint ready to go, install it now while 863 * we have the lock held and no one can screw with us. 864 */ 865 if (new_tp != NULL) { 866 int rc = 0; 867 868 new_tp->ftt_next = bucket->ftb_data; 869 membar_producer(); 870 bucket->ftb_data = new_tp; 871 membar_producer(); 872 mutex_exit(&bucket->ftb_mtx); 873 874 /* 875 * Activate the tracepoint in the ISA-specific manner. 876 * If this fails, we need to report the failure, but 877 * indicate that this tracepoint must still be disabled 878 * by calling fasttrap_tracepoint_disable(). 879 */ 880 if (fasttrap_tracepoint_install(p, new_tp) != 0) 881 rc = FASTTRAP_ENABLE_PARTIAL; 882 883 /* 884 * Increment the count of the number of tracepoints active in 885 * the victim process. 886 */ 887#ifdef illumos 888 ASSERT(p->p_proc_flag & P_PR_LOCK); 889#endif 890 p->p_dtrace_count++; 891 892 return (rc); 893 } 894 895 mutex_exit(&bucket->ftb_mtx); 896 897 /* 898 * Initialize the tracepoint that's been preallocated with the probe. 899 */ 900 new_tp = probe->ftp_tps[index].fit_tp; 901 902 ASSERT(new_tp->ftt_pid == pid); 903 ASSERT(new_tp->ftt_pc == pc); 904 ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc); 905 ASSERT(new_tp->ftt_ids == NULL); 906 ASSERT(new_tp->ftt_retids == NULL); 907 908 switch (id->fti_ptype) { 909 case DTFTP_ENTRY: 910 case DTFTP_OFFSETS: 911 case DTFTP_IS_ENABLED: 912 id->fti_next = NULL; 913 new_tp->ftt_ids = id; 914 break; 915 916 case DTFTP_RETURN: 917 case DTFTP_POST_OFFSETS: 918 id->fti_next = NULL; 919 new_tp->ftt_retids = id; 920 break; 921 922 default: 923 ASSERT(0); 924 } 925 926 /* 927 * If the ISA-dependent initialization goes to plan, go back to the 928 * beginning and try to install this freshly made tracepoint. 929 */ 930 if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0) 931 goto again; 932 933 new_tp->ftt_ids = NULL; 934 new_tp->ftt_retids = NULL; 935 936 return (FASTTRAP_ENABLE_FAIL); 937} 938 939static void 940fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 941{ 942 fasttrap_bucket_t *bucket; 943 fasttrap_provider_t *provider = probe->ftp_prov; 944 fasttrap_tracepoint_t **pp, *tp; 945 fasttrap_id_t *id, **idp = NULL; 946 pid_t pid; 947 uintptr_t pc; 948 949 ASSERT(index < probe->ftp_ntps); 950 951 pid = probe->ftp_pid; 952 pc = probe->ftp_tps[index].fit_tp->ftt_pc; 953 id = &probe->ftp_tps[index].fit_id; 954 955 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 956 957 /* 958 * Find the tracepoint and make sure that our id is one of the 959 * ones registered with it. 960 */ 961 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 962 mutex_enter(&bucket->ftb_mtx); 963 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 964 if (tp->ftt_pid == pid && tp->ftt_pc == pc && 965 tp->ftt_proc == provider->ftp_proc) 966 break; 967 } 968 969 /* 970 * If we somehow lost this tracepoint, we're in a world of hurt. 971 */ 972 ASSERT(tp != NULL); 973 974 switch (id->fti_ptype) { 975 case DTFTP_ENTRY: 976 case DTFTP_OFFSETS: 977 case DTFTP_IS_ENABLED: 978 ASSERT(tp->ftt_ids != NULL); 979 idp = &tp->ftt_ids; 980 break; 981 982 case DTFTP_RETURN: 983 case DTFTP_POST_OFFSETS: 984 ASSERT(tp->ftt_retids != NULL); 985 idp = &tp->ftt_retids; 986 break; 987 988 default: 989 ASSERT(0); 990 } 991 992 while ((*idp)->fti_probe != probe) { 993 idp = &(*idp)->fti_next; 994 ASSERT(*idp != NULL); 995 } 996 997 id = *idp; 998 *idp = id->fti_next; 999 membar_producer(); 1000 1001 ASSERT(id->fti_probe == probe); 1002 1003 /* 1004 * If there are other registered enablings of this tracepoint, we're 1005 * all done, but if this was the last probe assocated with this 1006 * this tracepoint, we need to remove and free it. 1007 */ 1008 if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) { 1009 1010 /* 1011 * If the current probe's tracepoint is in use, swap it 1012 * for an unused tracepoint. 1013 */ 1014 if (tp == probe->ftp_tps[index].fit_tp) { 1015 fasttrap_probe_t *tmp_probe; 1016 fasttrap_tracepoint_t **tmp_tp; 1017 uint_t tmp_index; 1018 1019 if (tp->ftt_ids != NULL) { 1020 tmp_probe = tp->ftt_ids->fti_probe; 1021 /* LINTED - alignment */ 1022 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids); 1023 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 1024 } else { 1025 tmp_probe = tp->ftt_retids->fti_probe; 1026 /* LINTED - alignment */ 1027 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids); 1028 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 1029 } 1030 1031 ASSERT(*tmp_tp != NULL); 1032 ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp); 1033 ASSERT((*tmp_tp)->ftt_ids == NULL); 1034 ASSERT((*tmp_tp)->ftt_retids == NULL); 1035 1036 probe->ftp_tps[index].fit_tp = *tmp_tp; 1037 *tmp_tp = tp; 1038 } 1039 1040 mutex_exit(&bucket->ftb_mtx); 1041 1042 /* 1043 * Tag the modified probe with the generation in which it was 1044 * changed. 1045 */ 1046 probe->ftp_gen = fasttrap_mod_gen; 1047 return; 1048 } 1049 1050 mutex_exit(&bucket->ftb_mtx); 1051 1052 /* 1053 * We can't safely remove the tracepoint from the set of active 1054 * tracepoints until we've actually removed the fasttrap instruction 1055 * from the process's text. We can, however, operate on this 1056 * tracepoint secure in the knowledge that no other thread is going to 1057 * be looking at it since we hold P_PR_LOCK on the process if it's 1058 * live or we hold the provider lock on the process if it's dead and 1059 * gone. 1060 */ 1061 1062 /* 1063 * We only need to remove the actual instruction if we're looking 1064 * at an existing process 1065 */ 1066 if (p != NULL) { 1067 /* 1068 * If we fail to restore the instruction we need to kill 1069 * this process since it's in a completely unrecoverable 1070 * state. 1071 */ 1072 if (fasttrap_tracepoint_remove(p, tp) != 0) 1073 fasttrap_sigtrap(p, NULL, pc); 1074 1075 /* 1076 * Decrement the count of the number of tracepoints active 1077 * in the victim process. 1078 */ 1079#ifdef illumos 1080 ASSERT(p->p_proc_flag & P_PR_LOCK); 1081#endif 1082 p->p_dtrace_count--; 1083 } 1084 1085 /* 1086 * Remove the probe from the hash table of active tracepoints. 1087 */ 1088 mutex_enter(&bucket->ftb_mtx); 1089 pp = (fasttrap_tracepoint_t **)&bucket->ftb_data; 1090 ASSERT(*pp != NULL); 1091 while (*pp != tp) { 1092 pp = &(*pp)->ftt_next; 1093 ASSERT(*pp != NULL); 1094 } 1095 1096 *pp = tp->ftt_next; 1097 membar_producer(); 1098 1099 mutex_exit(&bucket->ftb_mtx); 1100 1101 /* 1102 * Tag the modified probe with the generation in which it was changed. 1103 */ 1104 probe->ftp_gen = fasttrap_mod_gen; 1105} 1106 1107static void 1108fasttrap_enable_callbacks(void) 1109{ 1110 /* 1111 * We don't have to play the rw lock game here because we're 1112 * providing something rather than taking something away -- 1113 * we can be sure that no threads have tried to follow this 1114 * function pointer yet. 1115 */ 1116 mutex_enter(&fasttrap_count_mtx); 1117 if (fasttrap_pid_count == 0) { 1118 ASSERT(dtrace_pid_probe_ptr == NULL); 1119 ASSERT(dtrace_return_probe_ptr == NULL); 1120 dtrace_pid_probe_ptr = &fasttrap_pid_probe; 1121 dtrace_return_probe_ptr = &fasttrap_return_probe; 1122 } 1123 ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe); 1124 ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe); 1125 fasttrap_pid_count++; 1126 mutex_exit(&fasttrap_count_mtx); 1127} 1128 1129static void 1130fasttrap_disable_callbacks(void) 1131{ 1132#ifdef illumos 1133 ASSERT(MUTEX_HELD(&cpu_lock)); 1134#endif 1135 1136 1137 mutex_enter(&fasttrap_count_mtx); 1138 ASSERT(fasttrap_pid_count > 0); 1139 fasttrap_pid_count--; 1140 if (fasttrap_pid_count == 0) { 1141#ifdef illumos 1142 cpu_t *cur, *cpu = CPU; 1143 1144 for (cur = cpu->cpu_next_onln; cur != cpu; 1145 cur = cur->cpu_next_onln) { 1146 rw_enter(&cur->cpu_ft_lock, RW_WRITER); 1147 } 1148#endif 1149 dtrace_pid_probe_ptr = NULL; 1150 dtrace_return_probe_ptr = NULL; 1151#ifdef illumos 1152 for (cur = cpu->cpu_next_onln; cur != cpu; 1153 cur = cur->cpu_next_onln) { 1154 rw_exit(&cur->cpu_ft_lock); 1155 } 1156#endif 1157 } 1158 mutex_exit(&fasttrap_count_mtx); 1159} 1160 1161/*ARGSUSED*/ 1162static void 1163fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg) 1164{ 1165 fasttrap_probe_t *probe = parg; 1166 proc_t *p = NULL; 1167 int i, rc; 1168 1169 ASSERT(probe != NULL); 1170 ASSERT(!probe->ftp_enabled); 1171 ASSERT(id == probe->ftp_id); 1172#ifdef illumos 1173 ASSERT(MUTEX_HELD(&cpu_lock)); 1174#endif 1175 1176 /* 1177 * Increment the count of enabled probes on this probe's provider; 1178 * the provider can't go away while the probe still exists. We 1179 * must increment this even if we aren't able to properly enable 1180 * this probe. 1181 */ 1182 mutex_enter(&probe->ftp_prov->ftp_mtx); 1183 probe->ftp_prov->ftp_rcount++; 1184 mutex_exit(&probe->ftp_prov->ftp_mtx); 1185 1186 /* 1187 * If this probe's provider is retired (meaning it was valid in a 1188 * previously exec'ed incarnation of this address space), bail out. The 1189 * provider can't go away while we're in this code path. 1190 */ 1191 if (probe->ftp_prov->ftp_retired) 1192 return; 1193 1194 /* 1195 * If we can't find the process, it may be that we're in the context of 1196 * a fork in which the traced process is being born and we're copying 1197 * USDT probes. Otherwise, the process is gone so bail. 1198 */ 1199#ifdef illumos 1200 if ((p = sprlock(probe->ftp_pid)) == NULL) { 1201 if ((curproc->p_flag & SFORKING) == 0) 1202 return; 1203 1204 mutex_enter(&pidlock); 1205 p = prfind(probe->ftp_pid); 1206 1207 /* 1208 * Confirm that curproc is indeed forking the process in which 1209 * we're trying to enable probes. 1210 */ 1211 ASSERT(p != NULL); 1212 ASSERT(p->p_parent == curproc); 1213 ASSERT(p->p_stat == SIDL); 1214 1215 mutex_enter(&p->p_lock); 1216 mutex_exit(&pidlock); 1217 1218 sprlock_proc(p); 1219 } 1220 1221 ASSERT(!(p->p_flag & SVFORK)); 1222 mutex_exit(&p->p_lock); 1223#else 1224 if ((p = pfind(probe->ftp_pid)) == NULL) 1225 return; 1226#endif 1227 1228 /* 1229 * We have to enable the trap entry point before any user threads have 1230 * the chance to execute the trap instruction we're about to place 1231 * in their process's text. 1232 */ 1233#ifdef __FreeBSD__ 1234 /* 1235 * pfind() returns a locked process. 1236 */ 1237 _PHOLD(p); 1238 PROC_UNLOCK(p); 1239#endif 1240 fasttrap_enable_callbacks(); 1241 1242 /* 1243 * Enable all the tracepoints and add this probe's id to each 1244 * tracepoint's list of active probes. 1245 */ 1246 for (i = 0; i < probe->ftp_ntps; i++) { 1247 if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) { 1248 /* 1249 * If enabling the tracepoint failed completely, 1250 * we don't have to disable it; if the failure 1251 * was only partial we must disable it. 1252 */ 1253 if (rc == FASTTRAP_ENABLE_FAIL) 1254 i--; 1255 else 1256 ASSERT(rc == FASTTRAP_ENABLE_PARTIAL); 1257 1258 /* 1259 * Back up and pull out all the tracepoints we've 1260 * created so far for this probe. 1261 */ 1262 while (i >= 0) { 1263 fasttrap_tracepoint_disable(p, probe, i); 1264 i--; 1265 } 1266 1267#ifdef illumos 1268 mutex_enter(&p->p_lock); 1269 sprunlock(p); 1270#else 1271 PRELE(p); 1272#endif 1273 1274 /* 1275 * Since we're not actually enabling this probe, 1276 * drop our reference on the trap table entry. 1277 */ 1278 fasttrap_disable_callbacks(); 1279 return; 1280 } 1281 } 1282#ifdef illumos 1283 mutex_enter(&p->p_lock); 1284 sprunlock(p); 1285#else 1286 PRELE(p); 1287#endif 1288 1289 probe->ftp_enabled = 1; 1290} 1291 1292/*ARGSUSED*/ 1293static void 1294fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg) 1295{ 1296 fasttrap_probe_t *probe = parg; 1297 fasttrap_provider_t *provider = probe->ftp_prov; 1298 proc_t *p; 1299 int i, whack = 0; 1300 1301 ASSERT(id == probe->ftp_id); 1302 1303 mutex_enter(&provider->ftp_mtx); 1304 1305 /* 1306 * We won't be able to acquire a /proc-esque lock on the process 1307 * iff the process is dead and gone. In this case, we rely on the 1308 * provider lock as a point of mutual exclusion to prevent other 1309 * DTrace consumers from disabling this probe. 1310 */ 1311 if ((p = pfind(probe->ftp_pid)) != NULL) { 1312#ifdef __FreeBSD__ 1313 if (p->p_flag & P_WEXIT) { 1314 PROC_UNLOCK(p); 1315 p = NULL; 1316 } else { 1317 _PHOLD(p); 1318 PROC_UNLOCK(p); 1319 } 1320#endif 1321 } 1322 1323 /* 1324 * Disable all the associated tracepoints (for fully enabled probes). 1325 */ 1326 if (probe->ftp_enabled) { 1327 for (i = 0; i < probe->ftp_ntps; i++) { 1328 fasttrap_tracepoint_disable(p, probe, i); 1329 } 1330 } 1331 1332 ASSERT(provider->ftp_rcount > 0); 1333 provider->ftp_rcount--; 1334 1335 if (p != NULL) { 1336 /* 1337 * Even though we may not be able to remove it entirely, we 1338 * mark this retired provider to get a chance to remove some 1339 * of the associated probes. 1340 */ 1341 if (provider->ftp_retired && !provider->ftp_marked) 1342 whack = provider->ftp_marked = 1; 1343 mutex_exit(&provider->ftp_mtx); 1344 } else { 1345 /* 1346 * If the process is dead, we're just waiting for the 1347 * last probe to be disabled to be able to free it. 1348 */ 1349 if (provider->ftp_rcount == 0 && !provider->ftp_marked) 1350 whack = provider->ftp_marked = 1; 1351 mutex_exit(&provider->ftp_mtx); 1352 } 1353 1354 if (whack) 1355 fasttrap_pid_cleanup(); 1356 1357#ifdef __FreeBSD__ 1358 if (p != NULL) 1359 PRELE(p); 1360#endif 1361 if (!probe->ftp_enabled) 1362 return; 1363 1364 probe->ftp_enabled = 0; 1365 1366#ifdef illumos 1367 ASSERT(MUTEX_HELD(&cpu_lock)); 1368#endif 1369 fasttrap_disable_callbacks(); 1370} 1371 1372/*ARGSUSED*/ 1373static void 1374fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg, 1375 dtrace_argdesc_t *desc) 1376{ 1377 fasttrap_probe_t *probe = parg; 1378 char *str; 1379 int i, ndx; 1380 1381 desc->dtargd_native[0] = '\0'; 1382 desc->dtargd_xlate[0] = '\0'; 1383 1384 if (probe->ftp_prov->ftp_retired != 0 || 1385 desc->dtargd_ndx >= probe->ftp_nargs) { 1386 desc->dtargd_ndx = DTRACE_ARGNONE; 1387 return; 1388 } 1389 1390 ndx = (probe->ftp_argmap != NULL) ? 1391 probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx; 1392 1393 str = probe->ftp_ntypes; 1394 for (i = 0; i < ndx; i++) { 1395 str += strlen(str) + 1; 1396 } 1397 1398 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native)); 1399 (void) strcpy(desc->dtargd_native, str); 1400 1401 if (probe->ftp_xtypes == NULL) 1402 return; 1403 1404 str = probe->ftp_xtypes; 1405 for (i = 0; i < desc->dtargd_ndx; i++) { 1406 str += strlen(str) + 1; 1407 } 1408 1409 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate)); 1410 (void) strcpy(desc->dtargd_xlate, str); 1411} 1412 1413/*ARGSUSED*/ 1414static void 1415fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg) 1416{ 1417 fasttrap_probe_t *probe = parg; 1418 int i; 1419 size_t size; 1420 1421 ASSERT(probe != NULL); 1422 ASSERT(!probe->ftp_enabled); 1423 ASSERT(fasttrap_total >= probe->ftp_ntps); 1424 1425 atomic_add_32(&fasttrap_total, -probe->ftp_ntps); 1426 size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]); 1427 1428 if (probe->ftp_gen + 1 >= fasttrap_mod_gen) 1429 fasttrap_mod_barrier(probe->ftp_gen); 1430 1431 for (i = 0; i < probe->ftp_ntps; i++) { 1432 kmem_free(probe->ftp_tps[i].fit_tp, 1433 sizeof (fasttrap_tracepoint_t)); 1434 } 1435 1436 kmem_free(probe, size); 1437} 1438 1439 1440static const dtrace_pattr_t pid_attr = { 1441{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1442{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1443{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1444{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1445{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1446}; 1447 1448static dtrace_pops_t pid_pops = { 1449 fasttrap_pid_provide, 1450 NULL, 1451 fasttrap_pid_enable, 1452 fasttrap_pid_disable, 1453 NULL, 1454 NULL, 1455 fasttrap_pid_getargdesc, 1456 fasttrap_pid_getarg, 1457 NULL, 1458 fasttrap_pid_destroy 1459}; 1460 1461static dtrace_pops_t usdt_pops = { 1462 fasttrap_pid_provide, 1463 NULL, 1464 fasttrap_pid_enable, 1465 fasttrap_pid_disable, 1466 NULL, 1467 NULL, 1468 fasttrap_pid_getargdesc, 1469 fasttrap_usdt_getarg, 1470 NULL, 1471 fasttrap_pid_destroy 1472}; 1473 1474static fasttrap_proc_t * 1475fasttrap_proc_lookup(pid_t pid) 1476{ 1477 fasttrap_bucket_t *bucket; 1478 fasttrap_proc_t *fprc, *new_fprc; 1479 1480 1481 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1482 mutex_enter(&bucket->ftb_mtx); 1483 1484 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1485 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1486 mutex_enter(&fprc->ftpc_mtx); 1487 mutex_exit(&bucket->ftb_mtx); 1488 fprc->ftpc_rcount++; 1489 atomic_inc_64(&fprc->ftpc_acount); 1490 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1491 mutex_exit(&fprc->ftpc_mtx); 1492 1493 return (fprc); 1494 } 1495 } 1496 1497 /* 1498 * Drop the bucket lock so we don't try to perform a sleeping 1499 * allocation under it. 1500 */ 1501 mutex_exit(&bucket->ftb_mtx); 1502 1503 new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP); 1504 new_fprc->ftpc_pid = pid; 1505 new_fprc->ftpc_rcount = 1; 1506 new_fprc->ftpc_acount = 1; 1507#ifndef illumos 1508 mutex_init(&new_fprc->ftpc_mtx, "fasttrap proc mtx", MUTEX_DEFAULT, 1509 NULL); 1510#endif 1511 1512 mutex_enter(&bucket->ftb_mtx); 1513 1514 /* 1515 * Take another lap through the list to make sure a proc hasn't 1516 * been created for this pid while we weren't under the bucket lock. 1517 */ 1518 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1519 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1520 mutex_enter(&fprc->ftpc_mtx); 1521 mutex_exit(&bucket->ftb_mtx); 1522 fprc->ftpc_rcount++; 1523 atomic_inc_64(&fprc->ftpc_acount); 1524 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1525 mutex_exit(&fprc->ftpc_mtx); 1526 1527 kmem_free(new_fprc, sizeof (fasttrap_proc_t)); 1528 1529 return (fprc); 1530 } 1531 } 1532 1533 new_fprc->ftpc_next = bucket->ftb_data; 1534 bucket->ftb_data = new_fprc; 1535 1536 mutex_exit(&bucket->ftb_mtx); 1537 1538 return (new_fprc); 1539} 1540 1541static void 1542fasttrap_proc_release(fasttrap_proc_t *proc) 1543{ 1544 fasttrap_bucket_t *bucket; 1545 fasttrap_proc_t *fprc, **fprcp; 1546 pid_t pid = proc->ftpc_pid; 1547#ifndef illumos 1548 fasttrap_scrblock_t *scrblk, *scrblktmp; 1549 fasttrap_scrspace_t *scrspc, *scrspctmp; 1550 struct proc *p; 1551 struct thread *td; 1552#endif 1553 1554 mutex_enter(&proc->ftpc_mtx); 1555 1556 ASSERT(proc->ftpc_rcount != 0); 1557 ASSERT(proc->ftpc_acount <= proc->ftpc_rcount); 1558 1559 if (--proc->ftpc_rcount != 0) { 1560 mutex_exit(&proc->ftpc_mtx); 1561 return; 1562 } 1563 1564#ifndef illumos 1565 /* 1566 * Free all structures used to manage per-thread scratch space. 1567 */ 1568 LIST_FOREACH_SAFE(scrblk, &proc->ftpc_scrblks, ftsb_next, 1569 scrblktmp) { 1570 LIST_REMOVE(scrblk, ftsb_next); 1571 free(scrblk, M_SOLARIS); 1572 } 1573 LIST_FOREACH_SAFE(scrspc, &proc->ftpc_fscr, ftss_next, scrspctmp) { 1574 LIST_REMOVE(scrspc, ftss_next); 1575 free(scrspc, M_SOLARIS); 1576 } 1577 LIST_FOREACH_SAFE(scrspc, &proc->ftpc_ascr, ftss_next, scrspctmp) { 1578 LIST_REMOVE(scrspc, ftss_next); 1579 free(scrspc, M_SOLARIS); 1580 } 1581 1582 if ((p = pfind(pid)) != NULL) { 1583 FOREACH_THREAD_IN_PROC(p, td) 1584 td->t_dtrace_sscr = NULL; 1585 PROC_UNLOCK(p); 1586 } 1587#endif 1588 1589 mutex_exit(&proc->ftpc_mtx); 1590 1591 /* 1592 * There should definitely be no live providers associated with this 1593 * process at this point. 1594 */ 1595 ASSERT(proc->ftpc_acount == 0); 1596 1597 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1598 mutex_enter(&bucket->ftb_mtx); 1599 1600 fprcp = (fasttrap_proc_t **)&bucket->ftb_data; 1601 while ((fprc = *fprcp) != NULL) { 1602 if (fprc == proc) 1603 break; 1604 1605 fprcp = &fprc->ftpc_next; 1606 } 1607 1608 /* 1609 * Something strange has happened if we can't find the proc. 1610 */ 1611 ASSERT(fprc != NULL); 1612 1613 *fprcp = fprc->ftpc_next; 1614 1615 mutex_exit(&bucket->ftb_mtx); 1616 1617 kmem_free(fprc, sizeof (fasttrap_proc_t)); 1618} 1619 1620/* 1621 * Lookup a fasttrap-managed provider based on its name and associated pid. 1622 * If the pattr argument is non-NULL, this function instantiates the provider 1623 * if it doesn't exist otherwise it returns NULL. The provider is returned 1624 * with its lock held. 1625 */ 1626static fasttrap_provider_t * 1627fasttrap_provider_lookup(pid_t pid, const char *name, 1628 const dtrace_pattr_t *pattr) 1629{ 1630 fasttrap_provider_t *fp, *new_fp = NULL; 1631 fasttrap_bucket_t *bucket; 1632 char provname[DTRACE_PROVNAMELEN]; 1633 proc_t *p; 1634 cred_t *cred; 1635 1636 ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1637 ASSERT(pattr != NULL); 1638 1639 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1640 mutex_enter(&bucket->ftb_mtx); 1641 1642 /* 1643 * Take a lap through the list and return the match if we find it. 1644 */ 1645 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1646 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1647 !fp->ftp_retired) { 1648 mutex_enter(&fp->ftp_mtx); 1649 mutex_exit(&bucket->ftb_mtx); 1650 return (fp); 1651 } 1652 } 1653 1654 /* 1655 * Drop the bucket lock so we don't try to perform a sleeping 1656 * allocation under it. 1657 */ 1658 mutex_exit(&bucket->ftb_mtx); 1659 1660 /* 1661 * Make sure the process exists, isn't a child created as the result 1662 * of a vfork(2), and isn't a zombie (but may be in fork). 1663 */ 1664 if ((p = pfind(pid)) == NULL) 1665 return (NULL); 1666 1667 /* 1668 * Increment p_dtrace_probes so that the process knows to inform us 1669 * when it exits or execs. fasttrap_provider_free() decrements this 1670 * when we're done with this provider. 1671 */ 1672 p->p_dtrace_probes++; 1673 1674 /* 1675 * Grab the credentials for this process so we have 1676 * something to pass to dtrace_register(). 1677 */ 1678 PROC_LOCK_ASSERT(p, MA_OWNED); 1679 crhold(p->p_ucred); 1680 cred = p->p_ucred; 1681 PROC_UNLOCK(p); 1682 1683 new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP); 1684 new_fp->ftp_pid = pid; 1685 new_fp->ftp_proc = fasttrap_proc_lookup(pid); 1686#ifndef illumos 1687 mutex_init(&new_fp->ftp_mtx, "provider mtx", MUTEX_DEFAULT, NULL); 1688 mutex_init(&new_fp->ftp_cmtx, "lock on creating", MUTEX_DEFAULT, NULL); 1689#endif 1690 1691 ASSERT(new_fp->ftp_proc != NULL); 1692 1693 mutex_enter(&bucket->ftb_mtx); 1694 1695 /* 1696 * Take another lap through the list to make sure a provider hasn't 1697 * been created for this pid while we weren't under the bucket lock. 1698 */ 1699 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1700 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1701 !fp->ftp_retired) { 1702 mutex_enter(&fp->ftp_mtx); 1703 mutex_exit(&bucket->ftb_mtx); 1704 fasttrap_provider_free(new_fp); 1705 crfree(cred); 1706 return (fp); 1707 } 1708 } 1709 1710 (void) strcpy(new_fp->ftp_name, name); 1711 1712 /* 1713 * Fail and return NULL if either the provider name is too long 1714 * or we fail to register this new provider with the DTrace 1715 * framework. Note that this is the only place we ever construct 1716 * the full provider name -- we keep it in pieces in the provider 1717 * structure. 1718 */ 1719 if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >= 1720 sizeof (provname) || 1721 dtrace_register(provname, pattr, 1722 DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred, 1723 pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp, 1724 &new_fp->ftp_provid) != 0) { 1725 mutex_exit(&bucket->ftb_mtx); 1726 fasttrap_provider_free(new_fp); 1727 crfree(cred); 1728 return (NULL); 1729 } 1730 1731 new_fp->ftp_next = bucket->ftb_data; 1732 bucket->ftb_data = new_fp; 1733 1734 mutex_enter(&new_fp->ftp_mtx); 1735 mutex_exit(&bucket->ftb_mtx); 1736 1737 crfree(cred); 1738 return (new_fp); 1739} 1740 1741static void 1742fasttrap_provider_free(fasttrap_provider_t *provider) 1743{ 1744 pid_t pid = provider->ftp_pid; 1745 proc_t *p; 1746 1747 /* 1748 * There need to be no associated enabled probes, no consumers 1749 * creating probes, and no meta providers referencing this provider. 1750 */ 1751 ASSERT(provider->ftp_rcount == 0); 1752 ASSERT(provider->ftp_ccount == 0); 1753 ASSERT(provider->ftp_mcount == 0); 1754 1755 /* 1756 * If this provider hasn't been retired, we need to explicitly drop the 1757 * count of active providers on the associated process structure. 1758 */ 1759 if (!provider->ftp_retired) { 1760 atomic_dec_64(&provider->ftp_proc->ftpc_acount); 1761 ASSERT(provider->ftp_proc->ftpc_acount < 1762 provider->ftp_proc->ftpc_rcount); 1763 } 1764 1765 fasttrap_proc_release(provider->ftp_proc); 1766 1767#ifndef illumos 1768 mutex_destroy(&provider->ftp_mtx); 1769 mutex_destroy(&provider->ftp_cmtx); 1770#endif 1771 kmem_free(provider, sizeof (fasttrap_provider_t)); 1772 1773 /* 1774 * Decrement p_dtrace_probes on the process whose provider we're 1775 * freeing. We don't have to worry about clobbering somone else's 1776 * modifications to it because we have locked the bucket that 1777 * corresponds to this process's hash chain in the provider hash 1778 * table. Don't sweat it if we can't find the process. 1779 */ 1780 if ((p = pfind(pid)) == NULL) { 1781 return; 1782 } 1783 1784 p->p_dtrace_probes--; 1785#ifndef illumos 1786 PROC_UNLOCK(p); 1787#endif 1788} 1789 1790static void 1791fasttrap_provider_retire(pid_t pid, const char *name, int mprov) 1792{ 1793 fasttrap_provider_t *fp; 1794 fasttrap_bucket_t *bucket; 1795 dtrace_provider_id_t provid; 1796 1797 ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1798 1799 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1800 mutex_enter(&bucket->ftb_mtx); 1801 1802 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1803 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1804 !fp->ftp_retired) 1805 break; 1806 } 1807 1808 if (fp == NULL) { 1809 mutex_exit(&bucket->ftb_mtx); 1810 return; 1811 } 1812 1813 mutex_enter(&fp->ftp_mtx); 1814 ASSERT(!mprov || fp->ftp_mcount > 0); 1815 if (mprov && --fp->ftp_mcount != 0) { 1816 mutex_exit(&fp->ftp_mtx); 1817 mutex_exit(&bucket->ftb_mtx); 1818 return; 1819 } 1820 1821 /* 1822 * Mark the provider to be removed in our post-processing step, mark it 1823 * retired, and drop the active count on its proc. Marking it indicates 1824 * that we should try to remove it; setting the retired flag indicates 1825 * that we're done with this provider; dropping the active the proc 1826 * releases our hold, and when this reaches zero (as it will during 1827 * exit or exec) the proc and associated providers become defunct. 1828 * 1829 * We obviously need to take the bucket lock before the provider lock 1830 * to perform the lookup, but we need to drop the provider lock 1831 * before calling into the DTrace framework since we acquire the 1832 * provider lock in callbacks invoked from the DTrace framework. The 1833 * bucket lock therefore protects the integrity of the provider hash 1834 * table. 1835 */ 1836 atomic_dec_64(&fp->ftp_proc->ftpc_acount); 1837 ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount); 1838 1839 fp->ftp_retired = 1; 1840 fp->ftp_marked = 1; 1841 provid = fp->ftp_provid; 1842 mutex_exit(&fp->ftp_mtx); 1843 1844 /* 1845 * We don't have to worry about invalidating the same provider twice 1846 * since fasttrap_provider_lookup() will ignore provider that have 1847 * been marked as retired. 1848 */ 1849 dtrace_invalidate(provid); 1850 1851 mutex_exit(&bucket->ftb_mtx); 1852 1853 fasttrap_pid_cleanup(); 1854} 1855 1856static int 1857fasttrap_uint32_cmp(const void *ap, const void *bp) 1858{ 1859 return (*(const uint32_t *)ap - *(const uint32_t *)bp); 1860} 1861 1862static int 1863fasttrap_uint64_cmp(const void *ap, const void *bp) 1864{ 1865 return (*(const uint64_t *)ap - *(const uint64_t *)bp); 1866} 1867 1868static int 1869fasttrap_add_probe(fasttrap_probe_spec_t *pdata) 1870{ 1871 fasttrap_provider_t *provider; 1872 fasttrap_probe_t *pp; 1873 fasttrap_tracepoint_t *tp; 1874 char *name; 1875 int i, aframes = 0, whack; 1876 1877 /* 1878 * There needs to be at least one desired trace point. 1879 */ 1880 if (pdata->ftps_noffs == 0) 1881 return (EINVAL); 1882 1883 switch (pdata->ftps_type) { 1884 case DTFTP_ENTRY: 1885 name = "entry"; 1886 aframes = FASTTRAP_ENTRY_AFRAMES; 1887 break; 1888 case DTFTP_RETURN: 1889 name = "return"; 1890 aframes = FASTTRAP_RETURN_AFRAMES; 1891 break; 1892 case DTFTP_OFFSETS: 1893 name = NULL; 1894 break; 1895 default: 1896 return (EINVAL); 1897 } 1898 1899 if ((provider = fasttrap_provider_lookup(pdata->ftps_pid, 1900 FASTTRAP_PID_NAME, &pid_attr)) == NULL) 1901 return (ESRCH); 1902 1903 /* 1904 * Increment this reference count to indicate that a consumer is 1905 * actively adding a new probe associated with this provider. This 1906 * prevents the provider from being deleted -- we'll need to check 1907 * for pending deletions when we drop this reference count. 1908 */ 1909 provider->ftp_ccount++; 1910 mutex_exit(&provider->ftp_mtx); 1911 1912 /* 1913 * Grab the creation lock to ensure consistency between calls to 1914 * dtrace_probe_lookup() and dtrace_probe_create() in the face of 1915 * other threads creating probes. We must drop the provider lock 1916 * before taking this lock to avoid a three-way deadlock with the 1917 * DTrace framework. 1918 */ 1919 mutex_enter(&provider->ftp_cmtx); 1920 1921 if (name == NULL) { 1922 for (i = 0; i < pdata->ftps_noffs; i++) { 1923 char name_str[17]; 1924 1925 (void) sprintf(name_str, "%llx", 1926 (unsigned long long)pdata->ftps_offs[i]); 1927 1928 if (dtrace_probe_lookup(provider->ftp_provid, 1929 pdata->ftps_mod, pdata->ftps_func, name_str) != 0) 1930 continue; 1931 1932 atomic_inc_32(&fasttrap_total); 1933 1934 if (fasttrap_total > fasttrap_max) { 1935 atomic_dec_32(&fasttrap_total); 1936 goto no_mem; 1937 } 1938 1939 pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP); 1940 1941 pp->ftp_prov = provider; 1942 pp->ftp_faddr = pdata->ftps_pc; 1943 pp->ftp_fsize = pdata->ftps_size; 1944 pp->ftp_pid = pdata->ftps_pid; 1945 pp->ftp_ntps = 1; 1946 1947 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1948 KM_SLEEP); 1949 1950 tp->ftt_proc = provider->ftp_proc; 1951 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 1952 tp->ftt_pid = pdata->ftps_pid; 1953 1954 pp->ftp_tps[0].fit_tp = tp; 1955 pp->ftp_tps[0].fit_id.fti_probe = pp; 1956 pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type; 1957 1958 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 1959 pdata->ftps_mod, pdata->ftps_func, name_str, 1960 FASTTRAP_OFFSET_AFRAMES, pp); 1961 } 1962 1963 } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod, 1964 pdata->ftps_func, name) == 0) { 1965 atomic_add_32(&fasttrap_total, pdata->ftps_noffs); 1966 1967 if (fasttrap_total > fasttrap_max) { 1968 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1969 goto no_mem; 1970 } 1971 1972 /* 1973 * Make sure all tracepoint program counter values are unique. 1974 * We later assume that each probe has exactly one tracepoint 1975 * for a given pc. 1976 */ 1977 qsort(pdata->ftps_offs, pdata->ftps_noffs, 1978 sizeof (uint64_t), fasttrap_uint64_cmp); 1979 for (i = 1; i < pdata->ftps_noffs; i++) { 1980 if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1]) 1981 continue; 1982 1983 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1984 goto no_mem; 1985 } 1986 1987 ASSERT(pdata->ftps_noffs > 0); 1988 pp = kmem_zalloc(offsetof(fasttrap_probe_t, 1989 ftp_tps[pdata->ftps_noffs]), KM_SLEEP); 1990 1991 pp->ftp_prov = provider; 1992 pp->ftp_faddr = pdata->ftps_pc; 1993 pp->ftp_fsize = pdata->ftps_size; 1994 pp->ftp_pid = pdata->ftps_pid; 1995 pp->ftp_ntps = pdata->ftps_noffs; 1996 1997 for (i = 0; i < pdata->ftps_noffs; i++) { 1998 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1999 KM_SLEEP); 2000 2001 tp->ftt_proc = provider->ftp_proc; 2002 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 2003 tp->ftt_pid = pdata->ftps_pid; 2004 2005 pp->ftp_tps[i].fit_tp = tp; 2006 pp->ftp_tps[i].fit_id.fti_probe = pp; 2007 pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type; 2008 } 2009 2010 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 2011 pdata->ftps_mod, pdata->ftps_func, name, aframes, pp); 2012 } 2013 2014 mutex_exit(&provider->ftp_cmtx); 2015 2016 /* 2017 * We know that the provider is still valid since we incremented the 2018 * creation reference count. If someone tried to clean up this provider 2019 * while we were using it (e.g. because the process called exec(2) or 2020 * exit(2)), take note of that and try to clean it up now. 2021 */ 2022 mutex_enter(&provider->ftp_mtx); 2023 provider->ftp_ccount--; 2024 whack = provider->ftp_retired; 2025 mutex_exit(&provider->ftp_mtx); 2026 2027 if (whack) 2028 fasttrap_pid_cleanup(); 2029 2030 return (0); 2031 2032no_mem: 2033 /* 2034 * If we've exhausted the allowable resources, we'll try to remove 2035 * this provider to free some up. This is to cover the case where 2036 * the user has accidentally created many more probes than was 2037 * intended (e.g. pid123:::). 2038 */ 2039 mutex_exit(&provider->ftp_cmtx); 2040 mutex_enter(&provider->ftp_mtx); 2041 provider->ftp_ccount--; 2042 provider->ftp_marked = 1; 2043 mutex_exit(&provider->ftp_mtx); 2044 2045 fasttrap_pid_cleanup(); 2046 2047 return (ENOMEM); 2048} 2049 2050/*ARGSUSED*/ 2051static void * 2052fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 2053{ 2054 fasttrap_provider_t *provider; 2055 2056 /* 2057 * A 32-bit unsigned integer (like a pid for example) can be 2058 * expressed in 10 or fewer decimal digits. Make sure that we'll 2059 * have enough space for the provider name. 2060 */ 2061 if (strlen(dhpv->dthpv_provname) + 10 >= 2062 sizeof (provider->ftp_name)) { 2063 printf("failed to instantiate provider %s: " 2064 "name too long to accomodate pid", dhpv->dthpv_provname); 2065 return (NULL); 2066 } 2067 2068 /* 2069 * Don't let folks spoof the true pid provider. 2070 */ 2071 if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) { 2072 printf("failed to instantiate provider %s: " 2073 "%s is an invalid name", dhpv->dthpv_provname, 2074 FASTTRAP_PID_NAME); 2075 return (NULL); 2076 } 2077 2078 /* 2079 * The highest stability class that fasttrap supports is ISA; cap 2080 * the stability of the new provider accordingly. 2081 */ 2082 if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA) 2083 dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA; 2084 if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA) 2085 dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA; 2086 if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA) 2087 dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA; 2088 if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA) 2089 dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA; 2090 if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA) 2091 dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA; 2092 2093 if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname, 2094 &dhpv->dthpv_pattr)) == NULL) { 2095 printf("failed to instantiate provider %s for " 2096 "process %u", dhpv->dthpv_provname, (uint_t)pid); 2097 return (NULL); 2098 } 2099 2100 /* 2101 * Up the meta provider count so this provider isn't removed until 2102 * the meta provider has been told to remove it. 2103 */ 2104 provider->ftp_mcount++; 2105 2106 mutex_exit(&provider->ftp_mtx); 2107 2108 return (provider); 2109} 2110 2111/*ARGSUSED*/ 2112static void 2113fasttrap_meta_create_probe(void *arg, void *parg, 2114 dtrace_helper_probedesc_t *dhpb) 2115{ 2116 fasttrap_provider_t *provider = parg; 2117 fasttrap_probe_t *pp; 2118 fasttrap_tracepoint_t *tp; 2119 int i, j; 2120 uint32_t ntps; 2121 2122 /* 2123 * Since the meta provider count is non-zero we don't have to worry 2124 * about this provider disappearing. 2125 */ 2126 ASSERT(provider->ftp_mcount > 0); 2127 2128 /* 2129 * The offsets must be unique. 2130 */ 2131 qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t), 2132 fasttrap_uint32_cmp); 2133 for (i = 1; i < dhpb->dthpb_noffs; i++) { 2134 if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <= 2135 dhpb->dthpb_base + dhpb->dthpb_offs[i - 1]) 2136 return; 2137 } 2138 2139 qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t), 2140 fasttrap_uint32_cmp); 2141 for (i = 1; i < dhpb->dthpb_nenoffs; i++) { 2142 if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <= 2143 dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1]) 2144 return; 2145 } 2146 2147 /* 2148 * Grab the creation lock to ensure consistency between calls to 2149 * dtrace_probe_lookup() and dtrace_probe_create() in the face of 2150 * other threads creating probes. 2151 */ 2152 mutex_enter(&provider->ftp_cmtx); 2153 2154 if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod, 2155 dhpb->dthpb_func, dhpb->dthpb_name) != 0) { 2156 mutex_exit(&provider->ftp_cmtx); 2157 return; 2158 } 2159 2160 ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs; 2161 ASSERT(ntps > 0); 2162 2163 atomic_add_32(&fasttrap_total, ntps); 2164 2165 if (fasttrap_total > fasttrap_max) { 2166 atomic_add_32(&fasttrap_total, -ntps); 2167 mutex_exit(&provider->ftp_cmtx); 2168 return; 2169 } 2170 2171 pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP); 2172 2173 pp->ftp_prov = provider; 2174 pp->ftp_pid = provider->ftp_pid; 2175 pp->ftp_ntps = ntps; 2176 pp->ftp_nargs = dhpb->dthpb_xargc; 2177 pp->ftp_xtypes = dhpb->dthpb_xtypes; 2178 pp->ftp_ntypes = dhpb->dthpb_ntypes; 2179 2180 /* 2181 * First create a tracepoint for each actual point of interest. 2182 */ 2183 for (i = 0; i < dhpb->dthpb_noffs; i++) { 2184 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 2185 2186 tp->ftt_proc = provider->ftp_proc; 2187 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i]; 2188 tp->ftt_pid = provider->ftp_pid; 2189 2190 pp->ftp_tps[i].fit_tp = tp; 2191 pp->ftp_tps[i].fit_id.fti_probe = pp; 2192#ifdef __sparc 2193 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS; 2194#else 2195 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS; 2196#endif 2197 } 2198 2199 /* 2200 * Then create a tracepoint for each is-enabled point. 2201 */ 2202 for (j = 0; i < ntps; i++, j++) { 2203 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 2204 2205 tp->ftt_proc = provider->ftp_proc; 2206 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j]; 2207 tp->ftt_pid = provider->ftp_pid; 2208 2209 pp->ftp_tps[i].fit_tp = tp; 2210 pp->ftp_tps[i].fit_id.fti_probe = pp; 2211 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED; 2212 } 2213 2214 /* 2215 * If the arguments are shuffled around we set the argument remapping 2216 * table. Later, when the probe fires, we only remap the arguments 2217 * if the table is non-NULL. 2218 */ 2219 for (i = 0; i < dhpb->dthpb_xargc; i++) { 2220 if (dhpb->dthpb_args[i] != i) { 2221 pp->ftp_argmap = dhpb->dthpb_args; 2222 break; 2223 } 2224 } 2225 2226 /* 2227 * The probe is fully constructed -- register it with DTrace. 2228 */ 2229 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod, 2230 dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp); 2231 2232 mutex_exit(&provider->ftp_cmtx); 2233} 2234 2235/*ARGSUSED*/ 2236static void 2237fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 2238{ 2239 /* 2240 * Clean up the USDT provider. There may be active consumers of the 2241 * provider busy adding probes, no damage will actually befall the 2242 * provider until that count has dropped to zero. This just puts 2243 * the provider on death row. 2244 */ 2245 fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1); 2246} 2247 2248static dtrace_mops_t fasttrap_mops = { 2249 fasttrap_meta_create_probe, 2250 fasttrap_meta_provide, 2251 fasttrap_meta_remove 2252}; 2253 2254/*ARGSUSED*/ 2255static int 2256fasttrap_open(struct cdev *dev __unused, int oflags __unused, 2257 int devtype __unused, struct thread *td __unused) 2258{ 2259 return (0); 2260} 2261 2262/*ARGSUSED*/ 2263static int 2264fasttrap_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int fflag, 2265 struct thread *td) 2266{ 2267#ifdef notyet 2268 struct kinfo_proc kp; 2269 const cred_t *cr = td->td_ucred; 2270#endif 2271 if (!dtrace_attached()) 2272 return (EAGAIN); 2273 2274 if (cmd == FASTTRAPIOC_MAKEPROBE) { 2275 fasttrap_probe_spec_t *uprobe = *(fasttrap_probe_spec_t **)arg; 2276 fasttrap_probe_spec_t *probe; 2277 uint64_t noffs; 2278 size_t size; 2279 int ret, err; 2280 2281 if (copyin(&uprobe->ftps_noffs, &noffs, 2282 sizeof (uprobe->ftps_noffs))) 2283 return (EFAULT); 2284 2285 /* 2286 * Probes must have at least one tracepoint. 2287 */ 2288 if (noffs == 0) 2289 return (EINVAL); 2290 2291 size = sizeof (fasttrap_probe_spec_t) + 2292 sizeof (probe->ftps_offs[0]) * (noffs - 1); 2293 2294 if (size > 1024 * 1024) 2295 return (ENOMEM); 2296 2297 probe = kmem_alloc(size, KM_SLEEP); 2298 2299 if (copyin(uprobe, probe, size) != 0 || 2300 probe->ftps_noffs != noffs) { 2301 kmem_free(probe, size); 2302 return (EFAULT); 2303 } 2304 2305 /* 2306 * Verify that the function and module strings contain no 2307 * funny characters. 2308 */ 2309 if (u8_validate(probe->ftps_func, strlen(probe->ftps_func), 2310 NULL, U8_VALIDATE_ENTIRE, &err) < 0) { 2311 ret = EINVAL; 2312 goto err; 2313 } 2314 2315 if (u8_validate(probe->ftps_mod, strlen(probe->ftps_mod), 2316 NULL, U8_VALIDATE_ENTIRE, &err) < 0) { 2317 ret = EINVAL; 2318 goto err; 2319 } 2320 2321#ifdef notyet 2322 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 2323 proc_t *p; 2324 pid_t pid = probe->ftps_pid; 2325 2326#ifdef illumos 2327 mutex_enter(&pidlock); 2328#endif 2329 /* 2330 * Report an error if the process doesn't exist 2331 * or is actively being birthed. 2332 */ 2333 p = pfind(pid); 2334 if (p) 2335 fill_kinfo_proc(p, &kp); 2336 if (p == NULL || kp.ki_stat == SIDL) { 2337#ifdef illumos 2338 mutex_exit(&pidlock); 2339#endif 2340 return (ESRCH); 2341 } 2342#ifdef illumos 2343 mutex_enter(&p->p_lock); 2344 mutex_exit(&pidlock); 2345#else 2346 PROC_LOCK_ASSERT(p, MA_OWNED); 2347#endif 2348 2349#ifdef notyet 2350 if ((ret = priv_proc_cred_perm(cr, p, NULL, 2351 VREAD | VWRITE)) != 0) { 2352#ifdef illumos 2353 mutex_exit(&p->p_lock); 2354#else 2355 PROC_UNLOCK(p); 2356#endif 2357 return (ret); 2358 } 2359#endif /* notyet */ 2360#ifdef illumos 2361 mutex_exit(&p->p_lock); 2362#else 2363 PROC_UNLOCK(p); 2364#endif 2365 } 2366#endif /* notyet */ 2367 2368 ret = fasttrap_add_probe(probe); 2369err: 2370 kmem_free(probe, size); 2371 2372 return (ret); 2373 2374 } else if (cmd == FASTTRAPIOC_GETINSTR) { 2375 fasttrap_instr_query_t instr; 2376 fasttrap_tracepoint_t *tp; 2377 uint_t index; 2378#ifdef illumos 2379 int ret; 2380#endif 2381 2382#ifdef illumos 2383 if (copyin((void *)arg, &instr, sizeof (instr)) != 0) 2384 return (EFAULT); 2385#endif 2386 2387#ifdef notyet 2388 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 2389 proc_t *p; 2390 pid_t pid = instr.ftiq_pid; 2391 2392#ifdef illumos 2393 mutex_enter(&pidlock); 2394#endif 2395 /* 2396 * Report an error if the process doesn't exist 2397 * or is actively being birthed. 2398 */ 2399 p = pfind(pid); 2400 if (p) 2401 fill_kinfo_proc(p, &kp); 2402 if (p == NULL || kp.ki_stat == SIDL) { 2403#ifdef illumos 2404 mutex_exit(&pidlock); 2405#endif 2406 return (ESRCH); 2407 } 2408#ifdef illumos 2409 mutex_enter(&p->p_lock); 2410 mutex_exit(&pidlock); 2411#else 2412 PROC_LOCK_ASSERT(p, MA_OWNED); 2413#endif 2414 2415#ifdef notyet 2416 if ((ret = priv_proc_cred_perm(cr, p, NULL, 2417 VREAD)) != 0) { 2418#ifdef illumos 2419 mutex_exit(&p->p_lock); 2420#else 2421 PROC_UNLOCK(p); 2422#endif 2423 return (ret); 2424 } 2425#endif /* notyet */ 2426 2427#ifdef illumos 2428 mutex_exit(&p->p_lock); 2429#else 2430 PROC_UNLOCK(p); 2431#endif 2432 } 2433#endif /* notyet */ 2434 2435 index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc); 2436 2437 mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2438 tp = fasttrap_tpoints.fth_table[index].ftb_data; 2439 while (tp != NULL) { 2440 if (instr.ftiq_pid == tp->ftt_pid && 2441 instr.ftiq_pc == tp->ftt_pc && 2442 tp->ftt_proc->ftpc_acount != 0) 2443 break; 2444 2445 tp = tp->ftt_next; 2446 } 2447 2448 if (tp == NULL) { 2449 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2450 return (ENOENT); 2451 } 2452 2453 bcopy(&tp->ftt_instr, &instr.ftiq_instr, 2454 sizeof (instr.ftiq_instr)); 2455 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2456 2457 if (copyout(&instr, (void *)arg, sizeof (instr)) != 0) 2458 return (EFAULT); 2459 2460 return (0); 2461 } 2462 2463 return (EINVAL); 2464} 2465 2466static int 2467fasttrap_load(void) 2468{ 2469 ulong_t nent; 2470 int i, ret; 2471 2472 /* Create the /dev/dtrace/fasttrap entry. */ 2473 fasttrap_cdev = make_dev(&fasttrap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, 2474 "dtrace/fasttrap"); 2475 2476 mtx_init(&fasttrap_cleanup_mtx, "fasttrap clean", "dtrace", MTX_DEF); 2477 mutex_init(&fasttrap_count_mtx, "fasttrap count mtx", MUTEX_DEFAULT, 2478 NULL); 2479 2480#ifdef illumos 2481 fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2482 "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT); 2483#endif 2484 fasttrap_total = 0; 2485 2486 /* 2487 * Conjure up the tracepoints hashtable... 2488 */ 2489#ifdef illumos 2490 nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2491 "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE); 2492#else 2493 nent = tpoints_hash_size; 2494#endif 2495 2496 if (nent == 0 || nent > 0x1000000) 2497 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE; 2498 2499 tpoints_hash_size = nent; 2500 2501 if (ISP2(nent)) 2502 fasttrap_tpoints.fth_nent = nent; 2503 else 2504 fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent); 2505 ASSERT(fasttrap_tpoints.fth_nent > 0); 2506 fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1; 2507 fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent * 2508 sizeof (fasttrap_bucket_t), KM_SLEEP); 2509#ifndef illumos 2510 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2511 mutex_init(&fasttrap_tpoints.fth_table[i].ftb_mtx, 2512 "tracepoints bucket mtx", MUTEX_DEFAULT, NULL); 2513#endif 2514 2515 /* 2516 * ... and the providers hash table... 2517 */ 2518 nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE; 2519 if (ISP2(nent)) 2520 fasttrap_provs.fth_nent = nent; 2521 else 2522 fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent); 2523 ASSERT(fasttrap_provs.fth_nent > 0); 2524 fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1; 2525 fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent * 2526 sizeof (fasttrap_bucket_t), KM_SLEEP); 2527#ifndef illumos 2528 for (i = 0; i < fasttrap_provs.fth_nent; i++) 2529 mutex_init(&fasttrap_provs.fth_table[i].ftb_mtx, 2530 "providers bucket mtx", MUTEX_DEFAULT, NULL); 2531#endif 2532 2533 ret = kproc_create(fasttrap_pid_cleanup_cb, NULL, 2534 &fasttrap_cleanup_proc, 0, 0, "ftcleanup"); 2535 if (ret != 0) { 2536 destroy_dev(fasttrap_cdev); 2537#ifndef illumos 2538 for (i = 0; i < fasttrap_provs.fth_nent; i++) 2539 mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx); 2540 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2541 mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx); 2542#endif 2543 kmem_free(fasttrap_provs.fth_table, fasttrap_provs.fth_nent * 2544 sizeof (fasttrap_bucket_t)); 2545 mtx_destroy(&fasttrap_cleanup_mtx); 2546 mutex_destroy(&fasttrap_count_mtx); 2547 return (ret); 2548 } 2549 2550 2551 /* 2552 * ... and the procs hash table. 2553 */ 2554 nent = FASTTRAP_PROCS_DEFAULT_SIZE; 2555 if (ISP2(nent)) 2556 fasttrap_procs.fth_nent = nent; 2557 else 2558 fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent); 2559 ASSERT(fasttrap_procs.fth_nent > 0); 2560 fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1; 2561 fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent * 2562 sizeof (fasttrap_bucket_t), KM_SLEEP); 2563#ifndef illumos 2564 for (i = 0; i < fasttrap_procs.fth_nent; i++) 2565 mutex_init(&fasttrap_procs.fth_table[i].ftb_mtx, 2566 "processes bucket mtx", MUTEX_DEFAULT, NULL); 2567 2568 CPU_FOREACH(i) { 2569 mutex_init(&fasttrap_cpuc_pid_lock[i], "fasttrap barrier", 2570 MUTEX_DEFAULT, NULL); 2571 } 2572 2573 /* 2574 * This event handler must run before kdtrace_thread_dtor() since it 2575 * accesses the thread's struct kdtrace_thread. 2576 */ 2577 fasttrap_thread_dtor_tag = EVENTHANDLER_REGISTER(thread_dtor, 2578 fasttrap_thread_dtor, NULL, EVENTHANDLER_PRI_FIRST); 2579#endif 2580 2581 /* 2582 * Install our hooks into fork(2), exec(2), and exit(2). 2583 */ 2584 dtrace_fasttrap_fork = &fasttrap_fork; 2585 dtrace_fasttrap_exit = &fasttrap_exec_exit; 2586 dtrace_fasttrap_exec = &fasttrap_exec_exit; 2587 2588 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2589 &fasttrap_meta_id); 2590 2591 return (0); 2592} 2593 2594static int 2595fasttrap_unload(void) 2596{ 2597 int i, fail = 0; 2598 2599 /* 2600 * Unregister the meta-provider to make sure no new fasttrap- 2601 * managed providers come along while we're trying to close up 2602 * shop. If we fail to detach, we'll need to re-register as a 2603 * meta-provider. We can fail to unregister as a meta-provider 2604 * if providers we manage still exist. 2605 */ 2606 if (fasttrap_meta_id != DTRACE_METAPROVNONE && 2607 dtrace_meta_unregister(fasttrap_meta_id) != 0) 2608 return (-1); 2609 2610 /* 2611 * Iterate over all of our providers. If there's still a process 2612 * that corresponds to that pid, fail to detach. 2613 */ 2614 for (i = 0; i < fasttrap_provs.fth_nent; i++) { 2615 fasttrap_provider_t **fpp, *fp; 2616 fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i]; 2617 2618 mutex_enter(&bucket->ftb_mtx); 2619 fpp = (fasttrap_provider_t **)&bucket->ftb_data; 2620 while ((fp = *fpp) != NULL) { 2621 /* 2622 * Acquire and release the lock as a simple way of 2623 * waiting for any other consumer to finish with 2624 * this provider. A thread must first acquire the 2625 * bucket lock so there's no chance of another thread 2626 * blocking on the provider's lock. 2627 */ 2628 mutex_enter(&fp->ftp_mtx); 2629 mutex_exit(&fp->ftp_mtx); 2630 2631 if (dtrace_unregister(fp->ftp_provid) != 0) { 2632 fail = 1; 2633 fpp = &fp->ftp_next; 2634 } else { 2635 *fpp = fp->ftp_next; 2636 fasttrap_provider_free(fp); 2637 } 2638 } 2639 2640 mutex_exit(&bucket->ftb_mtx); 2641 } 2642 2643 if (fail) { 2644 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2645 &fasttrap_meta_id); 2646 2647 return (-1); 2648 } 2649 2650 /* 2651 * Stop new processes from entering these hooks now, before the 2652 * fasttrap_cleanup thread runs. That way all processes will hopefully 2653 * be out of these hooks before we free fasttrap_provs.fth_table 2654 */ 2655 ASSERT(dtrace_fasttrap_fork == &fasttrap_fork); 2656 dtrace_fasttrap_fork = NULL; 2657 2658 ASSERT(dtrace_fasttrap_exec == &fasttrap_exec_exit); 2659 dtrace_fasttrap_exec = NULL; 2660 2661 ASSERT(dtrace_fasttrap_exit == &fasttrap_exec_exit); 2662 dtrace_fasttrap_exit = NULL; 2663 2664 mtx_lock(&fasttrap_cleanup_mtx); 2665 fasttrap_cleanup_drain = 1; 2666 /* Wait for the cleanup thread to finish up and signal us. */ 2667 wakeup(&fasttrap_cleanup_cv); 2668 mtx_sleep(&fasttrap_cleanup_drain, &fasttrap_cleanup_mtx, 0, "ftcld", 2669 0); 2670 fasttrap_cleanup_proc = NULL; 2671 mtx_destroy(&fasttrap_cleanup_mtx); 2672 2673#ifdef DEBUG 2674 mutex_enter(&fasttrap_count_mtx); 2675 ASSERT(fasttrap_pid_count == 0); 2676 mutex_exit(&fasttrap_count_mtx); 2677#endif 2678 2679#ifndef illumos 2680 EVENTHANDLER_DEREGISTER(thread_dtor, fasttrap_thread_dtor_tag); 2681 2682 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2683 mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx); 2684 for (i = 0; i < fasttrap_provs.fth_nent; i++) 2685 mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx); 2686 for (i = 0; i < fasttrap_procs.fth_nent; i++) 2687 mutex_destroy(&fasttrap_procs.fth_table[i].ftb_mtx); 2688#endif 2689 kmem_free(fasttrap_tpoints.fth_table, 2690 fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t)); 2691 fasttrap_tpoints.fth_nent = 0; 2692 2693 kmem_free(fasttrap_provs.fth_table, 2694 fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t)); 2695 fasttrap_provs.fth_nent = 0; 2696 2697 kmem_free(fasttrap_procs.fth_table, 2698 fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t)); 2699 fasttrap_procs.fth_nent = 0; 2700 2701#ifndef illumos 2702 destroy_dev(fasttrap_cdev); 2703 mutex_destroy(&fasttrap_count_mtx); 2704 CPU_FOREACH(i) { 2705 mutex_destroy(&fasttrap_cpuc_pid_lock[i]); 2706 } 2707#endif 2708 2709 return (0); 2710} 2711 2712/* ARGSUSED */ 2713static int 2714fasttrap_modevent(module_t mod __unused, int type, void *data __unused) 2715{ 2716 int error = 0; 2717 2718 switch (type) { 2719 case MOD_LOAD: 2720 break; 2721 2722 case MOD_UNLOAD: 2723 break; 2724 2725 case MOD_SHUTDOWN: 2726 break; 2727 2728 default: 2729 error = EOPNOTSUPP; 2730 break; 2731 } 2732 return (error); 2733} 2734 2735SYSINIT(fasttrap_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fasttrap_load, 2736 NULL); 2737SYSUNINIT(fasttrap_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, 2738 fasttrap_unload, NULL); 2739 2740DEV_MODULE(fasttrap, fasttrap_modevent, NULL); 2741MODULE_VERSION(fasttrap, 1); 2742MODULE_DEPEND(fasttrap, dtrace, 1, 1, 1); 2743MODULE_DEPEND(fasttrap, opensolaris, 1, 1, 1); 2744