dt_module.c revision 297129
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25/*
26 * Copyright (c) 2013, Joyent, Inc.  All rights reserved.
27 * Copyright (c) 2016, Pedro Giffuni.  All rights reserved.
28 */
29
30#include <sys/types.h>
31#ifdef illumos
32#include <sys/modctl.h>
33#include <sys/kobj.h>
34#include <sys/kobj_impl.h>
35#include <sys/sysmacros.h>
36#include <sys/elf.h>
37#include <sys/task.h>
38#else
39#include <sys/param.h>
40#include <sys/linker.h>
41#include <sys/stat.h>
42#endif
43
44#include <unistd.h>
45#ifdef illumos
46#include <project.h>
47#endif
48#include <strings.h>
49#include <stdlib.h>
50#include <libelf.h>
51#include <limits.h>
52#include <assert.h>
53#include <errno.h>
54#include <dirent.h>
55#ifndef illumos
56#include <fcntl.h>
57#include <libproc_compat.h>
58#endif
59
60#include <dt_strtab.h>
61#include <dt_module.h>
62#include <dt_impl.h>
63
64static const char *dt_module_strtab; /* active strtab for qsort callbacks */
65
66static void
67dt_module_symhash_insert(dt_module_t *dmp, const char *name, uint_t id)
68{
69	dt_sym_t *dsp = &dmp->dm_symchains[dmp->dm_symfree];
70	uint_t h;
71
72	assert(dmp->dm_symfree < dmp->dm_nsymelems + 1);
73
74	dsp->ds_symid = id;
75	h = dt_strtab_hash(name, NULL) % dmp->dm_nsymbuckets;
76	dsp->ds_next = dmp->dm_symbuckets[h];
77	dmp->dm_symbuckets[h] = dmp->dm_symfree++;
78}
79
80static uint_t
81dt_module_syminit32(dt_module_t *dmp)
82{
83#if STT_NUM != (STT_TLS + 1)
84#error "STT_NUM has grown. update dt_module_syminit32()"
85#endif
86
87	Elf32_Sym *sym = dmp->dm_symtab.cts_data;
88	const char *base = dmp->dm_strtab.cts_data;
89	size_t ss_size = dmp->dm_strtab.cts_size;
90	uint_t i, n = dmp->dm_nsymelems;
91	uint_t asrsv = 0;
92
93#if defined(__FreeBSD__)
94	GElf_Ehdr ehdr;
95	int is_elf_obj;
96
97	gelf_getehdr(dmp->dm_elf, &ehdr);
98	is_elf_obj = (ehdr.e_type == ET_REL);
99#endif
100
101	for (i = 0; i < n; i++, sym++) {
102		const char *name = base + sym->st_name;
103		uchar_t type = ELF32_ST_TYPE(sym->st_info);
104
105		if (type >= STT_NUM || type == STT_SECTION)
106			continue; /* skip sections and unknown types */
107
108		if (sym->st_name == 0 || sym->st_name >= ss_size)
109			continue; /* skip null or invalid names */
110
111		if (sym->st_value != 0 &&
112		    (ELF32_ST_BIND(sym->st_info) != STB_LOCAL || sym->st_size)) {
113			asrsv++; /* reserve space in the address map */
114
115#if defined(__FreeBSD__)
116			sym->st_value += (Elf_Addr) dmp->dm_reloc_offset;
117			if (is_elf_obj && sym->st_shndx != SHN_UNDEF &&
118			    sym->st_shndx < ehdr.e_shnum)
119				sym->st_value +=
120				    dmp->dm_sec_offsets[sym->st_shndx];
121#endif
122		}
123
124		dt_module_symhash_insert(dmp, name, i);
125	}
126
127	return (asrsv);
128}
129
130static uint_t
131dt_module_syminit64(dt_module_t *dmp)
132{
133#if STT_NUM != (STT_TLS + 1)
134#error "STT_NUM has grown. update dt_module_syminit64()"
135#endif
136
137	Elf64_Sym *sym = dmp->dm_symtab.cts_data;
138	const char *base = dmp->dm_strtab.cts_data;
139	size_t ss_size = dmp->dm_strtab.cts_size;
140	uint_t i, n = dmp->dm_nsymelems;
141	uint_t asrsv = 0;
142
143#if defined(__FreeBSD__)
144	GElf_Ehdr ehdr;
145	int is_elf_obj;
146
147	gelf_getehdr(dmp->dm_elf, &ehdr);
148	is_elf_obj = (ehdr.e_type == ET_REL);
149#endif
150
151	for (i = 0; i < n; i++, sym++) {
152		const char *name = base + sym->st_name;
153		uchar_t type = ELF64_ST_TYPE(sym->st_info);
154
155		if (type >= STT_NUM || type == STT_SECTION)
156			continue; /* skip sections and unknown types */
157
158		if (sym->st_name == 0 || sym->st_name >= ss_size)
159			continue; /* skip null or invalid names */
160
161		if (sym->st_value != 0 &&
162		    (ELF64_ST_BIND(sym->st_info) != STB_LOCAL || sym->st_size)) {
163			asrsv++; /* reserve space in the address map */
164#if defined(__FreeBSD__)
165			sym->st_value += (Elf_Addr) dmp->dm_reloc_offset;
166			if (is_elf_obj && sym->st_shndx != SHN_UNDEF &&
167			    sym->st_shndx < ehdr.e_shnum)
168				sym->st_value +=
169				    dmp->dm_sec_offsets[sym->st_shndx];
170#endif
171		}
172
173		dt_module_symhash_insert(dmp, name, i);
174	}
175
176	return (asrsv);
177}
178
179/*
180 * Sort comparison function for 32-bit symbol address-to-name lookups.  We sort
181 * symbols by value.  If values are equal, we prefer the symbol that is
182 * non-zero sized, typed, not weak, or lexically first, in that order.
183 */
184static int
185dt_module_symcomp32(const void *lp, const void *rp)
186{
187	Elf32_Sym *lhs = *((Elf32_Sym **)lp);
188	Elf32_Sym *rhs = *((Elf32_Sym **)rp);
189
190	if (lhs->st_value != rhs->st_value)
191		return (lhs->st_value > rhs->st_value ? 1 : -1);
192
193	if ((lhs->st_size == 0) != (rhs->st_size == 0))
194		return (lhs->st_size == 0 ? 1 : -1);
195
196	if ((ELF32_ST_TYPE(lhs->st_info) == STT_NOTYPE) !=
197	    (ELF32_ST_TYPE(rhs->st_info) == STT_NOTYPE))
198		return (ELF32_ST_TYPE(lhs->st_info) == STT_NOTYPE ? 1 : -1);
199
200	if ((ELF32_ST_BIND(lhs->st_info) == STB_WEAK) !=
201	    (ELF32_ST_BIND(rhs->st_info) == STB_WEAK))
202		return (ELF32_ST_BIND(lhs->st_info) == STB_WEAK ? 1 : -1);
203
204	return (strcmp(dt_module_strtab + lhs->st_name,
205	    dt_module_strtab + rhs->st_name));
206}
207
208/*
209 * Sort comparison function for 64-bit symbol address-to-name lookups.  We sort
210 * symbols by value.  If values are equal, we prefer the symbol that is
211 * non-zero sized, typed, not weak, or lexically first, in that order.
212 */
213static int
214dt_module_symcomp64(const void *lp, const void *rp)
215{
216	Elf64_Sym *lhs = *((Elf64_Sym **)lp);
217	Elf64_Sym *rhs = *((Elf64_Sym **)rp);
218
219	if (lhs->st_value != rhs->st_value)
220		return (lhs->st_value > rhs->st_value ? 1 : -1);
221
222	if ((lhs->st_size == 0) != (rhs->st_size == 0))
223		return (lhs->st_size == 0 ? 1 : -1);
224
225	if ((ELF64_ST_TYPE(lhs->st_info) == STT_NOTYPE) !=
226	    (ELF64_ST_TYPE(rhs->st_info) == STT_NOTYPE))
227		return (ELF64_ST_TYPE(lhs->st_info) == STT_NOTYPE ? 1 : -1);
228
229	if ((ELF64_ST_BIND(lhs->st_info) == STB_WEAK) !=
230	    (ELF64_ST_BIND(rhs->st_info) == STB_WEAK))
231		return (ELF64_ST_BIND(lhs->st_info) == STB_WEAK ? 1 : -1);
232
233	return (strcmp(dt_module_strtab + lhs->st_name,
234	    dt_module_strtab + rhs->st_name));
235}
236
237static void
238dt_module_symsort32(dt_module_t *dmp)
239{
240	Elf32_Sym *symtab = (Elf32_Sym *)dmp->dm_symtab.cts_data;
241	Elf32_Sym **sympp = (Elf32_Sym **)dmp->dm_asmap;
242	const dt_sym_t *dsp = dmp->dm_symchains + 1;
243	uint_t i, n = dmp->dm_symfree;
244
245	for (i = 1; i < n; i++, dsp++) {
246		Elf32_Sym *sym = symtab + dsp->ds_symid;
247		if (sym->st_value != 0 &&
248		    (ELF32_ST_BIND(sym->st_info) != STB_LOCAL || sym->st_size))
249			*sympp++ = sym;
250	}
251
252	dmp->dm_aslen = (uint_t)(sympp - (Elf32_Sym **)dmp->dm_asmap);
253	assert(dmp->dm_aslen <= dmp->dm_asrsv);
254
255	dt_module_strtab = dmp->dm_strtab.cts_data;
256	qsort(dmp->dm_asmap, dmp->dm_aslen,
257	    sizeof (Elf32_Sym *), dt_module_symcomp32);
258	dt_module_strtab = NULL;
259}
260
261static void
262dt_module_symsort64(dt_module_t *dmp)
263{
264	Elf64_Sym *symtab = (Elf64_Sym *)dmp->dm_symtab.cts_data;
265	Elf64_Sym **sympp = (Elf64_Sym **)dmp->dm_asmap;
266	const dt_sym_t *dsp = dmp->dm_symchains + 1;
267	uint_t i, n = dmp->dm_symfree;
268
269	for (i = 1; i < n; i++, dsp++) {
270		Elf64_Sym *sym = symtab + dsp->ds_symid;
271		if (sym->st_value != 0 &&
272		    (ELF64_ST_BIND(sym->st_info) != STB_LOCAL || sym->st_size))
273			*sympp++ = sym;
274	}
275
276	dmp->dm_aslen = (uint_t)(sympp - (Elf64_Sym **)dmp->dm_asmap);
277	assert(dmp->dm_aslen <= dmp->dm_asrsv);
278
279	dt_module_strtab = dmp->dm_strtab.cts_data;
280	qsort(dmp->dm_asmap, dmp->dm_aslen,
281	    sizeof (Elf64_Sym *), dt_module_symcomp64);
282	dt_module_strtab = NULL;
283}
284
285static GElf_Sym *
286dt_module_symgelf32(const Elf32_Sym *src, GElf_Sym *dst)
287{
288	if (dst != NULL) {
289		dst->st_name = src->st_name;
290		dst->st_info = src->st_info;
291		dst->st_other = src->st_other;
292		dst->st_shndx = src->st_shndx;
293		dst->st_value = src->st_value;
294		dst->st_size = src->st_size;
295	}
296
297	return (dst);
298}
299
300static GElf_Sym *
301dt_module_symgelf64(const Elf64_Sym *src, GElf_Sym *dst)
302{
303	if (dst != NULL)
304		bcopy(src, dst, sizeof (GElf_Sym));
305
306	return (dst);
307}
308
309static GElf_Sym *
310dt_module_symname32(dt_module_t *dmp, const char *name,
311    GElf_Sym *symp, uint_t *idp)
312{
313	const Elf32_Sym *symtab = dmp->dm_symtab.cts_data;
314	const char *strtab = dmp->dm_strtab.cts_data;
315
316	const Elf32_Sym *sym;
317	const dt_sym_t *dsp;
318	uint_t i, h;
319
320	if (dmp->dm_nsymelems == 0)
321		return (NULL);
322
323	h = dt_strtab_hash(name, NULL) % dmp->dm_nsymbuckets;
324
325	for (i = dmp->dm_symbuckets[h]; i != 0; i = dsp->ds_next) {
326		dsp = &dmp->dm_symchains[i];
327		sym = symtab + dsp->ds_symid;
328
329		if (strcmp(name, strtab + sym->st_name) == 0) {
330			if (idp != NULL)
331				*idp = dsp->ds_symid;
332			return (dt_module_symgelf32(sym, symp));
333		}
334	}
335
336	return (NULL);
337}
338
339static GElf_Sym *
340dt_module_symname64(dt_module_t *dmp, const char *name,
341    GElf_Sym *symp, uint_t *idp)
342{
343	const Elf64_Sym *symtab = dmp->dm_symtab.cts_data;
344	const char *strtab = dmp->dm_strtab.cts_data;
345
346	const Elf64_Sym *sym;
347	const dt_sym_t *dsp;
348	uint_t i, h;
349
350	if (dmp->dm_nsymelems == 0)
351		return (NULL);
352
353	h = dt_strtab_hash(name, NULL) % dmp->dm_nsymbuckets;
354
355	for (i = dmp->dm_symbuckets[h]; i != 0; i = dsp->ds_next) {
356		dsp = &dmp->dm_symchains[i];
357		sym = symtab + dsp->ds_symid;
358
359		if (strcmp(name, strtab + sym->st_name) == 0) {
360			if (idp != NULL)
361				*idp = dsp->ds_symid;
362			return (dt_module_symgelf64(sym, symp));
363		}
364	}
365
366	return (NULL);
367}
368
369static GElf_Sym *
370dt_module_symaddr32(dt_module_t *dmp, GElf_Addr addr,
371    GElf_Sym *symp, uint_t *idp)
372{
373	const Elf32_Sym **asmap = (const Elf32_Sym **)dmp->dm_asmap;
374	const Elf32_Sym *symtab = dmp->dm_symtab.cts_data;
375	const Elf32_Sym *sym;
376
377	uint_t i, mid, lo = 0, hi = dmp->dm_aslen - 1;
378	Elf32_Addr v;
379
380	if (dmp->dm_aslen == 0)
381		return (NULL);
382
383	while (hi - lo > 1) {
384		mid = (lo + hi) / 2;
385		if (addr >= asmap[mid]->st_value)
386			lo = mid;
387		else
388			hi = mid;
389	}
390
391	i = addr < asmap[hi]->st_value ? lo : hi;
392	sym = asmap[i];
393	v = sym->st_value;
394
395	/*
396	 * If the previous entry has the same value, improve our choice.  The
397	 * order of equal-valued symbols is determined by the comparison func.
398	 */
399	while (i-- != 0 && asmap[i]->st_value == v)
400		sym = asmap[i];
401
402	if (addr - sym->st_value < MAX(sym->st_size, 1)) {
403		if (idp != NULL)
404			*idp = (uint_t)(sym - symtab);
405		return (dt_module_symgelf32(sym, symp));
406	}
407
408	return (NULL);
409}
410
411static GElf_Sym *
412dt_module_symaddr64(dt_module_t *dmp, GElf_Addr addr,
413    GElf_Sym *symp, uint_t *idp)
414{
415	const Elf64_Sym **asmap = (const Elf64_Sym **)dmp->dm_asmap;
416	const Elf64_Sym *symtab = dmp->dm_symtab.cts_data;
417	const Elf64_Sym *sym;
418
419	uint_t i, mid, lo = 0, hi = dmp->dm_aslen - 1;
420	Elf64_Addr v;
421
422	if (dmp->dm_aslen == 0)
423		return (NULL);
424
425	while (hi - lo > 1) {
426		mid = (lo + hi) / 2;
427		if (addr >= asmap[mid]->st_value)
428			lo = mid;
429		else
430			hi = mid;
431	}
432
433	i = addr < asmap[hi]->st_value ? lo : hi;
434	sym = asmap[i];
435	v = sym->st_value;
436
437	/*
438	 * If the previous entry has the same value, improve our choice.  The
439	 * order of equal-valued symbols is determined by the comparison func.
440	 */
441	while (i-- != 0 && asmap[i]->st_value == v)
442		sym = asmap[i];
443
444	if (addr - sym->st_value < MAX(sym->st_size, 1)) {
445		if (idp != NULL)
446			*idp = (uint_t)(sym - symtab);
447		return (dt_module_symgelf64(sym, symp));
448	}
449
450	return (NULL);
451}
452
453static const dt_modops_t dt_modops_32 = {
454	dt_module_syminit32,
455	dt_module_symsort32,
456	dt_module_symname32,
457	dt_module_symaddr32
458};
459
460static const dt_modops_t dt_modops_64 = {
461	dt_module_syminit64,
462	dt_module_symsort64,
463	dt_module_symname64,
464	dt_module_symaddr64
465};
466
467dt_module_t *
468dt_module_create(dtrace_hdl_t *dtp, const char *name)
469{
470	long pid;
471	char *eptr;
472	dt_ident_t *idp;
473	uint_t h = dt_strtab_hash(name, NULL) % dtp->dt_modbuckets;
474	dt_module_t *dmp;
475
476	for (dmp = dtp->dt_mods[h]; dmp != NULL; dmp = dmp->dm_next) {
477		if (strcmp(dmp->dm_name, name) == 0)
478			return (dmp);
479	}
480
481	if ((dmp = malloc(sizeof (dt_module_t))) == NULL)
482		return (NULL); /* caller must handle allocation failure */
483
484	bzero(dmp, sizeof (dt_module_t));
485	(void) strlcpy(dmp->dm_name, name, sizeof (dmp->dm_name));
486	dt_list_append(&dtp->dt_modlist, dmp);
487	dmp->dm_next = dtp->dt_mods[h];
488	dtp->dt_mods[h] = dmp;
489	dtp->dt_nmods++;
490
491	if (dtp->dt_conf.dtc_ctfmodel == CTF_MODEL_LP64)
492		dmp->dm_ops = &dt_modops_64;
493	else
494		dmp->dm_ops = &dt_modops_32;
495
496	/*
497	 * Modules for userland processes are special. They always refer to a
498	 * specific process and have a copy of their CTF data from a specific
499	 * instant in time. Any dt_module_t that begins with 'pid' is a module
500	 * for a specific process, much like how any probe description that
501	 * begins with 'pid' is special. pid123 refers to process 123. A module
502	 * that is just 'pid' refers specifically to pid$target. This is
503	 * generally done as D does not currently allow for macros to be
504	 * evaluated when working with types.
505	 */
506	if (strncmp(dmp->dm_name, "pid", 3) == 0) {
507		errno = 0;
508		if (dmp->dm_name[3] == '\0') {
509			idp = dt_idhash_lookup(dtp->dt_macros, "target");
510			if (idp != NULL && idp->di_id != 0)
511				dmp->dm_pid = idp->di_id;
512		} else {
513			pid = strtol(dmp->dm_name + 3, &eptr, 10);
514			if (errno == 0 && *eptr == '\0')
515				dmp->dm_pid = (pid_t)pid;
516			else
517				dt_dprintf("encountered malformed pid "
518				    "module: %s\n", dmp->dm_name);
519		}
520	}
521
522	return (dmp);
523}
524
525dt_module_t *
526dt_module_lookup_by_name(dtrace_hdl_t *dtp, const char *name)
527{
528	uint_t h = dt_strtab_hash(name, NULL) % dtp->dt_modbuckets;
529	dt_module_t *dmp;
530
531	for (dmp = dtp->dt_mods[h]; dmp != NULL; dmp = dmp->dm_next) {
532		if (strcmp(dmp->dm_name, name) == 0)
533			return (dmp);
534	}
535
536	return (NULL);
537}
538
539/*ARGSUSED*/
540dt_module_t *
541dt_module_lookup_by_ctf(dtrace_hdl_t *dtp, ctf_file_t *ctfp)
542{
543	return (ctfp ? ctf_getspecific(ctfp) : NULL);
544}
545
546static int
547dt_module_load_sect(dtrace_hdl_t *dtp, dt_module_t *dmp, ctf_sect_t *ctsp)
548{
549	const char *s;
550	size_t shstrs;
551	GElf_Shdr sh;
552	Elf_Data *dp;
553	Elf_Scn *sp;
554
555	if (elf_getshdrstrndx(dmp->dm_elf, &shstrs) == -1)
556		return (dt_set_errno(dtp, EDT_NOTLOADED));
557
558	for (sp = NULL; (sp = elf_nextscn(dmp->dm_elf, sp)) != NULL; ) {
559		if (gelf_getshdr(sp, &sh) == NULL || sh.sh_type == SHT_NULL ||
560		    (s = elf_strptr(dmp->dm_elf, shstrs, sh.sh_name)) == NULL)
561			continue; /* skip any malformed sections */
562
563		if (sh.sh_type == ctsp->cts_type &&
564		    sh.sh_entsize == ctsp->cts_entsize &&
565		    strcmp(s, ctsp->cts_name) == 0)
566			break; /* section matches specification */
567	}
568
569	/*
570	 * If the section isn't found, return success but leave cts_data set
571	 * to NULL and cts_size set to zero for our caller.
572	 */
573	if (sp == NULL || (dp = elf_getdata(sp, NULL)) == NULL)
574		return (0);
575
576#ifdef illumos
577	ctsp->cts_data = dp->d_buf;
578#else
579	if ((ctsp->cts_data = malloc(dp->d_size)) == NULL)
580		return (0);
581	memcpy(ctsp->cts_data, dp->d_buf, dp->d_size);
582#endif
583	ctsp->cts_size = dp->d_size;
584
585	dt_dprintf("loaded %s [%s] (%lu bytes)\n",
586	    dmp->dm_name, ctsp->cts_name, (ulong_t)ctsp->cts_size);
587
588	return (0);
589}
590
591typedef struct dt_module_cb_arg {
592	struct ps_prochandle *dpa_proc;
593	dtrace_hdl_t *dpa_dtp;
594	dt_module_t *dpa_dmp;
595	uint_t dpa_count;
596} dt_module_cb_arg_t;
597
598/* ARGSUSED */
599static int
600dt_module_load_proc_count(void *arg, const prmap_t *prmap, const char *obj)
601{
602	ctf_file_t *fp;
603	dt_module_cb_arg_t *dcp = arg;
604
605	/* Try to grab a ctf container if it exists */
606	fp = Pname_to_ctf(dcp->dpa_proc, obj);
607	if (fp != NULL)
608		dcp->dpa_count++;
609	return (0);
610}
611
612/* ARGSUSED */
613static int
614dt_module_load_proc_build(void *arg, const prmap_t *prmap, const char *obj)
615{
616	ctf_file_t *fp;
617	char buf[MAXPATHLEN], *p;
618	dt_module_cb_arg_t *dcp = arg;
619	int count = dcp->dpa_count;
620	Lmid_t lmid;
621
622	fp = Pname_to_ctf(dcp->dpa_proc, obj);
623	if (fp == NULL)
624		return (0);
625	fp = ctf_dup(fp);
626	if (fp == NULL)
627		return (0);
628	dcp->dpa_dmp->dm_libctfp[count] = fp;
629	/*
630	 * While it'd be nice to simply use objname here, because of our prior
631	 * actions we'll always get a resolved object name to its on disk file.
632	 * Like the pid provider, we need to tell a bit of a lie here. The type
633	 * that the user thinks of is in terms of the libraries they requested,
634	 * eg. libc.so.1, they don't care about the fact that it's
635	 * libc_hwcap.so.1.
636	 */
637	(void) Pobjname(dcp->dpa_proc, prmap->pr_vaddr, buf, sizeof (buf));
638	if ((p = strrchr(buf, '/')) == NULL)
639		p = buf;
640	else
641		p++;
642
643	/*
644	 * If for some reason we can't find a link map id for this module, which
645	 * would be really quite weird. We instead just say the link map id is
646	 * zero.
647	 */
648	if (Plmid(dcp->dpa_proc, prmap->pr_vaddr, &lmid) != 0)
649		lmid = 0;
650
651	if (lmid == 0)
652		dcp->dpa_dmp->dm_libctfn[count] = strdup(p);
653	else
654		(void) asprintf(&dcp->dpa_dmp->dm_libctfn[count],
655		    "LM%x`%s", lmid, p);
656	if (dcp->dpa_dmp->dm_libctfn[count] == NULL)
657		return (1);
658	ctf_setspecific(fp, dcp->dpa_dmp);
659	dcp->dpa_count++;
660	return (0);
661}
662
663/*
664 * We've been asked to load data that belongs to another process. As such we're
665 * going to pgrab it at this instant, load everything that we might ever care
666 * about, and then drive on. The reason for this is that the process that we're
667 * interested in might be changing. As long as we have grabbed it, then this
668 * can't be a problem for us.
669 *
670 * For now, we're actually going to punt on most things and just try to get CTF
671 * data, nothing else. Basically this is only useful as a source of type
672 * information, we can't go and do the stacktrace lookups, etc.
673 */
674static int
675dt_module_load_proc(dtrace_hdl_t *dtp, dt_module_t *dmp)
676{
677	struct ps_prochandle *p;
678	dt_module_cb_arg_t arg;
679
680	/*
681	 * Note that on success we do not release this hold. We must hold this
682	 * for our life time.
683	 */
684	p = dt_proc_grab(dtp, dmp->dm_pid, 0, PGRAB_RDONLY | PGRAB_FORCE);
685	if (p == NULL) {
686		dt_dprintf("failed to grab pid: %d\n", (int)dmp->dm_pid);
687		return (dt_set_errno(dtp, EDT_CANTLOAD));
688	}
689	dt_proc_lock(dtp, p);
690
691	arg.dpa_proc = p;
692	arg.dpa_dtp = dtp;
693	arg.dpa_dmp = dmp;
694	arg.dpa_count = 0;
695	if (Pobject_iter_resolved(p, dt_module_load_proc_count, &arg) != 0) {
696		dt_dprintf("failed to iterate objects\n");
697		dt_proc_release(dtp, p);
698		return (dt_set_errno(dtp, EDT_CANTLOAD));
699	}
700
701	if (arg.dpa_count == 0) {
702		dt_dprintf("no ctf data present\n");
703		dt_proc_unlock(dtp, p);
704		dt_proc_release(dtp, p);
705		return (dt_set_errno(dtp, EDT_CANTLOAD));
706	}
707
708	dmp->dm_libctfp = calloc(arg.dpa_count, sizeof (ctf_file_t *));
709	if (dmp->dm_libctfp == NULL) {
710		dt_proc_unlock(dtp, p);
711		dt_proc_release(dtp, p);
712		return (dt_set_errno(dtp, EDT_NOMEM));
713	}
714
715	dmp->dm_libctfn = calloc(arg.dpa_count, sizeof (char *));
716	if (dmp->dm_libctfn == NULL) {
717		free(dmp->dm_libctfp);
718		dt_proc_unlock(dtp, p);
719		dt_proc_release(dtp, p);
720		return (dt_set_errno(dtp, EDT_NOMEM));
721	}
722
723	dmp->dm_nctflibs = arg.dpa_count;
724
725	arg.dpa_count = 0;
726	if (Pobject_iter_resolved(p, dt_module_load_proc_build, &arg) != 0) {
727		dt_proc_unlock(dtp, p);
728		dt_module_unload(dtp, dmp);
729		dt_proc_release(dtp, p);
730		return (dt_set_errno(dtp, EDT_CANTLOAD));
731	}
732	assert(arg.dpa_count == dmp->dm_nctflibs);
733	dt_dprintf("loaded %d ctf modules for pid %d\n", arg.dpa_count,
734	    (int)dmp->dm_pid);
735
736	dt_proc_unlock(dtp, p);
737	dt_proc_release(dtp, p);
738	dmp->dm_flags |= DT_DM_LOADED;
739
740	return (0);
741}
742
743int
744dt_module_load(dtrace_hdl_t *dtp, dt_module_t *dmp)
745{
746	if (dmp->dm_flags & DT_DM_LOADED)
747		return (0); /* module is already loaded */
748
749	if (dmp->dm_pid != 0)
750		return (dt_module_load_proc(dtp, dmp));
751
752	dmp->dm_ctdata.cts_name = ".SUNW_ctf";
753	dmp->dm_ctdata.cts_type = SHT_PROGBITS;
754	dmp->dm_ctdata.cts_flags = 0;
755	dmp->dm_ctdata.cts_data = NULL;
756	dmp->dm_ctdata.cts_size = 0;
757	dmp->dm_ctdata.cts_entsize = 0;
758	dmp->dm_ctdata.cts_offset = 0;
759
760	dmp->dm_symtab.cts_name = ".symtab";
761	dmp->dm_symtab.cts_type = SHT_SYMTAB;
762	dmp->dm_symtab.cts_flags = 0;
763	dmp->dm_symtab.cts_data = NULL;
764	dmp->dm_symtab.cts_size = 0;
765	dmp->dm_symtab.cts_entsize = dmp->dm_ops == &dt_modops_64 ?
766	    sizeof (Elf64_Sym) : sizeof (Elf32_Sym);
767	dmp->dm_symtab.cts_offset = 0;
768
769	dmp->dm_strtab.cts_name = ".strtab";
770	dmp->dm_strtab.cts_type = SHT_STRTAB;
771	dmp->dm_strtab.cts_flags = 0;
772	dmp->dm_strtab.cts_data = NULL;
773	dmp->dm_strtab.cts_size = 0;
774	dmp->dm_strtab.cts_entsize = 0;
775	dmp->dm_strtab.cts_offset = 0;
776
777	/*
778	 * Attempt to load the module's CTF section, symbol table section, and
779	 * string table section.  Note that modules may not contain CTF data:
780	 * this will result in a successful load_sect but data of size zero.
781	 * We will then fail if dt_module_getctf() is called, as shown below.
782	 */
783	if (dt_module_load_sect(dtp, dmp, &dmp->dm_ctdata) == -1 ||
784	    dt_module_load_sect(dtp, dmp, &dmp->dm_symtab) == -1 ||
785	    dt_module_load_sect(dtp, dmp, &dmp->dm_strtab) == -1) {
786		dt_module_unload(dtp, dmp);
787		return (-1); /* dt_errno is set for us */
788	}
789
790	/*
791	 * Allocate the hash chains and hash buckets for symbol name lookup.
792	 * This is relatively simple since the symbol table is of fixed size
793	 * and is known in advance.  We allocate one extra element since we
794	 * use element indices instead of pointers and zero is our sentinel.
795	 */
796	dmp->dm_nsymelems =
797	    dmp->dm_symtab.cts_size / dmp->dm_symtab.cts_entsize;
798
799	dmp->dm_nsymbuckets = _dtrace_strbuckets;
800	dmp->dm_symfree = 1;		/* first free element is index 1 */
801
802	dmp->dm_symbuckets = calloc(dmp->dm_nsymbuckets, sizeof (uint_t));
803	dmp->dm_symchains = calloc(dmp->dm_nsymelems + 1, sizeof (dt_sym_t));
804
805	if (dmp->dm_symbuckets == NULL || dmp->dm_symchains == NULL) {
806		dt_module_unload(dtp, dmp);
807		return (dt_set_errno(dtp, EDT_NOMEM));
808	}
809
810	/*
811	 * Iterate over the symbol table data buffer and insert each symbol
812	 * name into the name hash if the name and type are valid.  Then
813	 * allocate the address map, fill it in, and sort it.
814	 */
815	dmp->dm_asrsv = dmp->dm_ops->do_syminit(dmp);
816
817	dt_dprintf("hashed %s [%s] (%u symbols)\n",
818	    dmp->dm_name, dmp->dm_symtab.cts_name, dmp->dm_symfree - 1);
819
820	if ((dmp->dm_asmap = malloc(sizeof (void *) * dmp->dm_asrsv)) == NULL) {
821		dt_module_unload(dtp, dmp);
822		return (dt_set_errno(dtp, EDT_NOMEM));
823	}
824
825	dmp->dm_ops->do_symsort(dmp);
826
827	dt_dprintf("sorted %s [%s] (%u symbols)\n",
828	    dmp->dm_name, dmp->dm_symtab.cts_name, dmp->dm_aslen);
829
830	dmp->dm_flags |= DT_DM_LOADED;
831	return (0);
832}
833
834int
835dt_module_hasctf(dtrace_hdl_t *dtp, dt_module_t *dmp)
836{
837	if (dmp->dm_pid != 0 && dmp->dm_nctflibs > 0)
838		return (1);
839	return (dt_module_getctf(dtp, dmp) != NULL);
840}
841
842ctf_file_t *
843dt_module_getctf(dtrace_hdl_t *dtp, dt_module_t *dmp)
844{
845	const char *parent;
846	dt_module_t *pmp;
847	ctf_file_t *pfp;
848	int model;
849
850	if (dmp->dm_ctfp != NULL || dt_module_load(dtp, dmp) != 0)
851		return (dmp->dm_ctfp);
852
853	if (dmp->dm_ops == &dt_modops_64)
854		model = CTF_MODEL_LP64;
855	else
856		model = CTF_MODEL_ILP32;
857
858	/*
859	 * If the data model of the module does not match our program data
860	 * model, then do not permit CTF from this module to be opened and
861	 * returned to the compiler.  If we support mixed data models in the
862	 * future for combined kernel/user tracing, this can be removed.
863	 */
864	if (dtp->dt_conf.dtc_ctfmodel != model) {
865		(void) dt_set_errno(dtp, EDT_DATAMODEL);
866		return (NULL);
867	}
868
869	if (dmp->dm_ctdata.cts_size == 0) {
870		(void) dt_set_errno(dtp, EDT_NOCTF);
871		return (NULL);
872	}
873
874	dmp->dm_ctfp = ctf_bufopen(&dmp->dm_ctdata,
875	    &dmp->dm_symtab, &dmp->dm_strtab, &dtp->dt_ctferr);
876
877	if (dmp->dm_ctfp == NULL) {
878		(void) dt_set_errno(dtp, EDT_CTF);
879		return (NULL);
880	}
881
882	(void) ctf_setmodel(dmp->dm_ctfp, model);
883	ctf_setspecific(dmp->dm_ctfp, dmp);
884
885	if ((parent = ctf_parent_name(dmp->dm_ctfp)) != NULL) {
886		if ((pmp = dt_module_create(dtp, parent)) == NULL ||
887		    (pfp = dt_module_getctf(dtp, pmp)) == NULL) {
888			if (pmp == NULL)
889				(void) dt_set_errno(dtp, EDT_NOMEM);
890			goto err;
891		}
892
893		if (ctf_import(dmp->dm_ctfp, pfp) == CTF_ERR) {
894			dtp->dt_ctferr = ctf_errno(dmp->dm_ctfp);
895			(void) dt_set_errno(dtp, EDT_CTF);
896			goto err;
897		}
898	}
899
900	dt_dprintf("loaded CTF container for %s (%p)\n",
901	    dmp->dm_name, (void *)dmp->dm_ctfp);
902
903	return (dmp->dm_ctfp);
904
905err:
906	ctf_close(dmp->dm_ctfp);
907	dmp->dm_ctfp = NULL;
908	return (NULL);
909}
910
911/*ARGSUSED*/
912void
913dt_module_unload(dtrace_hdl_t *dtp, dt_module_t *dmp)
914{
915	int i;
916
917	ctf_close(dmp->dm_ctfp);
918	dmp->dm_ctfp = NULL;
919
920#ifndef illumos
921	if (dmp->dm_ctdata.cts_data != NULL) {
922		free(dmp->dm_ctdata.cts_data);
923	}
924	if (dmp->dm_symtab.cts_data != NULL) {
925		free(dmp->dm_symtab.cts_data);
926	}
927	if (dmp->dm_strtab.cts_data != NULL) {
928		free(dmp->dm_strtab.cts_data);
929	}
930#endif
931
932	if (dmp->dm_libctfp != NULL) {
933		for (i = 0; i < dmp->dm_nctflibs; i++) {
934			ctf_close(dmp->dm_libctfp[i]);
935			free(dmp->dm_libctfn[i]);
936		}
937		free(dmp->dm_libctfp);
938		free(dmp->dm_libctfn);
939		dmp->dm_libctfp = NULL;
940		dmp->dm_nctflibs = 0;
941	}
942
943	bzero(&dmp->dm_ctdata, sizeof (ctf_sect_t));
944	bzero(&dmp->dm_symtab, sizeof (ctf_sect_t));
945	bzero(&dmp->dm_strtab, sizeof (ctf_sect_t));
946
947	if (dmp->dm_symbuckets != NULL) {
948		free(dmp->dm_symbuckets);
949		dmp->dm_symbuckets = NULL;
950	}
951
952	if (dmp->dm_symchains != NULL) {
953		free(dmp->dm_symchains);
954		dmp->dm_symchains = NULL;
955	}
956
957	if (dmp->dm_asmap != NULL) {
958		free(dmp->dm_asmap);
959		dmp->dm_asmap = NULL;
960	}
961#if defined(__FreeBSD__)
962	if (dmp->dm_sec_offsets != NULL) {
963		free(dmp->dm_sec_offsets);
964		dmp->dm_sec_offsets = NULL;
965	}
966#endif
967	dmp->dm_symfree = 0;
968	dmp->dm_nsymbuckets = 0;
969	dmp->dm_nsymelems = 0;
970	dmp->dm_asrsv = 0;
971	dmp->dm_aslen = 0;
972
973	dmp->dm_text_va = 0;
974	dmp->dm_text_size = 0;
975	dmp->dm_data_va = 0;
976	dmp->dm_data_size = 0;
977	dmp->dm_bss_va = 0;
978	dmp->dm_bss_size = 0;
979
980	if (dmp->dm_extern != NULL) {
981		dt_idhash_destroy(dmp->dm_extern);
982		dmp->dm_extern = NULL;
983	}
984
985	(void) elf_end(dmp->dm_elf);
986	dmp->dm_elf = NULL;
987
988	dmp->dm_pid = 0;
989
990	dmp->dm_flags &= ~DT_DM_LOADED;
991}
992
993void
994dt_module_destroy(dtrace_hdl_t *dtp, dt_module_t *dmp)
995{
996	uint_t h = dt_strtab_hash(dmp->dm_name, NULL) % dtp->dt_modbuckets;
997	dt_module_t **dmpp = &dtp->dt_mods[h];
998
999	dt_list_delete(&dtp->dt_modlist, dmp);
1000	assert(dtp->dt_nmods != 0);
1001	dtp->dt_nmods--;
1002
1003	/*
1004	 * Now remove this module from its hash chain.  We expect to always
1005	 * find the module on its hash chain, so in this loop we assert that
1006	 * we don't run off the end of the list.
1007	 */
1008	while (*dmpp != dmp) {
1009		dmpp = &((*dmpp)->dm_next);
1010		assert(*dmpp != NULL);
1011	}
1012
1013	*dmpp = dmp->dm_next;
1014
1015	dt_module_unload(dtp, dmp);
1016	free(dmp);
1017}
1018
1019/*
1020 * Insert a new external symbol reference into the specified module.  The new
1021 * symbol will be marked as undefined and is assigned a symbol index beyond
1022 * any existing cached symbols from this module.  We use the ident's di_data
1023 * field to store a pointer to a copy of the dtrace_syminfo_t for this symbol.
1024 */
1025dt_ident_t *
1026dt_module_extern(dtrace_hdl_t *dtp, dt_module_t *dmp,
1027    const char *name, const dtrace_typeinfo_t *tip)
1028{
1029	dtrace_syminfo_t *sip;
1030	dt_ident_t *idp;
1031	uint_t id;
1032
1033	if (dmp->dm_extern == NULL && (dmp->dm_extern = dt_idhash_create(
1034	    "extern", NULL, dmp->dm_nsymelems, UINT_MAX)) == NULL) {
1035		(void) dt_set_errno(dtp, EDT_NOMEM);
1036		return (NULL);
1037	}
1038
1039	if (dt_idhash_nextid(dmp->dm_extern, &id) == -1) {
1040		(void) dt_set_errno(dtp, EDT_SYMOFLOW);
1041		return (NULL);
1042	}
1043
1044	if ((sip = malloc(sizeof (dtrace_syminfo_t))) == NULL) {
1045		(void) dt_set_errno(dtp, EDT_NOMEM);
1046		return (NULL);
1047	}
1048
1049	idp = dt_idhash_insert(dmp->dm_extern, name, DT_IDENT_SYMBOL, 0, id,
1050	    _dtrace_symattr, 0, &dt_idops_thaw, NULL, dtp->dt_gen);
1051
1052	if (idp == NULL) {
1053		(void) dt_set_errno(dtp, EDT_NOMEM);
1054		free(sip);
1055		return (NULL);
1056	}
1057
1058	sip->dts_object = dmp->dm_name;
1059	sip->dts_name = idp->di_name;
1060	sip->dts_id = idp->di_id;
1061
1062	idp->di_data = sip;
1063	idp->di_ctfp = tip->dtt_ctfp;
1064	idp->di_type = tip->dtt_type;
1065
1066	return (idp);
1067}
1068
1069const char *
1070dt_module_modelname(dt_module_t *dmp)
1071{
1072	if (dmp->dm_ops == &dt_modops_64)
1073		return ("64-bit");
1074	else
1075		return ("32-bit");
1076}
1077
1078/* ARGSUSED */
1079int
1080dt_module_getlibid(dtrace_hdl_t *dtp, dt_module_t *dmp, const ctf_file_t *fp)
1081{
1082	int i;
1083
1084	for (i = 0; i < dmp->dm_nctflibs; i++) {
1085		if (dmp->dm_libctfp[i] == fp)
1086			return (i);
1087	}
1088
1089	return (-1);
1090}
1091
1092/* ARGSUSED */
1093ctf_file_t *
1094dt_module_getctflib(dtrace_hdl_t *dtp, dt_module_t *dmp, const char *name)
1095{
1096	int i;
1097
1098	for (i = 0; i < dmp->dm_nctflibs; i++) {
1099		if (strcmp(dmp->dm_libctfn[i], name) == 0)
1100			return (dmp->dm_libctfp[i]);
1101	}
1102
1103	return (NULL);
1104}
1105
1106/*
1107 * Update our module cache by adding an entry for the specified module 'name'.
1108 * We create the dt_module_t and populate it using /system/object/<name>/.
1109 *
1110 * On FreeBSD, the module name is passed as the full module file name,
1111 * including the path.
1112 */
1113static void
1114#ifdef illumos
1115dt_module_update(dtrace_hdl_t *dtp, const char *name)
1116#else
1117dt_module_update(dtrace_hdl_t *dtp, struct kld_file_stat *k_stat)
1118#endif
1119{
1120	char fname[MAXPATHLEN];
1121	struct stat64 st;
1122	int fd, err, bits;
1123
1124	dt_module_t *dmp;
1125	const char *s;
1126	size_t shstrs;
1127	GElf_Shdr sh;
1128	Elf_Data *dp;
1129	Elf_Scn *sp;
1130
1131#ifdef illumos
1132	(void) snprintf(fname, sizeof (fname),
1133	    "%s/%s/object", OBJFS_ROOT, name);
1134#else
1135	GElf_Ehdr ehdr;
1136	GElf_Phdr ph;
1137	char name[MAXPATHLEN];
1138	uintptr_t mapbase, alignmask;
1139	int i = 0;
1140	int is_elf_obj;
1141
1142	(void) strlcpy(name, k_stat->name, sizeof(name));
1143	(void) strlcpy(fname, k_stat->pathname, sizeof(fname));
1144#endif
1145
1146	if ((fd = open(fname, O_RDONLY)) == -1 || fstat64(fd, &st) == -1 ||
1147	    (dmp = dt_module_create(dtp, name)) == NULL) {
1148		dt_dprintf("failed to open %s: %s\n", fname, strerror(errno));
1149		(void) close(fd);
1150		return;
1151	}
1152
1153	/*
1154	 * Since the module can unload out from under us (and /system/object
1155	 * will return ENOENT), tell libelf to cook the entire file now and
1156	 * then close the underlying file descriptor immediately.  If this
1157	 * succeeds, we know that we can continue safely using dmp->dm_elf.
1158	 */
1159	dmp->dm_elf = elf_begin(fd, ELF_C_READ, NULL);
1160	err = elf_cntl(dmp->dm_elf, ELF_C_FDREAD);
1161	(void) close(fd);
1162
1163	if (dmp->dm_elf == NULL || err == -1 ||
1164	    elf_getshdrstrndx(dmp->dm_elf, &shstrs) == -1) {
1165		dt_dprintf("failed to load %s: %s\n",
1166		    fname, elf_errmsg(elf_errno()));
1167		dt_module_destroy(dtp, dmp);
1168		return;
1169	}
1170
1171	switch (gelf_getclass(dmp->dm_elf)) {
1172	case ELFCLASS32:
1173		dmp->dm_ops = &dt_modops_32;
1174		bits = 32;
1175		break;
1176	case ELFCLASS64:
1177		dmp->dm_ops = &dt_modops_64;
1178		bits = 64;
1179		break;
1180	default:
1181		dt_dprintf("failed to load %s: unknown ELF class\n", fname);
1182		dt_module_destroy(dtp, dmp);
1183		return;
1184	}
1185#if defined(__FreeBSD__)
1186	mapbase = (uintptr_t)k_stat->address;
1187	gelf_getehdr(dmp->dm_elf, &ehdr);
1188	is_elf_obj = (ehdr.e_type == ET_REL);
1189	if (is_elf_obj) {
1190		dmp->dm_sec_offsets =
1191		    malloc(ehdr.e_shnum * sizeof(*dmp->dm_sec_offsets));
1192		if (dmp->dm_sec_offsets == NULL) {
1193			dt_dprintf("failed to allocate memory\n");
1194			dt_module_destroy(dtp, dmp);
1195			return;
1196		}
1197	}
1198#endif
1199	/*
1200	 * Iterate over the section headers locating various sections of
1201	 * interest and use their attributes to flesh out the dt_module_t.
1202	 */
1203	for (sp = NULL; (sp = elf_nextscn(dmp->dm_elf, sp)) != NULL; ) {
1204		if (gelf_getshdr(sp, &sh) == NULL || sh.sh_type == SHT_NULL ||
1205		    (s = elf_strptr(dmp->dm_elf, shstrs, sh.sh_name)) == NULL)
1206			continue; /* skip any malformed sections */
1207#if defined(__FreeBSD__)
1208		if (sh.sh_size == 0)
1209			continue;
1210		if (sh.sh_type == SHT_PROGBITS || sh.sh_type == SHT_NOBITS) {
1211			alignmask = sh.sh_addralign - 1;
1212			mapbase += alignmask;
1213			mapbase &= ~alignmask;
1214			sh.sh_addr = mapbase;
1215			if (is_elf_obj)
1216				dmp->dm_sec_offsets[elf_ndxscn(sp)] = sh.sh_addr;
1217			mapbase += sh.sh_size;
1218		}
1219#endif
1220		if (strcmp(s, ".text") == 0) {
1221			dmp->dm_text_size = sh.sh_size;
1222			dmp->dm_text_va = sh.sh_addr;
1223		} else if (strcmp(s, ".data") == 0) {
1224			dmp->dm_data_size = sh.sh_size;
1225			dmp->dm_data_va = sh.sh_addr;
1226		} else if (strcmp(s, ".bss") == 0) {
1227			dmp->dm_bss_size = sh.sh_size;
1228			dmp->dm_bss_va = sh.sh_addr;
1229		} else if (strcmp(s, ".info") == 0 &&
1230		    (dp = elf_getdata(sp, NULL)) != NULL) {
1231			bcopy(dp->d_buf, &dmp->dm_info,
1232			    MIN(sh.sh_size, sizeof (dmp->dm_info)));
1233		} else if (strcmp(s, ".filename") == 0 &&
1234		    (dp = elf_getdata(sp, NULL)) != NULL) {
1235			(void) strlcpy(dmp->dm_file,
1236			    dp->d_buf, sizeof (dmp->dm_file));
1237		}
1238	}
1239
1240	dmp->dm_flags |= DT_DM_KERNEL;
1241#ifdef illumos
1242	dmp->dm_modid = (int)OBJFS_MODID(st.st_ino);
1243#else
1244	/*
1245	 * Include .rodata and special sections into .text.
1246	 * This depends on default section layout produced by GNU ld
1247	 * for ELF objects and libraries:
1248	 * [Text][R/O data][R/W data][Dynamic][BSS][Non loadable]
1249	 */
1250	dmp->dm_text_size = dmp->dm_data_va - dmp->dm_text_va;
1251#if defined(__i386__)
1252	/*
1253	 * Find the first load section and figure out the relocation
1254	 * offset for the symbols. The kernel module will not need
1255	 * relocation, but the kernel linker modules will.
1256	 */
1257	for (i = 0; gelf_getphdr(dmp->dm_elf, i, &ph) != NULL; i++) {
1258		if (ph.p_type == PT_LOAD) {
1259			dmp->dm_reloc_offset = k_stat->address - ph.p_vaddr;
1260			break;
1261		}
1262	}
1263#endif
1264#endif /* illumos */
1265
1266	if (dmp->dm_info.objfs_info_primary)
1267		dmp->dm_flags |= DT_DM_PRIMARY;
1268
1269	dt_dprintf("opened %d-bit module %s (%s) [%d]\n",
1270	    bits, dmp->dm_name, dmp->dm_file, dmp->dm_modid);
1271}
1272
1273/*
1274 * Unload all the loaded modules and then refresh the module cache with the
1275 * latest list of loaded modules and their address ranges.
1276 */
1277void
1278dtrace_update(dtrace_hdl_t *dtp)
1279{
1280	dt_module_t *dmp;
1281	DIR *dirp;
1282#if defined(__FreeBSD__)
1283	int fileid;
1284#endif
1285
1286	for (dmp = dt_list_next(&dtp->dt_modlist);
1287	    dmp != NULL; dmp = dt_list_next(dmp))
1288		dt_module_unload(dtp, dmp);
1289
1290#ifdef illumos
1291	/*
1292	 * Open /system/object and attempt to create a libdtrace module for
1293	 * each kernel module that is loaded on the current system.
1294	 */
1295	if (!(dtp->dt_oflags & DTRACE_O_NOSYS) &&
1296	    (dirp = opendir(OBJFS_ROOT)) != NULL) {
1297		struct dirent *dp;
1298
1299		while ((dp = readdir(dirp)) != NULL) {
1300			if (dp->d_name[0] != '.')
1301				dt_module_update(dtp, dp->d_name);
1302		}
1303
1304		(void) closedir(dirp);
1305	}
1306#elif defined(__FreeBSD__)
1307	/*
1308	 * Use FreeBSD's kernel loader interface to discover what kernel
1309	 * modules are loaded and create a libdtrace module for each one.
1310	 */
1311	for (fileid = kldnext(0); fileid > 0; fileid = kldnext(fileid)) {
1312		struct kld_file_stat k_stat;
1313		k_stat.version = sizeof(k_stat);
1314		if (kldstat(fileid, &k_stat) == 0)
1315			dt_module_update(dtp, &k_stat);
1316	}
1317#endif
1318
1319	/*
1320	 * Look up all the macro identifiers and set di_id to the latest value.
1321	 * This code collaborates with dt_lex.l on the use of di_id.  We will
1322	 * need to implement something fancier if we need to support non-ints.
1323	 */
1324	dt_idhash_lookup(dtp->dt_macros, "egid")->di_id = getegid();
1325	dt_idhash_lookup(dtp->dt_macros, "euid")->di_id = geteuid();
1326	dt_idhash_lookup(dtp->dt_macros, "gid")->di_id = getgid();
1327	dt_idhash_lookup(dtp->dt_macros, "pid")->di_id = getpid();
1328	dt_idhash_lookup(dtp->dt_macros, "pgid")->di_id = getpgid(0);
1329	dt_idhash_lookup(dtp->dt_macros, "ppid")->di_id = getppid();
1330#ifdef illumos
1331	dt_idhash_lookup(dtp->dt_macros, "projid")->di_id = getprojid();
1332#endif
1333	dt_idhash_lookup(dtp->dt_macros, "sid")->di_id = getsid(0);
1334#ifdef illumos
1335	dt_idhash_lookup(dtp->dt_macros, "taskid")->di_id = gettaskid();
1336#endif
1337	dt_idhash_lookup(dtp->dt_macros, "uid")->di_id = getuid();
1338
1339	/*
1340	 * Cache the pointers to the modules representing the base executable
1341	 * and the run-time linker in the dtrace client handle. Note that on
1342	 * x86 krtld is folded into unix, so if we don't find it, use unix
1343	 * instead.
1344	 */
1345	dtp->dt_exec = dt_module_lookup_by_name(dtp, "genunix");
1346	dtp->dt_rtld = dt_module_lookup_by_name(dtp, "krtld");
1347	if (dtp->dt_rtld == NULL)
1348		dtp->dt_rtld = dt_module_lookup_by_name(dtp, "unix");
1349
1350	/*
1351	 * If this is the first time we are initializing the module list,
1352	 * remove the module for genunix from the module list and then move it
1353	 * to the front of the module list.  We do this so that type and symbol
1354	 * queries encounter genunix and thereby optimize for the common case
1355	 * in dtrace_lookup_by_name() and dtrace_lookup_by_type(), below.
1356	 */
1357	if (dtp->dt_exec != NULL &&
1358	    dtp->dt_cdefs == NULL && dtp->dt_ddefs == NULL) {
1359		dt_list_delete(&dtp->dt_modlist, dtp->dt_exec);
1360		dt_list_prepend(&dtp->dt_modlist, dtp->dt_exec);
1361	}
1362}
1363
1364static dt_module_t *
1365dt_module_from_object(dtrace_hdl_t *dtp, const char *object)
1366{
1367	int err = EDT_NOMOD;
1368	dt_module_t *dmp;
1369
1370	switch ((uintptr_t)object) {
1371	case (uintptr_t)DTRACE_OBJ_EXEC:
1372		dmp = dtp->dt_exec;
1373		break;
1374	case (uintptr_t)DTRACE_OBJ_RTLD:
1375		dmp = dtp->dt_rtld;
1376		break;
1377	case (uintptr_t)DTRACE_OBJ_CDEFS:
1378		dmp = dtp->dt_cdefs;
1379		break;
1380	case (uintptr_t)DTRACE_OBJ_DDEFS:
1381		dmp = dtp->dt_ddefs;
1382		break;
1383	default:
1384		dmp = dt_module_create(dtp, object);
1385		err = EDT_NOMEM;
1386	}
1387
1388	if (dmp == NULL)
1389		(void) dt_set_errno(dtp, err);
1390
1391	return (dmp);
1392}
1393
1394/*
1395 * Exported interface to look up a symbol by name.  We return the GElf_Sym and
1396 * complete symbol information for the matching symbol.
1397 */
1398int
1399dtrace_lookup_by_name(dtrace_hdl_t *dtp, const char *object, const char *name,
1400    GElf_Sym *symp, dtrace_syminfo_t *sip)
1401{
1402	dt_module_t *dmp;
1403	dt_ident_t *idp;
1404	uint_t n, id;
1405	GElf_Sym sym;
1406
1407	uint_t mask = 0; /* mask of dt_module flags to match */
1408	uint_t bits = 0; /* flag bits that must be present */
1409
1410	if (object != DTRACE_OBJ_EVERY &&
1411	    object != DTRACE_OBJ_KMODS &&
1412	    object != DTRACE_OBJ_UMODS) {
1413		if ((dmp = dt_module_from_object(dtp, object)) == NULL)
1414			return (-1); /* dt_errno is set for us */
1415
1416		if (dt_module_load(dtp, dmp) == -1)
1417			return (-1); /* dt_errno is set for us */
1418		n = 1;
1419
1420	} else {
1421		if (object == DTRACE_OBJ_KMODS)
1422			mask = bits = DT_DM_KERNEL;
1423		else if (object == DTRACE_OBJ_UMODS)
1424			mask = DT_DM_KERNEL;
1425
1426		dmp = dt_list_next(&dtp->dt_modlist);
1427		n = dtp->dt_nmods;
1428	}
1429
1430	if (symp == NULL)
1431		symp = &sym;
1432
1433	for (; n > 0; n--, dmp = dt_list_next(dmp)) {
1434		if ((dmp->dm_flags & mask) != bits)
1435			continue; /* failed to match required attributes */
1436
1437		if (dt_module_load(dtp, dmp) == -1)
1438			continue; /* failed to load symbol table */
1439
1440		if (dmp->dm_ops->do_symname(dmp, name, symp, &id) != NULL) {
1441			if (sip != NULL) {
1442				sip->dts_object = dmp->dm_name;
1443				sip->dts_name = (const char *)
1444				    dmp->dm_strtab.cts_data + symp->st_name;
1445				sip->dts_id = id;
1446			}
1447			return (0);
1448		}
1449
1450		if (dmp->dm_extern != NULL &&
1451		    (idp = dt_idhash_lookup(dmp->dm_extern, name)) != NULL) {
1452			if (symp != &sym) {
1453				symp->st_name = (uintptr_t)idp->di_name;
1454				symp->st_info =
1455				    GELF_ST_INFO(STB_GLOBAL, STT_NOTYPE);
1456				symp->st_other = 0;
1457				symp->st_shndx = SHN_UNDEF;
1458				symp->st_value = 0;
1459				symp->st_size =
1460				    ctf_type_size(idp->di_ctfp, idp->di_type);
1461			}
1462
1463			if (sip != NULL) {
1464				sip->dts_object = dmp->dm_name;
1465				sip->dts_name = idp->di_name;
1466				sip->dts_id = idp->di_id;
1467			}
1468
1469			return (0);
1470		}
1471	}
1472
1473	return (dt_set_errno(dtp, EDT_NOSYM));
1474}
1475
1476/*
1477 * Exported interface to look up a symbol by address.  We return the GElf_Sym
1478 * and complete symbol information for the matching symbol.
1479 */
1480int
1481dtrace_lookup_by_addr(dtrace_hdl_t *dtp, GElf_Addr addr,
1482    GElf_Sym *symp, dtrace_syminfo_t *sip)
1483{
1484	dt_module_t *dmp;
1485	uint_t id;
1486	const dtrace_vector_t *v = dtp->dt_vector;
1487
1488	if (v != NULL)
1489		return (v->dtv_lookup_by_addr(dtp->dt_varg, addr, symp, sip));
1490
1491	for (dmp = dt_list_next(&dtp->dt_modlist); dmp != NULL;
1492	    dmp = dt_list_next(dmp)) {
1493		if (addr - dmp->dm_text_va < dmp->dm_text_size ||
1494		    addr - dmp->dm_data_va < dmp->dm_data_size ||
1495		    addr - dmp->dm_bss_va < dmp->dm_bss_size)
1496			break;
1497	}
1498
1499	if (dmp == NULL)
1500		return (dt_set_errno(dtp, EDT_NOSYMADDR));
1501
1502	if (dt_module_load(dtp, dmp) == -1)
1503		return (-1); /* dt_errno is set for us */
1504
1505	if (symp != NULL) {
1506		if (dmp->dm_ops->do_symaddr(dmp, addr, symp, &id) == NULL)
1507			return (dt_set_errno(dtp, EDT_NOSYMADDR));
1508	}
1509
1510	if (sip != NULL) {
1511		sip->dts_object = dmp->dm_name;
1512
1513		if (symp != NULL) {
1514			sip->dts_name = (const char *)
1515			    dmp->dm_strtab.cts_data + symp->st_name;
1516			sip->dts_id = id;
1517		} else {
1518			sip->dts_name = NULL;
1519			sip->dts_id = 0;
1520		}
1521	}
1522
1523	return (0);
1524}
1525
1526int
1527dtrace_lookup_by_type(dtrace_hdl_t *dtp, const char *object, const char *name,
1528    dtrace_typeinfo_t *tip)
1529{
1530	dtrace_typeinfo_t ti;
1531	dt_module_t *dmp;
1532	int found = 0;
1533	ctf_id_t id;
1534	uint_t n, i;
1535	int justone;
1536	ctf_file_t *fp;
1537	char *buf, *p, *q;
1538
1539	uint_t mask = 0; /* mask of dt_module flags to match */
1540	uint_t bits = 0; /* flag bits that must be present */
1541
1542	if (object != DTRACE_OBJ_EVERY &&
1543	    object != DTRACE_OBJ_KMODS &&
1544	    object != DTRACE_OBJ_UMODS) {
1545		if ((dmp = dt_module_from_object(dtp, object)) == NULL)
1546			return (-1); /* dt_errno is set for us */
1547
1548		if (dt_module_load(dtp, dmp) == -1)
1549			return (-1); /* dt_errno is set for us */
1550		n = 1;
1551		justone = 1;
1552	} else {
1553		if (object == DTRACE_OBJ_KMODS)
1554			mask = bits = DT_DM_KERNEL;
1555		else if (object == DTRACE_OBJ_UMODS)
1556			mask = DT_DM_KERNEL;
1557
1558		dmp = dt_list_next(&dtp->dt_modlist);
1559		n = dtp->dt_nmods;
1560		justone = 0;
1561	}
1562
1563	if (tip == NULL)
1564		tip = &ti;
1565
1566	for (; n > 0; n--, dmp = dt_list_next(dmp)) {
1567		if ((dmp->dm_flags & mask) != bits)
1568			continue; /* failed to match required attributes */
1569
1570		/*
1571		 * If we can't load the CTF container, continue on to the next
1572		 * module.  If our search was scoped to only one module then
1573		 * return immediately leaving dt_errno unmodified.
1574		 */
1575		if (dt_module_hasctf(dtp, dmp) == 0) {
1576			if (justone)
1577				return (-1);
1578			continue;
1579		}
1580
1581		/*
1582		 * Look up the type in the module's CTF container.  If our
1583		 * match is a forward declaration tag, save this choice in
1584		 * 'tip' and keep going in the hope that we will locate the
1585		 * underlying structure definition.  Otherwise just return.
1586		 */
1587		if (dmp->dm_pid == 0) {
1588			id = ctf_lookup_by_name(dmp->dm_ctfp, name);
1589			fp = dmp->dm_ctfp;
1590		} else {
1591			if ((p = strchr(name, '`')) != NULL) {
1592				buf = strdup(name);
1593				if (buf == NULL)
1594					return (dt_set_errno(dtp, EDT_NOMEM));
1595				p = strchr(buf, '`');
1596				if ((q = strchr(p + 1, '`')) != NULL)
1597					p = q;
1598				*p = '\0';
1599				fp = dt_module_getctflib(dtp, dmp, buf);
1600				if (fp == NULL || (id = ctf_lookup_by_name(fp,
1601				    p + 1)) == CTF_ERR)
1602					id = CTF_ERR;
1603				free(buf);
1604			} else {
1605				for (i = 0; i < dmp->dm_nctflibs; i++) {
1606					fp = dmp->dm_libctfp[i];
1607					id = ctf_lookup_by_name(fp, name);
1608					if (id != CTF_ERR)
1609						break;
1610				}
1611			}
1612		}
1613		if (id != CTF_ERR) {
1614			tip->dtt_object = dmp->dm_name;
1615			tip->dtt_ctfp = fp;
1616			tip->dtt_type = id;
1617			if (ctf_type_kind(fp, ctf_type_resolve(fp, id)) !=
1618			    CTF_K_FORWARD)
1619				return (0);
1620
1621			found++;
1622		}
1623	}
1624
1625	if (found == 0)
1626		return (dt_set_errno(dtp, EDT_NOTYPE));
1627
1628	return (0);
1629}
1630
1631int
1632dtrace_symbol_type(dtrace_hdl_t *dtp, const GElf_Sym *symp,
1633    const dtrace_syminfo_t *sip, dtrace_typeinfo_t *tip)
1634{
1635	dt_module_t *dmp;
1636
1637	tip->dtt_object = NULL;
1638	tip->dtt_ctfp = NULL;
1639	tip->dtt_type = CTF_ERR;
1640	tip->dtt_flags = 0;
1641
1642	if ((dmp = dt_module_lookup_by_name(dtp, sip->dts_object)) == NULL)
1643		return (dt_set_errno(dtp, EDT_NOMOD));
1644
1645	if (symp->st_shndx == SHN_UNDEF && dmp->dm_extern != NULL) {
1646		dt_ident_t *idp =
1647		    dt_idhash_lookup(dmp->dm_extern, sip->dts_name);
1648
1649		if (idp == NULL)
1650			return (dt_set_errno(dtp, EDT_NOSYM));
1651
1652		tip->dtt_ctfp = idp->di_ctfp;
1653		tip->dtt_type = idp->di_type;
1654
1655	} else if (GELF_ST_TYPE(symp->st_info) != STT_FUNC) {
1656		if (dt_module_getctf(dtp, dmp) == NULL)
1657			return (-1); /* errno is set for us */
1658
1659		tip->dtt_ctfp = dmp->dm_ctfp;
1660		tip->dtt_type = ctf_lookup_by_symbol(dmp->dm_ctfp, sip->dts_id);
1661
1662		if (tip->dtt_type == CTF_ERR) {
1663			dtp->dt_ctferr = ctf_errno(tip->dtt_ctfp);
1664			return (dt_set_errno(dtp, EDT_CTF));
1665		}
1666
1667	} else {
1668		tip->dtt_ctfp = DT_FPTR_CTFP(dtp);
1669		tip->dtt_type = DT_FPTR_TYPE(dtp);
1670	}
1671
1672	tip->dtt_object = dmp->dm_name;
1673	return (0);
1674}
1675
1676static dtrace_objinfo_t *
1677dt_module_info(const dt_module_t *dmp, dtrace_objinfo_t *dto)
1678{
1679	dto->dto_name = dmp->dm_name;
1680	dto->dto_file = dmp->dm_file;
1681	dto->dto_id = dmp->dm_modid;
1682	dto->dto_flags = 0;
1683
1684	if (dmp->dm_flags & DT_DM_KERNEL)
1685		dto->dto_flags |= DTRACE_OBJ_F_KERNEL;
1686	if (dmp->dm_flags & DT_DM_PRIMARY)
1687		dto->dto_flags |= DTRACE_OBJ_F_PRIMARY;
1688
1689	dto->dto_text_va = dmp->dm_text_va;
1690	dto->dto_text_size = dmp->dm_text_size;
1691	dto->dto_data_va = dmp->dm_data_va;
1692	dto->dto_data_size = dmp->dm_data_size;
1693	dto->dto_bss_va = dmp->dm_bss_va;
1694	dto->dto_bss_size = dmp->dm_bss_size;
1695
1696	return (dto);
1697}
1698
1699int
1700dtrace_object_iter(dtrace_hdl_t *dtp, dtrace_obj_f *func, void *data)
1701{
1702	const dt_module_t *dmp = dt_list_next(&dtp->dt_modlist);
1703	dtrace_objinfo_t dto;
1704	int rv;
1705
1706	for (; dmp != NULL; dmp = dt_list_next(dmp)) {
1707		if ((rv = (*func)(dtp, dt_module_info(dmp, &dto), data)) != 0)
1708			return (rv);
1709	}
1710
1711	return (0);
1712}
1713
1714int
1715dtrace_object_info(dtrace_hdl_t *dtp, const char *object, dtrace_objinfo_t *dto)
1716{
1717	dt_module_t *dmp;
1718
1719	if (object == DTRACE_OBJ_EVERY || object == DTRACE_OBJ_KMODS ||
1720	    object == DTRACE_OBJ_UMODS || dto == NULL)
1721		return (dt_set_errno(dtp, EINVAL));
1722
1723	if ((dmp = dt_module_from_object(dtp, object)) == NULL)
1724		return (-1); /* dt_errno is set for us */
1725
1726	if (dt_module_load(dtp, dmp) == -1)
1727		return (-1); /* dt_errno is set for us */
1728
1729	(void) dt_module_info(dmp, dto);
1730	return (0);
1731}
1732