1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
23 *
24 * Copyright (c) 2016, Intel Corporation.
25 * Copyright (c) 2023, Klara Inc.
26 */
27
28/*
29 * This file implements the minimal FMD module API required to support the
30 * fault logic modules in ZED. This support includes module registration,
31 * memory allocation, module property accessors, basic case management,
32 * one-shot timers and SERD engines.
33 *
34 * In the ZED runtime, the modules are called from a single thread so no
35 * locking is required in this emulated FMD environment.
36 */
37
38#include <sys/types.h>
39#include <sys/fm/protocol.h>
40#include <uuid/uuid.h>
41#include <signal.h>
42#include <string.h>
43#include <time.h>
44
45#include "fmd_api.h"
46#include "fmd_serd.h"
47
48#include "zfs_agents.h"
49#include "../zed_log.h"
50
51typedef struct fmd_modstat {
52	fmd_stat_t	ms_accepted;	/* total events accepted by module */
53	fmd_stat_t	ms_caseopen;	/* cases currently open */
54	fmd_stat_t	ms_casesolved;	/* total cases solved by module */
55	fmd_stat_t	ms_caseclosed;	/* total cases closed by module */
56} fmd_modstat_t;
57
58typedef struct fmd_module {
59	const char	*mod_name;	/* basename of module (ro) */
60	const fmd_hdl_info_t *mod_info;	/* module info registered with handle */
61	void		*mod_spec;	/* fmd_hdl_get/setspecific data value */
62	fmd_stat_t	*mod_ustat;	/* module specific custom stats */
63	uint_t		mod_ustat_cnt;	/* count of ustat stats */
64	fmd_modstat_t	mod_stats;	/* fmd built-in per-module statistics */
65	fmd_serd_hash_t	mod_serds;	/* hash of serd engs owned by module */
66	char		*mod_vers;	/* a copy of module version string */
67} fmd_module_t;
68
69/*
70 * ZED has two FMD hardwired module instances
71 */
72fmd_module_t	zfs_retire_module;
73fmd_module_t	zfs_diagnosis_module;
74
75/*
76 * Enable a reasonable set of defaults for libumem debugging on DEBUG builds.
77 */
78
79#ifdef DEBUG
80const char *
81_umem_debug_init(void)
82{
83	return ("default,verbose"); /* $UMEM_DEBUG setting */
84}
85
86const char *
87_umem_logging_init(void)
88{
89	return ("fail,contents"); /* $UMEM_LOGGING setting */
90}
91#endif
92
93/*
94 * Register a module with fmd and finish module initialization.
95 * Returns an integer indicating whether it succeeded (zero) or
96 * failed (non-zero).
97 */
98int
99fmd_hdl_register(fmd_hdl_t *hdl, int version, const fmd_hdl_info_t *mip)
100{
101	(void) version;
102	fmd_module_t *mp = (fmd_module_t *)hdl;
103
104	mp->mod_info = mip;
105	mp->mod_name = mip->fmdi_desc + 4;	/* drop 'ZFS ' prefix */
106	mp->mod_spec = NULL;
107
108	/* bare minimum module stats */
109	(void) strcpy(mp->mod_stats.ms_accepted.fmds_name, "fmd.accepted");
110	(void) strcpy(mp->mod_stats.ms_caseopen.fmds_name, "fmd.caseopen");
111	(void) strcpy(mp->mod_stats.ms_casesolved.fmds_name, "fmd.casesolved");
112	(void) strcpy(mp->mod_stats.ms_caseclosed.fmds_name, "fmd.caseclosed");
113
114	fmd_serd_hash_create(&mp->mod_serds);
115
116	fmd_hdl_debug(hdl, "register module");
117
118	return (0);
119}
120
121void
122fmd_hdl_unregister(fmd_hdl_t *hdl)
123{
124	fmd_module_t *mp = (fmd_module_t *)hdl;
125	fmd_modstat_t *msp = &mp->mod_stats;
126	const fmd_hdl_ops_t *ops = mp->mod_info->fmdi_ops;
127
128	/* dump generic module stats */
129	fmd_hdl_debug(hdl, "%s: %llu", msp->ms_accepted.fmds_name,
130	    msp->ms_accepted.fmds_value.ui64);
131	if (ops->fmdo_close != NULL) {
132		fmd_hdl_debug(hdl, "%s: %llu", msp->ms_caseopen.fmds_name,
133		    msp->ms_caseopen.fmds_value.ui64);
134		fmd_hdl_debug(hdl, "%s: %llu", msp->ms_casesolved.fmds_name,
135		    msp->ms_casesolved.fmds_value.ui64);
136		fmd_hdl_debug(hdl, "%s: %llu", msp->ms_caseclosed.fmds_name,
137		    msp->ms_caseclosed.fmds_value.ui64);
138	}
139
140	/* dump module specific stats */
141	if (mp->mod_ustat != NULL) {
142		int i;
143
144		for (i = 0; i < mp->mod_ustat_cnt; i++) {
145			fmd_hdl_debug(hdl, "%s: %llu",
146			    mp->mod_ustat[i].fmds_name,
147			    mp->mod_ustat[i].fmds_value.ui64);
148		}
149	}
150
151	fmd_serd_hash_destroy(&mp->mod_serds);
152
153	fmd_hdl_debug(hdl, "unregister module");
154}
155
156/*
157 * fmd_hdl_setspecific() is used to associate a data pointer with
158 * the specified handle for the duration of the module's lifetime.
159 * This pointer can be retrieved using fmd_hdl_getspecific().
160 */
161void
162fmd_hdl_setspecific(fmd_hdl_t *hdl, void *spec)
163{
164	fmd_module_t *mp = (fmd_module_t *)hdl;
165
166	mp->mod_spec = spec;
167}
168
169/*
170 * Return the module-specific data pointer previously associated
171 * with the handle using fmd_hdl_setspecific().
172 */
173void *
174fmd_hdl_getspecific(fmd_hdl_t *hdl)
175{
176	fmd_module_t *mp = (fmd_module_t *)hdl;
177
178	return (mp->mod_spec);
179}
180
181void *
182fmd_hdl_alloc(fmd_hdl_t *hdl, size_t size, int flags)
183{
184	(void) hdl;
185	return (umem_alloc(size, flags));
186}
187
188void *
189fmd_hdl_zalloc(fmd_hdl_t *hdl, size_t size, int flags)
190{
191	(void) hdl;
192	return (umem_zalloc(size, flags));
193}
194
195void
196fmd_hdl_free(fmd_hdl_t *hdl, void *data, size_t size)
197{
198	(void) hdl;
199	umem_free(data, size);
200}
201
202/*
203 * Record a module debug message using the specified format.
204 */
205void
206fmd_hdl_debug(fmd_hdl_t *hdl, const char *format, ...)
207{
208	char message[256];
209	va_list vargs;
210	fmd_module_t *mp = (fmd_module_t *)hdl;
211
212	va_start(vargs, format);
213	(void) vsnprintf(message, sizeof (message), format, vargs);
214	va_end(vargs);
215
216	/* prefix message with module name */
217	zed_log_msg(LOG_INFO, "%s: %s", mp->mod_name, message);
218}
219
220/* Property Retrieval */
221
222int32_t
223fmd_prop_get_int32(fmd_hdl_t *hdl, const char *name)
224{
225	(void) hdl;
226
227	/*
228	 * These can be looked up in mp->modinfo->fmdi_props
229	 * For now we just hard code for phase 2. In the
230	 * future, there can be a ZED based override.
231	 */
232	if (strcmp(name, "spare_on_remove") == 0)
233		return (1);
234
235	return (0);
236}
237
238/* FMD Statistics */
239
240fmd_stat_t *
241fmd_stat_create(fmd_hdl_t *hdl, uint_t flags, uint_t nstats, fmd_stat_t *statv)
242{
243	fmd_module_t *mp = (fmd_module_t *)hdl;
244
245	if (flags == FMD_STAT_NOALLOC) {
246		mp->mod_ustat = statv;
247		mp->mod_ustat_cnt = nstats;
248	}
249
250	return (statv);
251}
252
253/* Case Management */
254
255fmd_case_t *
256fmd_case_open(fmd_hdl_t *hdl, void *data)
257{
258	fmd_module_t *mp = (fmd_module_t *)hdl;
259	uuid_t uuid;
260
261	fmd_case_t *cp;
262
263	cp = fmd_hdl_zalloc(hdl, sizeof (fmd_case_t), FMD_SLEEP);
264	cp->ci_mod = hdl;
265	cp->ci_state = FMD_CASE_UNSOLVED;
266	cp->ci_flags = FMD_CF_DIRTY;
267	cp->ci_data = data;
268	cp->ci_bufptr = NULL;
269	cp->ci_bufsiz = 0;
270
271	uuid_generate(uuid);
272	uuid_unparse(uuid, cp->ci_uuid);
273
274	fmd_hdl_debug(hdl, "case opened (%s)", cp->ci_uuid);
275	mp->mod_stats.ms_caseopen.fmds_value.ui64++;
276
277	return (cp);
278}
279
280void
281fmd_case_solve(fmd_hdl_t *hdl, fmd_case_t *cp)
282{
283	fmd_module_t *mp = (fmd_module_t *)hdl;
284
285	/*
286	 * For ZED, the event was already sent from fmd_case_add_suspect()
287	 */
288
289	if (cp->ci_state >= FMD_CASE_SOLVED)
290		fmd_hdl_debug(hdl, "case is already solved or closed");
291
292	cp->ci_state = FMD_CASE_SOLVED;
293
294	fmd_hdl_debug(hdl, "case solved (%s)", cp->ci_uuid);
295	mp->mod_stats.ms_casesolved.fmds_value.ui64++;
296}
297
298void
299fmd_case_close(fmd_hdl_t *hdl, fmd_case_t *cp)
300{
301	fmd_module_t *mp = (fmd_module_t *)hdl;
302	const fmd_hdl_ops_t *ops = mp->mod_info->fmdi_ops;
303
304	fmd_hdl_debug(hdl, "case closed (%s)", cp->ci_uuid);
305
306	if (ops->fmdo_close != NULL)
307		ops->fmdo_close(hdl, cp);
308
309	mp->mod_stats.ms_caseopen.fmds_value.ui64--;
310	mp->mod_stats.ms_caseclosed.fmds_value.ui64++;
311
312	if (cp->ci_bufptr != NULL && cp->ci_bufsiz > 0)
313		fmd_hdl_free(hdl, cp->ci_bufptr, cp->ci_bufsiz);
314
315	fmd_hdl_free(hdl, cp, sizeof (fmd_case_t));
316}
317
318void
319fmd_case_uuresolved(fmd_hdl_t *hdl, const char *uuid)
320{
321	fmd_hdl_debug(hdl, "case resolved by uuid (%s)", uuid);
322}
323
324boolean_t
325fmd_case_solved(fmd_hdl_t *hdl, fmd_case_t *cp)
326{
327	(void) hdl;
328	return (cp->ci_state >= FMD_CASE_SOLVED);
329}
330
331void
332fmd_case_add_ereport(fmd_hdl_t *hdl, fmd_case_t *cp, fmd_event_t *ep)
333{
334	(void) hdl, (void) cp, (void) ep;
335}
336
337static void
338zed_log_fault(nvlist_t *nvl, const char *uuid, const char *code)
339{
340	nvlist_t *rsrc;
341	const char *strval;
342	uint64_t guid;
343	uint8_t byte;
344
345	zed_log_msg(LOG_INFO, "\nzed_fault_event:");
346
347	if (uuid != NULL)
348		zed_log_msg(LOG_INFO, "\t%s: %s", FM_SUSPECT_UUID, uuid);
349	if (nvlist_lookup_string(nvl, FM_CLASS, &strval) == 0)
350		zed_log_msg(LOG_INFO, "\t%s: %s", FM_CLASS, strval);
351	if (code != NULL)
352		zed_log_msg(LOG_INFO, "\t%s: %s", FM_SUSPECT_DIAG_CODE, code);
353	if (nvlist_lookup_uint8(nvl, FM_FAULT_CERTAINTY, &byte) == 0)
354		zed_log_msg(LOG_INFO, "\t%s: %hhu", FM_FAULT_CERTAINTY, byte);
355	if (nvlist_lookup_nvlist(nvl, FM_FAULT_RESOURCE, &rsrc) == 0) {
356		if (nvlist_lookup_string(rsrc, FM_FMRI_SCHEME, &strval) == 0)
357			zed_log_msg(LOG_INFO, "\t%s: %s", FM_FMRI_SCHEME,
358			    strval);
359		if (nvlist_lookup_uint64(rsrc, FM_FMRI_ZFS_POOL, &guid) == 0)
360			zed_log_msg(LOG_INFO, "\t%s: %llu", FM_FMRI_ZFS_POOL,
361			    guid);
362		if (nvlist_lookup_uint64(rsrc, FM_FMRI_ZFS_VDEV, &guid) == 0)
363			zed_log_msg(LOG_INFO, "\t%s: %llu \n", FM_FMRI_ZFS_VDEV,
364			    guid);
365	}
366}
367
368static const char *
369fmd_fault_mkcode(nvlist_t *fault)
370{
371	const char *class;
372	const char *code = "-";
373
374	/*
375	 * Note: message codes come from: openzfs/usr/src/cmd/fm/dicts/ZFS.po
376	 */
377	if (nvlist_lookup_string(fault, FM_CLASS, &class) == 0) {
378		if (strcmp(class, "fault.fs.zfs.vdev.io") == 0)
379			code = "ZFS-8000-FD";
380		else if (strcmp(class, "fault.fs.zfs.vdev.checksum") == 0)
381			code = "ZFS-8000-GH";
382		else if (strcmp(class, "fault.fs.zfs.io_failure_wait") == 0)
383			code = "ZFS-8000-HC";
384		else if (strcmp(class, "fault.fs.zfs.io_failure_continue") == 0)
385			code = "ZFS-8000-JQ";
386		else if (strcmp(class, "fault.fs.zfs.log_replay") == 0)
387			code = "ZFS-8000-K4";
388		else if (strcmp(class, "fault.fs.zfs.pool") == 0)
389			code = "ZFS-8000-CS";
390		else if (strcmp(class, "fault.fs.zfs.device") == 0)
391			code = "ZFS-8000-D3";
392
393	}
394	return (code);
395}
396
397void
398fmd_case_add_suspect(fmd_hdl_t *hdl, fmd_case_t *cp, nvlist_t *fault)
399{
400	nvlist_t *nvl;
401	const char *code = fmd_fault_mkcode(fault);
402	int64_t tod[2];
403	int err = 0;
404
405	/*
406	 * payload derived from fmd_protocol_list()
407	 */
408
409	(void) gettimeofday(&cp->ci_tv, NULL);
410	tod[0] = cp->ci_tv.tv_sec;
411	tod[1] = cp->ci_tv.tv_usec;
412
413	nvl = fmd_nvl_alloc(hdl, FMD_SLEEP);
414
415	err |= nvlist_add_uint8(nvl, FM_VERSION, FM_SUSPECT_VERSION);
416	err |= nvlist_add_string(nvl, FM_CLASS, FM_LIST_SUSPECT_CLASS);
417	err |= nvlist_add_string(nvl, FM_SUSPECT_UUID, cp->ci_uuid);
418	err |= nvlist_add_string(nvl, FM_SUSPECT_DIAG_CODE, code);
419	err |= nvlist_add_int64_array(nvl, FM_SUSPECT_DIAG_TIME, tod, 2);
420	err |= nvlist_add_uint32(nvl, FM_SUSPECT_FAULT_SZ, 1);
421	err |= nvlist_add_nvlist_array(nvl, FM_SUSPECT_FAULT_LIST,
422	    (const nvlist_t **)&fault, 1);
423
424	if (err)
425		zed_log_die("failed to populate nvlist");
426
427	zed_log_fault(fault, cp->ci_uuid, code);
428	zfs_agent_post_event(FM_LIST_SUSPECT_CLASS, NULL, nvl);
429
430	nvlist_free(nvl);
431	nvlist_free(fault);
432}
433
434void
435fmd_case_setspecific(fmd_hdl_t *hdl, fmd_case_t *cp, void *data)
436{
437	(void) hdl;
438	cp->ci_data = data;
439}
440
441void *
442fmd_case_getspecific(fmd_hdl_t *hdl, fmd_case_t *cp)
443{
444	(void) hdl;
445	return (cp->ci_data);
446}
447
448void
449fmd_buf_create(fmd_hdl_t *hdl, fmd_case_t *cp, const char *name, size_t size)
450{
451	assert(strcmp(name, "data") == 0), (void) name;
452	assert(cp->ci_bufptr == NULL);
453	assert(size < (1024 * 1024));
454
455	cp->ci_bufptr = fmd_hdl_alloc(hdl, size, FMD_SLEEP);
456	cp->ci_bufsiz = size;
457}
458
459void
460fmd_buf_read(fmd_hdl_t *hdl, fmd_case_t *cp,
461    const char *name, void *buf, size_t size)
462{
463	(void) hdl;
464	assert(strcmp(name, "data") == 0), (void) name;
465	assert(cp->ci_bufptr != NULL);
466	assert(size <= cp->ci_bufsiz);
467
468	memcpy(buf, cp->ci_bufptr, size);
469}
470
471void
472fmd_buf_write(fmd_hdl_t *hdl, fmd_case_t *cp,
473    const char *name, const void *buf, size_t size)
474{
475	(void) hdl;
476	assert(strcmp(name, "data") == 0), (void) name;
477	assert(cp->ci_bufptr != NULL);
478	assert(cp->ci_bufsiz >= size);
479
480	memcpy(cp->ci_bufptr, buf, size);
481}
482
483/* SERD Engines */
484
485void
486fmd_serd_create(fmd_hdl_t *hdl, const char *name, uint_t n, hrtime_t t)
487{
488	fmd_module_t *mp = (fmd_module_t *)hdl;
489
490	if (fmd_serd_eng_lookup(&mp->mod_serds, name) != NULL) {
491		zed_log_msg(LOG_ERR, "failed to create SERD engine '%s': "
492		    " name already exists", name);
493		return;
494	}
495
496	(void) fmd_serd_eng_insert(&mp->mod_serds, name, n, t);
497}
498
499void
500fmd_serd_destroy(fmd_hdl_t *hdl, const char *name)
501{
502	fmd_module_t *mp = (fmd_module_t *)hdl;
503
504	fmd_serd_eng_delete(&mp->mod_serds, name);
505
506	fmd_hdl_debug(hdl, "serd_destroy %s", name);
507}
508
509int
510fmd_serd_exists(fmd_hdl_t *hdl, const char *name)
511{
512	fmd_module_t *mp = (fmd_module_t *)hdl;
513
514	return (fmd_serd_eng_lookup(&mp->mod_serds, name) != NULL);
515}
516
517int
518fmd_serd_active(fmd_hdl_t *hdl, const char *name)
519{
520	fmd_module_t *mp = (fmd_module_t *)hdl;
521	fmd_serd_eng_t *sgp;
522
523	if ((sgp = fmd_serd_eng_lookup(&mp->mod_serds, name)) == NULL) {
524		zed_log_msg(LOG_ERR, "serd engine '%s' does not exist", name);
525		return (0);
526	}
527	return (fmd_serd_eng_fired(sgp) || !fmd_serd_eng_empty(sgp));
528}
529
530void
531fmd_serd_reset(fmd_hdl_t *hdl, const char *name)
532{
533	fmd_module_t *mp = (fmd_module_t *)hdl;
534	fmd_serd_eng_t *sgp;
535
536	if ((sgp = fmd_serd_eng_lookup(&mp->mod_serds, name)) == NULL) {
537		zed_log_msg(LOG_ERR, "serd engine '%s' does not exist", name);
538	} else {
539		fmd_serd_eng_reset(sgp);
540		fmd_hdl_debug(hdl, "serd_reset %s", name);
541	}
542}
543
544int
545fmd_serd_record(fmd_hdl_t *hdl, const char *name, fmd_event_t *ep)
546{
547	fmd_module_t *mp = (fmd_module_t *)hdl;
548	fmd_serd_eng_t *sgp;
549
550	if ((sgp = fmd_serd_eng_lookup(&mp->mod_serds, name)) == NULL) {
551		zed_log_msg(LOG_ERR, "failed to add record to SERD engine '%s'",
552		    name);
553		return (0);
554	}
555	return (fmd_serd_eng_record(sgp, ep->ev_hrt));
556}
557
558void
559fmd_serd_gc(fmd_hdl_t *hdl)
560{
561	fmd_module_t *mp = (fmd_module_t *)hdl;
562
563	fmd_serd_hash_apply(&mp->mod_serds, fmd_serd_eng_gc, NULL);
564}
565
566/* FMD Timers */
567
568static void
569_timer_notify(union sigval sv)
570{
571	fmd_timer_t *ftp = sv.sival_ptr;
572	fmd_hdl_t *hdl = ftp->ft_hdl;
573	fmd_module_t *mp = (fmd_module_t *)hdl;
574	const fmd_hdl_ops_t *ops = mp->mod_info->fmdi_ops;
575	struct itimerspec its;
576
577	fmd_hdl_debug(hdl, "%s timer fired (%p)", mp->mod_name, ftp->ft_tid);
578
579	/* disarm the timer */
580	memset(&its, 0, sizeof (struct itimerspec));
581	timer_settime(ftp->ft_tid, 0, &its, NULL);
582
583	/* Note that the fmdo_timeout can remove this timer */
584	if (ops->fmdo_timeout != NULL)
585		ops->fmdo_timeout(hdl, ftp, ftp->ft_arg);
586}
587
588/*
589 * Install a new timer which will fire at least delta nanoseconds after the
590 * current time. After the timeout has expired, the module's fmdo_timeout
591 * entry point is called.
592 */
593fmd_timer_t *
594fmd_timer_install(fmd_hdl_t *hdl, void *arg, fmd_event_t *ep, hrtime_t delta)
595{
596	(void) ep;
597	struct sigevent sev;
598	struct itimerspec its;
599	fmd_timer_t *ftp;
600
601	ftp = fmd_hdl_alloc(hdl, sizeof (fmd_timer_t), FMD_SLEEP);
602	ftp->ft_arg = arg;
603	ftp->ft_hdl = hdl;
604
605	its.it_value.tv_sec = delta / 1000000000;
606	its.it_value.tv_nsec = delta % 1000000000;
607	its.it_interval.tv_sec = its.it_value.tv_sec;
608	its.it_interval.tv_nsec = its.it_value.tv_nsec;
609
610	sev.sigev_notify = SIGEV_THREAD;
611	sev.sigev_notify_function = _timer_notify;
612	sev.sigev_notify_attributes = NULL;
613	sev.sigev_value.sival_ptr = ftp;
614	sev.sigev_signo = 0;
615
616	timer_create(CLOCK_REALTIME, &sev, &ftp->ft_tid);
617	timer_settime(ftp->ft_tid, 0, &its, NULL);
618
619	fmd_hdl_debug(hdl, "installing timer for %d secs (%p)",
620	    (int)its.it_value.tv_sec, ftp->ft_tid);
621
622	return (ftp);
623}
624
625void
626fmd_timer_remove(fmd_hdl_t *hdl, fmd_timer_t *ftp)
627{
628	fmd_hdl_debug(hdl, "removing timer (%p)", ftp->ft_tid);
629
630	timer_delete(ftp->ft_tid);
631
632	fmd_hdl_free(hdl, ftp, sizeof (fmd_timer_t));
633}
634
635/* Name-Value Pair Lists */
636
637nvlist_t *
638fmd_nvl_create_fault(fmd_hdl_t *hdl, const char *class, uint8_t certainty,
639    nvlist_t *asru, nvlist_t *fru, nvlist_t *resource)
640{
641	(void) hdl;
642	nvlist_t *nvl;
643	int err = 0;
644
645	if (nvlist_alloc(&nvl, NV_UNIQUE_NAME, 0) != 0)
646		zed_log_die("failed to xalloc fault nvlist");
647
648	err |= nvlist_add_uint8(nvl, FM_VERSION, FM_FAULT_VERSION);
649	err |= nvlist_add_string(nvl, FM_CLASS, class);
650	err |= nvlist_add_uint8(nvl, FM_FAULT_CERTAINTY, certainty);
651
652	if (asru != NULL)
653		err |= nvlist_add_nvlist(nvl, FM_FAULT_ASRU, asru);
654	if (fru != NULL)
655		err |= nvlist_add_nvlist(nvl, FM_FAULT_FRU, fru);
656	if (resource != NULL)
657		err |= nvlist_add_nvlist(nvl, FM_FAULT_RESOURCE, resource);
658
659	if (err)
660		zed_log_die("failed to populate nvlist: %s\n", strerror(err));
661
662	return (nvl);
663}
664
665/*
666 * sourced from fmd_string.c
667 */
668static int
669fmd_strmatch(const char *s, const char *p)
670{
671	char c;
672
673	if (p == NULL)
674		return (0);
675
676	if (s == NULL)
677		s = ""; /* treat NULL string as the empty string */
678
679	do {
680		if ((c = *p++) == '\0')
681			return (*s == '\0');
682
683		if (c == '*') {
684			while (*p == '*')
685				p++; /* consecutive *'s can be collapsed */
686
687			if (*p == '\0')
688				return (1);
689
690			while (*s != '\0') {
691				if (fmd_strmatch(s++, p) != 0)
692					return (1);
693			}
694
695			return (0);
696		}
697	} while (c == *s++);
698
699	return (0);
700}
701
702int
703fmd_nvl_class_match(fmd_hdl_t *hdl, nvlist_t *nvl, const char *pattern)
704{
705	(void) hdl;
706	const char *class;
707
708	return (nvl != NULL &&
709	    nvlist_lookup_string(nvl, FM_CLASS, &class) == 0 &&
710	    fmd_strmatch(class, pattern));
711}
712
713nvlist_t *
714fmd_nvl_alloc(fmd_hdl_t *hdl, int flags)
715{
716	(void) hdl, (void) flags;
717	nvlist_t *nvl = NULL;
718
719	if (nvlist_alloc(&nvl, NV_UNIQUE_NAME, 0) != 0)
720		return (NULL);
721
722	return (nvl);
723}
724
725
726/*
727 * ZED Agent specific APIs
728 */
729
730fmd_hdl_t *
731fmd_module_hdl(const char *name)
732{
733	if (strcmp(name, "zfs-retire") == 0)
734		return ((fmd_hdl_t *)&zfs_retire_module);
735	if (strcmp(name, "zfs-diagnosis") == 0)
736		return ((fmd_hdl_t *)&zfs_diagnosis_module);
737
738	return (NULL);
739}
740
741boolean_t
742fmd_module_initialized(fmd_hdl_t *hdl)
743{
744	fmd_module_t *mp = (fmd_module_t *)hdl;
745
746	return (mp->mod_info != NULL);
747}
748
749/*
750 * fmd_module_recv is called for each event that is received by
751 * the fault manager that has a class that matches one of the
752 * module's subscriptions.
753 */
754void
755fmd_module_recv(fmd_hdl_t *hdl, nvlist_t *nvl, const char *class)
756{
757	fmd_module_t *mp = (fmd_module_t *)hdl;
758	const fmd_hdl_ops_t *ops = mp->mod_info->fmdi_ops;
759	fmd_event_t faux_event = {0};
760	int64_t *tv;
761	uint_t n;
762
763	/*
764	 * Will need to normalized this if we persistently store the case data
765	 */
766	if (nvlist_lookup_int64_array(nvl, FM_EREPORT_TIME, &tv, &n) == 0)
767		faux_event.ev_hrt = tv[0] * NANOSEC + tv[1];
768	else
769		faux_event.ev_hrt = 0;
770
771	ops->fmdo_recv(hdl, &faux_event, nvl, class);
772
773	mp->mod_stats.ms_accepted.fmds_value.ui64++;
774
775	/* TBD - should we initiate fm_module_gc() periodically? */
776}
777