intel_fault.c revision 259512 
1/*-
2 * Copyright (c) 2013 The FreeBSD Foundation
3 * All rights reserved.
4 *
5 * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
6 * under sponsorship from the FreeBSD Foundation.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 *    notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 *    notice, this list of conditions and the following disclaimer in the
15 *    documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD: stable/10/sys/x86/iommu/intel_fault.c 259512 2013-12-17 13:49:35Z kib $");
32
33#include "opt_acpi.h"
34
35#include <sys/param.h>
36#include <sys/bus.h>
37#include <sys/kernel.h>
38#include <sys/malloc.h>
39#include <sys/memdesc.h>
40#include <sys/module.h>
41#include <sys/rman.h>
42#include <sys/taskqueue.h>
43#include <sys/tree.h>
44#include <machine/bus.h>
45#include <contrib/dev/acpica/include/acpi.h>
46#include <contrib/dev/acpica/include/accommon.h>
47#include <dev/acpica/acpivar.h>
48#include <vm/vm.h>
49#include <vm/vm_extern.h>
50#include <vm/vm_kern.h>
51#include <vm/vm_page.h>
52#include <vm/vm_map.h>
53#include <x86/include/busdma_impl.h>
54#include <x86/iommu/intel_reg.h>
55#include <x86/iommu/busdma_dmar.h>
56#include <x86/iommu/intel_dmar.h>
57
58/*
59 * Fault interrupt handling for DMARs.  If advanced fault logging is
60 * not implemented by hardware, the code emulates it.  Fast interrupt
61 * handler flushes the fault registers into circular buffer at
62 * unit->fault_log, and schedules a task.
63 *
64 * The fast handler is used since faults usually come in bursts, and
65 * number of fault log registers is limited, e.g. down to one for 5400
66 * MCH.  We are trying to reduce the latency for clearing the fault
67 * register file.  The task is usually long-running, since printf() is
68 * slow, but this is not problematic because bursts are rare.
69 *
70 * For the same reason, each translation unit task is executed in its
71 * own thread.
72 *
73 * XXXKIB It seems there is no hardware available which implements
74 * advanced fault logging, so the code to handle AFL is not written.
75 */
76
77static int
78dmar_fault_next(struct dmar_unit *unit, int faultp)
79{
80
81	faultp += 2;
82	if (faultp == unit->fault_log_size)
83		faultp = 0;
84	return (faultp);
85}
86
87static void
88dmar_fault_intr_clear(struct dmar_unit *unit, uint32_t fsts)
89{
90	uint32_t clear;
91
92	clear = 0;
93	if ((fsts & DMAR_FSTS_ITE) != 0) {
94		printf("DMAR%d: Invalidation timed out\n", unit->unit);
95		clear |= DMAR_FSTS_ITE;
96	}
97	if ((fsts & DMAR_FSTS_ICE) != 0) {
98		printf("DMAR%d: Invalidation completion error\n",
99		    unit->unit);
100		clear |= DMAR_FSTS_ICE;
101	}
102	if ((fsts & DMAR_FSTS_IQE) != 0) {
103		printf("DMAR%d: Invalidation queue error\n",
104		    unit->unit);
105		clear |= DMAR_FSTS_IQE;
106	}
107	if ((fsts & DMAR_FSTS_APF) != 0) {
108		printf("DMAR%d: Advanced pending fault\n", unit->unit);
109		clear |= DMAR_FSTS_APF;
110	}
111	if ((fsts & DMAR_FSTS_AFO) != 0) {
112		printf("DMAR%d: Advanced fault overflow\n", unit->unit);
113		clear |= DMAR_FSTS_AFO;
114	}
115	if (clear != 0)
116		dmar_write4(unit, DMAR_FSTS_REG, clear);
117}
118
119int
120dmar_fault_intr(void *arg)
121{
122	struct dmar_unit *unit;
123	uint64_t fault_rec[2];
124	uint32_t fsts;
125	int fri, frir, faultp;
126	bool enqueue;
127
128	unit = arg;
129	enqueue = false;
130	fsts = dmar_read4(unit, DMAR_FSTS_REG);
131	dmar_fault_intr_clear(unit, fsts);
132
133	if ((fsts & DMAR_FSTS_PPF) == 0)
134		goto done;
135
136	fri = DMAR_FSTS_FRI(fsts);
137	for (;;) {
138		frir = (DMAR_CAP_FRO(unit->hw_cap) + fri) * 16;
139		fault_rec[1] = dmar_read8(unit, frir + 8);
140		if ((fault_rec[1] & DMAR_FRCD2_F) == 0)
141			break;
142		fault_rec[0] = dmar_read8(unit, frir);
143		dmar_write4(unit, frir + 12, DMAR_FRCD2_F32);
144		DMAR_FAULT_LOCK(unit);
145		faultp = unit->fault_log_head;
146		if (dmar_fault_next(unit, faultp) == unit->fault_log_tail) {
147			/* XXXKIB log overflow */
148		} else {
149			unit->fault_log[faultp] = fault_rec[0];
150			unit->fault_log[faultp + 1] = fault_rec[1];
151			unit->fault_log_head = dmar_fault_next(unit, faultp);
152			enqueue = true;
153		}
154		DMAR_FAULT_UNLOCK(unit);
155		fri += 1;
156		if (fri >= DMAR_CAP_NFR(unit->hw_cap))
157			fri = 0;
158	}
159
160done:
161	/*
162	 * On SandyBridge, due to errata BJ124, IvyBridge errata
163	 * BV100, and Haswell errata HSD40, "Spurious Intel VT-d
164	 * Interrupts May Occur When the PFO Bit is Set".  Handle the
165	 * cases by clearing overflow bit even if no fault is
166	 * reported.
167	 *
168	 * On IvyBridge, errata BV30 states that clearing clear
169	 * DMAR_FRCD2_F bit in the fault register causes spurious
170	 * interrupt.  Do nothing.
171	 *
172	 */
173	if ((fsts & DMAR_FSTS_PFO) != 0) {
174		printf("DMAR%d: Fault Overflow\n", unit->unit);
175		dmar_write4(unit, DMAR_FSTS_REG, DMAR_FSTS_PFO);
176	}
177
178	if (enqueue) {
179		taskqueue_enqueue_fast(unit->fault_taskqueue,
180		    &unit->fault_task);
181	}
182	return (FILTER_HANDLED);
183}
184
185static void
186dmar_fault_task(void *arg, int pending __unused)
187{
188	struct dmar_unit *unit;
189	struct dmar_ctx *ctx;
190	uint64_t fault_rec[2];
191	int sid, bus, slot, func, faultp;
192
193	unit = arg;
194	DMAR_FAULT_LOCK(unit);
195	for (;;) {
196		faultp = unit->fault_log_tail;
197		if (faultp == unit->fault_log_head)
198			break;
199
200		fault_rec[0] = unit->fault_log[faultp];
201		fault_rec[1] = unit->fault_log[faultp + 1];
202		unit->fault_log_tail = dmar_fault_next(unit, faultp);
203		DMAR_FAULT_UNLOCK(unit);
204
205		sid = DMAR_FRCD2_SID(fault_rec[1]);
206		bus = (sid >> 8) & 0xf;
207		slot = (sid >> 3) & 0x1f;
208		func = sid & 0x7;
209		printf("DMAR%d: ", unit->unit);
210		DMAR_LOCK(unit);
211		ctx = dmar_find_ctx_locked(unit, bus, slot, func);
212		if (ctx == NULL) {
213			printf("<unknown dev>:");
214		} else {
215			ctx->flags |= DMAR_CTX_FAULTED;
216			ctx->last_fault_rec[0] = fault_rec[0];
217			ctx->last_fault_rec[1] = fault_rec[1];
218			device_print_prettyname(ctx->ctx_tag.owner);
219		}
220		DMAR_UNLOCK(unit);
221		printf(
222		    "pci%d:%d:%d fault acc %x adt 0x%x reason 0x%x addr %jx\n",
223		    bus, slot, func, DMAR_FRCD2_T(fault_rec[1]),
224		    DMAR_FRCD2_AT(fault_rec[1]), DMAR_FRCD2_FR(fault_rec[1]),
225		    (uintmax_t)fault_rec[0]);
226		DMAR_FAULT_LOCK(unit);
227	}
228	DMAR_FAULT_UNLOCK(unit);
229}
230
231static void
232dmar_clear_faults(struct dmar_unit *unit)
233{
234	uint32_t frec, frir, fsts;
235	int i;
236
237	for (i = 0; i < DMAR_CAP_NFR(unit->hw_cap); i++) {
238		frir = (DMAR_CAP_FRO(unit->hw_cap) + i) * 16;
239		frec = dmar_read4(unit, frir + 12);
240		if ((frec & DMAR_FRCD2_F32) == 0)
241			continue;
242		dmar_write4(unit, frir + 12, DMAR_FRCD2_F32);
243	}
244	fsts = dmar_read4(unit, DMAR_FSTS_REG);
245	dmar_write4(unit, DMAR_FSTS_REG, fsts);
246}
247
248int
249dmar_init_fault_log(struct dmar_unit *unit)
250{
251
252	mtx_init(&unit->fault_lock, "dmarflt", NULL, MTX_SPIN);
253	unit->fault_log_size = 256; /* 128 fault log entries */
254	TUNABLE_INT_FETCH("hw.dmar.fault_log_size", &unit->fault_log_size);
255	if (unit->fault_log_size % 2 != 0)
256		panic("hw.dmar_fault_log_size must be even");
257	unit->fault_log = malloc(sizeof(uint64_t) * unit->fault_log_size,
258	    M_DEVBUF, M_WAITOK | M_ZERO);
259
260	TASK_INIT(&unit->fault_task, 0, dmar_fault_task, unit);
261	unit->fault_taskqueue = taskqueue_create_fast("dmar", M_WAITOK,
262	    taskqueue_thread_enqueue, &unit->fault_taskqueue);
263	taskqueue_start_threads(&unit->fault_taskqueue, 1, PI_AV,
264	    "dmar%d fault taskq", unit->unit);
265
266	DMAR_LOCK(unit);
267	dmar_disable_fault_intr(unit);
268	dmar_clear_faults(unit);
269	dmar_enable_fault_intr(unit);
270	DMAR_UNLOCK(unit);
271
272	return (0);
273}
274
275void
276dmar_fini_fault_log(struct dmar_unit *unit)
277{
278
279	DMAR_LOCK(unit);
280	dmar_disable_fault_intr(unit);
281	DMAR_UNLOCK(unit);
282
283	if (unit->fault_taskqueue == NULL)
284		return;
285
286	taskqueue_drain(unit->fault_taskqueue, &unit->fault_task);
287	taskqueue_free(unit->fault_taskqueue);
288	unit->fault_taskqueue = NULL;
289	mtx_destroy(&unit->fault_lock);
290
291	free(unit->fault_log, M_DEVBUF);
292	unit->fault_log = NULL;
293	unit->fault_log_head = unit->fault_log_tail = 0;
294}
295
296void
297dmar_enable_fault_intr(struct dmar_unit *unit)
298{
299	uint32_t fectl;
300
301	DMAR_ASSERT_LOCKED(unit);
302	fectl = dmar_read4(unit, DMAR_FECTL_REG);
303	fectl &= ~DMAR_FECTL_IM;
304	dmar_write4(unit, DMAR_FECTL_REG, fectl);
305}
306
307void
308dmar_disable_fault_intr(struct dmar_unit *unit)
309{
310	uint32_t fectl;
311
312	DMAR_ASSERT_LOCKED(unit);
313	fectl = dmar_read4(unit, DMAR_FECTL_REG);
314	dmar_write4(unit, DMAR_FECTL_REG, fectl | DMAR_FECTL_IM);
315}
316