11901Swollman/****************************************************************************** 21901Swollman * xen_intr.c 31901Swollman * 41901Swollman * Xen event and interrupt services for x86 HVM guests. 51901Swollman * 61901Swollman * Copyright (c) 2002-2005, K A Fraser 71901Swollman * Copyright (c) 2005, Intel Corporation <xiaofeng.ling@intel.com> 88870Srgrimes * Copyright (c) 2012, Spectra Logic Corporation 91901Swollman * Copyright �� 2021-2023, Elliott Mitchell 101901Swollman * 111901Swollman * This file may be distributed separately from the Linux kernel, or 128870Srgrimes * incorporated into other software packages, subject to the following license: 131901Swollman * 141901Swollman * Permission is hereby granted, free of charge, to any person obtaining a copy 151901Swollman * of this source file (the "Software"), to deal in the Software without 168870Srgrimes * restriction, including without limitation the rights to use, copy, modify, 171901Swollman * merge, publish, distribute, sublicense, and/or sell copies of the Software, 181901Swollman * and to permit persons to whom the Software is furnished to do so, subject to 191901Swollman * the following conditions: 208870Srgrimes * 211901Swollman * The above copyright notice and this permission notice shall be included in 221901Swollman * all copies or substantial portions of the Software. 231901Swollman * 248870Srgrimes * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 251901Swollman * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 261901Swollman * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 271901Swollman * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 281901Swollman * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 291901Swollman * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 301901Swollman * IN THE SOFTWARE. 311901Swollman */ 321901Swollman 3350476Speter#include <sys/cdefs.h> 341901Swollman#include "opt_ddb.h" 351901Swollman 361901Swollman#include <sys/param.h> 371901Swollman#include <sys/systm.h> 381901Swollman#include <sys/bus.h> 391901Swollman#include <sys/kernel.h> 401901Swollman#include <sys/limits.h> 411901Swollman#include <sys/lock.h> 421901Swollman#include <sys/mutex.h> 431901Swollman#include <sys/interrupt.h> 441901Swollman#include <sys/pcpu.h> 451901Swollman#include <sys/proc.h> 461901Swollman#include <sys/smp.h> 471901Swollman#include <sys/refcount.h> 481901Swollman 491901Swollman#include <vm/vm.h> 5021081Speter#include <vm/pmap.h> 5111666Sphk 521901Swollman#include <machine/smp.h> 5311666Sphk#include <machine/stdarg.h> 541901Swollman 551901Swollman#include <xen/xen-os.h> 561901Swollman#include <xen/hypervisor.h> 571901Swollman#include <xen/xen_intr.h> 581901Swollman#include <xen/evtchn/evtchnvar.h> 591901Swollman 601901Swollman#include <machine/xen/arch-intr.h> 611901Swollman 621901Swollman#ifdef DDB 631901Swollman#include <ddb/ddb.h> 641901Swollman#endif 651901Swollman 661901Swollman/** 671901Swollman * Per-cpu event channel processing state. 681901Swollman */ 691901Swollmanstruct xen_intr_pcpu_data { 701901Swollman /** 711901Swollman * The last event channel bitmap section (level one bit) processed. 721901Swollman * This is used to ensure we scan all ports before 731901Swollman * servicing an already servied port again. 741901Swollman */ 751901Swollman u_int last_processed_l1i; 761901Swollman 771901Swollman /** 781901Swollman * The last event channel processed within the event channel 791901Swollman * bitmap being scanned. 801901Swollman */ 811901Swollman u_int last_processed_l2i; 821901Swollman 831901Swollman /** 841901Swollman * A bitmap of ports that can be serviced from this CPU. 851901Swollman * A set bit means interrupt handling is enabled. 861901Swollman */ 871901Swollman xen_ulong_t evtchn_enabled[sizeof(xen_ulong_t) * 8]; 881901Swollman}; 891901Swollman 901901Swollman/* 911901Swollman * Start the scan at port 0 by initializing the last scanned 921901Swollman * location as the highest numbered event channel port. 931901Swollman */ 941901SwollmanDPCPU_DEFINE_STATIC(struct xen_intr_pcpu_data, xen_intr_pcpu) = { 951901Swollman .last_processed_l1i = LONG_BIT - 1, 961901Swollman .last_processed_l2i = LONG_BIT - 1 971901Swollman}; 981901Swollman 991901SwollmanDPCPU_DECLARE(struct vcpu_info *, vcpu_info); 10021081Speter 10156698Sjasone#define INVALID_EVTCHN (~(evtchn_port_t)0) /* Invalid event channel */ 1021901Swollman#define is_valid_evtchn(x) ((uintmax_t)(x) < NR_EVENT_CHANNELS) 1031901Swollman 1041901Swollman/* 1051901Swollman * Lock for interrupt core data. 1061901Swollman * 1071901Swollman * Modifying xen_intr_port_to_isrc[], or isrc->xi_port (implies the former) 1081901Swollman * requires this lock be held. Any time this lock is not held, the condition 1091901Swollman * `!xen_intr_port_to_isrc[i] || (xen_intr_port_to_isrc[i]->ix_port == i)` 1101901Swollman * MUST be true for all values of i which are valid indicies of the array. 1111901Swollman * 1121901Swollman * Acquire/release operations for isrc->xi_refcount require this lock be held. 1131901Swollman */ 1141901Swollmanstatic struct mtx xen_intr_isrc_lock; 1151901Swollmanstatic struct xenisrc *xen_intr_port_to_isrc[NR_EVENT_CHANNELS]; 1161901Swollman 1171901Swollman/*------------------------- Private Functions --------------------------------*/ 1181901Swollman 1191901Swollman/** 1201901Swollman * Retrieve a handle for a Xen interrupt source. 1211901Swollman * 1221901Swollman * \param isrc A valid Xen interrupt source structure. 1231901Swollman * 1241901Swollman * \returns A handle suitable for use with xen_intr_isrc_from_handle() 1251901Swollman * to retrieve the original Xen interrupt source structure. 1261901Swollman */ 1271901Swollman 1281901Swollmanstatic inline xen_intr_handle_t 1291901Swollmanxen_intr_handle_from_isrc(struct xenisrc *isrc) 1301901Swollman{ 1311901Swollman return (isrc); 1321901Swollman} 1331901Swollman 1341901Swollman/** 1351901Swollman * Lookup a Xen interrupt source object given an interrupt binding handle. 1361901Swollman * 1371901Swollman * \param handle A handle initialized by a previous call to 1381901Swollman * xen_intr_bind_isrc(). 1391901Swollman * 1401901Swollman * \returns A pointer to the Xen interrupt source object associated 1411901Swollman * with the given interrupt handle. NULL if no association 1421901Swollman * currently exists. 1431901Swollman */ 1441901Swollmanstatic inline struct xenisrc * 1451901Swollmanxen_intr_isrc_from_handle(xen_intr_handle_t handle) 1461901Swollman{ 1471901Swollman return ((struct xenisrc *)handle); 1481901Swollman} 1491901Swollman 1501901Swollman/** 1511901Swollman * Disable signal delivery for an event channel port on the 1521901Swollman * specified CPU. 1531901Swollman * 1541901Swollman * \param port The event channel port to mask. 1551901Swollman * 1561901Swollman * This API is used to manage the port<=>CPU binding of event 1571901Swollman * channel handlers. 1581901Swollman * 1591901Swollman * \note This operation does not preclude reception of an event 1601901Swollman * for this event channel on another CPU. To mask the 1618870Srgrimes * event channel globally, use evtchn_mask(). 1621901Swollman */ 1631901Swollmanstatic inline void 1641901Swollmanevtchn_cpu_mask_port(u_int cpu, evtchn_port_t port) 1651901Swollman{ 1661901Swollman struct xen_intr_pcpu_data *pcpu; 1671901Swollman 1681901Swollman pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu); 1691901Swollman KASSERT(is_valid_evtchn(port), ("Invalid event channel port")); 1701901Swollman xen_clear_bit(port, pcpu->evtchn_enabled); 1711901Swollman} 17221081Speter 1731901Swollman/** 17414659Sguido * Enable signal delivery for an event channel port on the 17521081Speter * specified CPU. 17621081Speter * 1771901Swollman * \param port The event channel port to unmask. 1781901Swollman * 1791901Swollman * This API is used to manage the port<=>CPU binding of event 1801901Swollman * channel handlers. 1811901Swollman * 1821901Swollman * \note This operation does not guarantee that event delivery 1831901Swollman * is enabled for this event channel port. The port must 1841901Swollman * also be globally enabled. See evtchn_unmask(). 1851901Swollman */ 1861901Swollmanstatic inline void 1871901Swollmanevtchn_cpu_unmask_port(u_int cpu, evtchn_port_t port) 1881901Swollman{ 1891901Swollman struct xen_intr_pcpu_data *pcpu; 1901901Swollman 1911901Swollman pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu); 1921901Swollman KASSERT(is_valid_evtchn(port), ("Invalid event channel port")); 1931901Swollman xen_set_bit(port, pcpu->evtchn_enabled); 1941901Swollman} 1951901Swollman 1961901Swollman/** 1971901Swollman * Attempt to free an active Xen interrupt source object. 1981901Swollman * 1991901Swollman * \param isrc The interrupt source object to release. 2001901Swollman * 2011901Swollman * \returns EBUSY if the source is still in use, otherwise 0. 20214659Sguido */ 2031901Swollmanstatic int 2041901Swollmanxen_intr_release_isrc(struct xenisrc *isrc) 2051901Swollman{ 20614659Sguido 2071901Swollman mtx_lock(&xen_intr_isrc_lock); 2081901Swollman if (is_valid_evtchn(isrc->xi_port)) { 2091901Swollman evtchn_mask_port(isrc->xi_port); 21014659Sguido evtchn_clear_port(isrc->xi_port); 2111901Swollman 2121901Swollman /* Rebind port to CPU 0. */ 21314659Sguido evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port); 21414659Sguido evtchn_cpu_unmask_port(0, isrc->xi_port); 21514659Sguido 21614659Sguido if (isrc->xi_close != 0) { 21714659Sguido struct evtchn_close close = { .port = isrc->xi_port }; 21814659Sguido 21914659Sguido if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) 2201901Swollman panic("EVTCHNOP_close failed"); 2211901Swollman } 2221901Swollman 2231901Swollman xen_intr_port_to_isrc[isrc->xi_port] = NULL; 2241901Swollman } 2251901Swollman /* not reachable from xen_intr_port_to_isrc[], unlock */ 2261901Swollman mtx_unlock(&xen_intr_isrc_lock); 2278870Srgrimes 2281901Swollman xen_arch_intr_release(isrc); 2291901Swollman return (0); 2301901Swollman} 2311901Swollman 2321901Swollman/** 2331901Swollman * Associate an interrupt handler with an already allocated local Xen 2341901Swollman * event channel port. 2351901Swollman * 2361901Swollman * \param isrcp The returned Xen interrupt object associated with 2371901Swollman * the specified local port. 2381901Swollman * \param local_port The event channel to bind. 2391901Swollman * \param type The event channel type of local_port. 2401901Swollman * \param intr_owner The device making this bind request. 2411901Swollman * \param filter An interrupt filter handler. Specify NULL 2421901Swollman * to always dispatch to the ithread handler. 2431901Swollman * \param handler An interrupt ithread handler. Optional (can 2441901Swollman * specify NULL) if all necessary event actions 24521081Speter * are performed by filter. 2461901Swollman * \param arg Argument to present to both filter and handler. 2471901Swollman * \param irqflags Interrupt handler flags. See sys/bus.h. 2481901Swollman * \param handlep Pointer to an opaque handle used to manage this 2491901Swollman * registration. 2501901Swollman * 2511901Swollman * \returns 0 on success, otherwise an errno. 2521901Swollman */ 2531901Swollmanstatic int 2541901Swollmanxen_intr_bind_isrc(struct xenisrc **isrcp, evtchn_port_t local_port, 25521081Speter enum evtchn_type type, const char *intr_owner, driver_filter_t filter, 2561901Swollman driver_intr_t handler, void *arg, enum intr_type flags, 25721081Speter xen_intr_handle_t *const port_handlep) 2581901Swollman{ 25921081Speter struct xenisrc *isrc; 26021081Speter int error; 26121081Speter 2621901Swollman *isrcp = NULL; 2631901Swollman if (port_handlep == NULL) { 2641901Swollman printf("%s: %s: Bad event handle\n", intr_owner, __func__); 2651901Swollman return (EINVAL); 2661901Swollman } 2671901Swollman *port_handlep = NULL; 2681901Swollman 2691901Swollman isrc = xen_arch_intr_alloc(); 2701901Swollman if (isrc == NULL) 2711901Swollman return (ENOSPC); 2721901Swollman 2731901Swollman isrc->xi_cookie = NULL; 2741901Swollman isrc->xi_type = type; 2751901Swollman isrc->xi_port = local_port; 2761901Swollman isrc->xi_close = false; 2771901Swollman isrc->xi_cpu = 0; 27821081Speter refcount_init(&isrc->xi_refcount, 1); 27921081Speter mtx_lock(&xen_intr_isrc_lock); 28021081Speter xen_intr_port_to_isrc[isrc->xi_port] = isrc; 28121081Speter mtx_unlock(&xen_intr_isrc_lock); 28221081Speter 28321081Speter#ifdef SMP 28421081Speter if (type == EVTCHN_TYPE_PORT) { 28521081Speter /* 28621081Speter * By default all interrupts are assigned to vCPU#0 28721081Speter * unless specified otherwise, so shuffle them to balance 28821081Speter * the interrupt load. 28921081Speter */ 29021081Speter xen_intr_assign_cpu(isrc, xen_arch_intr_next_cpu(isrc)); 2911901Swollman } 29221081Speter#endif 29317540Speter 2941901Swollman /* 2951901Swollman * If a filter or handler function is provided, add it to the event. 2961901Swollman * Otherwise the event channel is left masked and without a handler, 2971901Swollman * the caller is in charge of setting that up. 29821081Speter */ 2991901Swollman if (filter != NULL || handler != NULL) { 3001901Swollman error = xen_intr_add_handler(intr_owner, filter, handler, arg, 3011901Swollman flags, xen_intr_handle_from_isrc(isrc)); 3021901Swollman if (error != 0) { 3031901Swollman xen_intr_release_isrc(isrc); 3041901Swollman return (error); 3051901Swollman } 3061901Swollman } 3071901Swollman 3081901Swollman *isrcp = isrc; 3091901Swollman /* Assign the opaque handler */ 3101901Swollman *port_handlep = xen_intr_handle_from_isrc(isrc); 3111901Swollman return (0); 3121901Swollman} 3131901Swollman 3141901Swollman/** 3151901Swollman * Determine the event channel ports at the given section of the 3161901Swollman * event port bitmap which have pending events for the given cpu. 3171901Swollman * 3181901Swollman * \param pcpu The Xen interrupt pcpu data for the cpu being queried. 3191901Swollman * \param sh The Xen shared info area. 3201901Swollman * \param idx The index of the section of the event channel bitmap to 3211901Swollman * inspect. 3221901Swollman * 3231901Swollman * \returns A u_long with bits set for every event channel with pending 3241901Swollman * events. 32521081Speter */ 32621081Speterstatic inline u_long 32721081Speterxen_intr_active_ports(const struct xen_intr_pcpu_data *const pcpu, 32821081Speter const u_int idx) 32921081Speter{ 33021081Speter volatile const shared_info_t *const sh = HYPERVISOR_shared_info; 3311901Swollman 3321901Swollman CTASSERT(sizeof(sh->evtchn_mask[0]) == sizeof(sh->evtchn_pending[0])); 3331901Swollman CTASSERT(sizeof(sh->evtchn_mask[0]) == sizeof(pcpu->evtchn_enabled[0])); 3341901Swollman CTASSERT(sizeof(sh->evtchn_mask) == sizeof(sh->evtchn_pending)); 3351901Swollman CTASSERT(sizeof(sh->evtchn_mask) == sizeof(pcpu->evtchn_enabled)); 3361901Swollman return (sh->evtchn_pending[idx] 3371901Swollman & ~sh->evtchn_mask[idx] 3381901Swollman & pcpu->evtchn_enabled[idx]); 3391901Swollman} 3401901Swollman 3411901Swollman/** 3421901Swollman * Interrupt handler for processing all Xen event channel events. 3431901Swollman * 3441901Swollman * \param unused 3451901Swollman */ 3461901Swollmanint 3471901Swollmanxen_intr_handle_upcall(void *unused __unused) 3481901Swollman{ 3491901Swollman struct trapframe *trap_frame = curthread->td_intr_frame; 35021081Speter u_int l1i, l2i, port, cpu __diagused; 35121081Speter u_long masked_l1, masked_l2; 35221081Speter struct xenisrc *isrc; 35321081Speter vcpu_info_t *v; 35421081Speter struct xen_intr_pcpu_data *pc; 3551901Swollman u_long l1, l2; 3561901Swollman 3571901Swollman /* 3581901Swollman * The upcall handler is an interrupt handler itself (that calls other 3591901Swollman * interrupt handlers), hence the caller has the responsibility to 3601901Swollman * increase td_intr_nesting_level ahead of dispatching the upcall 3611901Swollman * handler. 3621901Swollman */ 3631901Swollman KASSERT(curthread->td_intr_nesting_level > 0, 3641901Swollman ("Unexpected thread context")); 3651901Swollman 3661901Swollman /* We must remain on the same vCPU during this function */ 3671901Swollman CRITICAL_ASSERT(curthread); 3681901Swollman 3691901Swollman cpu = PCPU_GET(cpuid); 3701901Swollman pc = DPCPU_PTR(xen_intr_pcpu); 3711901Swollman v = DPCPU_GET(vcpu_info); 3721901Swollman 3731901Swollman if (!xen_has_percpu_evtchn()) { 3741901Swollman KASSERT((cpu == 0), ("Fired PCI event callback on wrong CPU")); 3751901Swollman } 3761901Swollman 3771901Swollman v->evtchn_upcall_pending = 0; 3781901Swollman/* No need for a barrier on x86 -- XCHG is a barrier on x86. */ 37921081Speter#if !defined(__amd64__) && !defined(__i386__) 3801901Swollman /* Clear master flag /before/ clearing selector flag. */ 3811901Swollman wmb(); 3821901Swollman#endif 3831901Swollman l1 = atomic_readandclear_xen_ulong(&v->evtchn_pending_sel); 3841901Swollman 3851901Swollman l1i = pc->last_processed_l1i; 3861901Swollman l2i = pc->last_processed_l2i; 3871901Swollman 3881901Swollman while (l1 != 0) { 3891901Swollman l1i = (l1i + 1) % LONG_BIT; 3901901Swollman masked_l1 = l1 & ((~0UL) << l1i); 3911901Swollman 3921901Swollman if (masked_l1 == 0) { 3931901Swollman /* 3941901Swollman * if we masked out all events, wrap around 3951901Swollman * to the beginning. 3961901Swollman */ 3971901Swollman l1i = LONG_BIT - 1; 3981901Swollman l2i = LONG_BIT - 1; 3991901Swollman continue; 4001901Swollman } 4011901Swollman l1i = ffsl(masked_l1) - 1; 4021901Swollman 4031901Swollman do { 4041901Swollman l2 = xen_intr_active_ports(pc, l1i); 4051901Swollman 4061901Swollman l2i = (l2i + 1) % LONG_BIT; 4071901Swollman masked_l2 = l2 & ((~0UL) << l2i); 40821081Speter 40921081Speter if (masked_l2 == 0) { 41021081Speter /* if we masked out all events, move on */ 41156698Sjasone l2i = LONG_BIT - 1; 4121901Swollman break; 4131901Swollman } 4141901Swollman l2i = ffsl(masked_l2) - 1; 4151901Swollman 416 /* process port */ 417 port = (l1i * LONG_BIT) + l2i; 418 evtchn_clear_port(port); 419 420 isrc = xen_intr_port_to_isrc[port]; 421 if (__predict_false(isrc == NULL)) 422 continue; 423 424 /* Make sure we are firing on the right vCPU */ 425 KASSERT((isrc->xi_cpu == PCPU_GET(cpuid)), 426 ("Received unexpected event on vCPU#%u, event bound to vCPU#%u", 427 PCPU_GET(cpuid), isrc->xi_cpu)); 428 429 /* 430 * Reduce interrupt nesting level ahead of calling the 431 * per-arch interrupt dispatch helper. This is 432 * required because the per-arch dispatcher will also 433 * increase td_intr_nesting_level, and then handlers 434 * would wrongly see td_intr_nesting_level = 2 when 435 * there's no nesting at all. 436 */ 437 curthread->td_intr_nesting_level--; 438 xen_arch_intr_execute_handlers(isrc, trap_frame); 439 curthread->td_intr_nesting_level++; 440 441 /* 442 * If this is the final port processed, 443 * we'll pick up here+1 next time. 444 */ 445 pc->last_processed_l1i = l1i; 446 pc->last_processed_l2i = l2i; 447 448 } while (l2i != LONG_BIT - 1); 449 450 l2 = xen_intr_active_ports(pc, l1i); 451 if (l2 == 0) { 452 /* 453 * We handled all ports, so we can clear the 454 * selector bit. 455 */ 456 l1 &= ~(1UL << l1i); 457 } 458 } 459 460 return (FILTER_HANDLED); 461} 462 463static int 464xen_intr_init(void *dummy __unused) 465{ 466 shared_info_t *s = HYPERVISOR_shared_info; 467 struct xen_intr_pcpu_data *pcpu; 468 int i; 469 470 if (!xen_domain()) 471 return (0); 472 473 _Static_assert(is_valid_evtchn(0), 474 "is_valid_evtchn(0) fails (unused by Xen, but valid by interface"); 475 _Static_assert(is_valid_evtchn(NR_EVENT_CHANNELS - 1), 476 "is_valid_evtchn(max) fails (is a valid channel)"); 477 _Static_assert(!is_valid_evtchn(NR_EVENT_CHANNELS), 478 "is_valid_evtchn(>max) fails (NOT a valid channel)"); 479 _Static_assert(!is_valid_evtchn(~(evtchn_port_t)0), 480 "is_valid_evtchn(maxint) fails (overflow?)"); 481 _Static_assert(!is_valid_evtchn(INVALID_EVTCHN), 482 "is_valid_evtchn(INVALID_EVTCHN) fails (must be invalid!)"); 483 _Static_assert(!is_valid_evtchn(-1), 484 "is_valid_evtchn(-1) fails (negative are invalid)"); 485 486 mtx_init(&xen_intr_isrc_lock, "xen-irq-lock", NULL, MTX_DEF); 487 488 /* 489 * Set the per-cpu mask of CPU#0 to enable all, since by default all 490 * event channels are bound to CPU#0. 491 */ 492 CPU_FOREACH(i) { 493 pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu); 494 memset(pcpu->evtchn_enabled, i == 0 ? ~0 : 0, 495 sizeof(pcpu->evtchn_enabled)); 496 } 497 498 for (i = 0; i < nitems(s->evtchn_mask); i++) 499 atomic_store_rel_xen_ulong(&s->evtchn_mask[i], ~0); 500 501 xen_arch_intr_init(); 502 503 if (bootverbose) 504 printf("Xen interrupt system initialized\n"); 505 506 return (0); 507} 508SYSINIT(xen_intr_init, SI_SUB_INTR, SI_ORDER_SECOND, xen_intr_init, NULL); 509 510/*--------------------------- Common PIC Functions ---------------------------*/ 511 512static void 513xen_rebind_ipi(struct xenisrc *isrc) 514{ 515#ifdef SMP 516 u_int cpu = isrc->xi_cpu; 517 u_int vcpu_id = XEN_CPUID_TO_VCPUID(cpu); 518 int error; 519 struct evtchn_bind_ipi bind_ipi = { .vcpu = vcpu_id }; 520 521 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, 522 &bind_ipi); 523 if (error != 0) 524 panic("unable to rebind xen IPI: %d", error); 525 526 isrc->xi_port = bind_ipi.port; 527#else 528 panic("Resume IPI event channel on UP"); 529#endif 530} 531 532static void 533xen_rebind_virq(struct xenisrc *isrc) 534{ 535 u_int cpu = isrc->xi_cpu; 536 u_int vcpu_id = XEN_CPUID_TO_VCPUID(cpu); 537 int error; 538 struct evtchn_bind_virq bind_virq = { .virq = isrc->xi_virq, 539 .vcpu = vcpu_id }; 540 541 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, 542 &bind_virq); 543 if (error != 0) 544 panic("unable to rebind xen VIRQ#%u: %d", isrc->xi_virq, error); 545 546 isrc->xi_port = bind_virq.port; 547} 548 549static struct xenisrc * 550xen_intr_rebind_isrc(struct xenisrc *isrc) 551{ 552#ifdef SMP 553 u_int cpu = isrc->xi_cpu; 554 int error; 555#endif 556 struct xenisrc *prev; 557 558 switch (isrc->xi_type) { 559 case EVTCHN_TYPE_IPI: 560 xen_rebind_ipi(isrc); 561 break; 562 case EVTCHN_TYPE_VIRQ: 563 xen_rebind_virq(isrc); 564 break; 565 default: 566 return (NULL); 567 } 568 569 prev = xen_intr_port_to_isrc[isrc->xi_port]; 570 xen_intr_port_to_isrc[isrc->xi_port] = isrc; 571 572#ifdef SMP 573 isrc->xi_cpu = 0; 574 error = xen_intr_assign_cpu(isrc, cpu); 575 if (error) 576 panic("%s(): unable to rebind Xen channel %u to vCPU%u: %d", 577 __func__, isrc->xi_port, cpu, error); 578#endif 579 580 evtchn_unmask_port(isrc->xi_port); 581 582 return (prev); 583} 584 585/** 586 * Return this PIC to service after being suspended. 587 */ 588void 589xen_intr_resume(void) 590{ 591 shared_info_t *s = HYPERVISOR_shared_info; 592 u_int isrc_idx; 593 int i; 594 595 /* Reset the per-CPU masks */ 596 CPU_FOREACH(i) { 597 struct xen_intr_pcpu_data *pcpu; 598 599 pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu); 600 memset(pcpu->evtchn_enabled, i == 0 ? ~0 : 0, 601 sizeof(pcpu->evtchn_enabled)); 602 } 603 604 /* Mask all event channels. */ 605 for (i = 0; i < nitems(s->evtchn_mask); i++) 606 atomic_store_rel_xen_ulong(&s->evtchn_mask[i], ~0); 607 608 /* Clear existing port mappings */ 609 for (isrc_idx = 0; isrc_idx < NR_EVENT_CHANNELS; ++isrc_idx) 610 if (xen_intr_port_to_isrc[isrc_idx] != NULL) 611 xen_intr_port_to_isrc[isrc_idx]->xi_port = 612 INVALID_EVTCHN; 613 614 /* Remap in-use isrcs, using xen_intr_port_to_isrc as listing */ 615 for (isrc_idx = 0; isrc_idx < NR_EVENT_CHANNELS; ++isrc_idx) { 616 struct xenisrc *cur = xen_intr_port_to_isrc[isrc_idx]; 617 618 /* empty or entry already taken care of */ 619 if (cur == NULL || cur->xi_port == isrc_idx) 620 continue; 621 622 xen_intr_port_to_isrc[isrc_idx] = NULL; 623 624 do { 625 KASSERT(!is_valid_evtchn(cur->xi_port), 626 ("%s(): Multiple channels on single intr?", 627 __func__)); 628 629 cur = xen_intr_rebind_isrc(cur); 630 } while (cur != NULL); 631 } 632} 633 634/** 635 * Disable a Xen interrupt source. 636 * 637 * \param isrc The interrupt source to disable. 638 */ 639void 640xen_intr_disable_intr(struct xenisrc *isrc) 641{ 642 643 if (__predict_true(is_valid_evtchn(isrc->xi_port))) 644 evtchn_mask_port(isrc->xi_port); 645} 646 647/** 648 * Configure CPU affinity for interrupt source event delivery. 649 * 650 * \param isrc The interrupt source to configure. 651 * \param to_cpu The id of the CPU for handling future events. 652 * 653 * \returns 0 if successful, otherwise an errno. 654 */ 655int 656xen_intr_assign_cpu(struct xenisrc *isrc, u_int to_cpu) 657{ 658#ifdef SMP 659 struct evtchn_bind_vcpu bind_vcpu; 660 u_int vcpu_id = XEN_CPUID_TO_VCPUID(to_cpu); 661 int error, masked; 662 663 if (!xen_has_percpu_evtchn()) 664 return (EOPNOTSUPP); 665 666 mtx_lock(&xen_intr_isrc_lock); 667 if (!is_valid_evtchn(isrc->xi_port)) { 668 mtx_unlock(&xen_intr_isrc_lock); 669 return (EINVAL); 670 } 671 672 /* 673 * Mask the event channel while binding it to prevent interrupt 674 * delivery with an inconsistent state in isrc->xi_cpu. 675 */ 676 masked = evtchn_test_and_set_mask(isrc->xi_port); 677 if ((isrc->xi_type == EVTCHN_TYPE_VIRQ) || 678 (isrc->xi_type == EVTCHN_TYPE_IPI)) { 679 /* 680 * Virtual IRQs are associated with a cpu by 681 * the Hypervisor at evtchn_bind_virq time, so 682 * all we need to do is update the per-CPU masks. 683 */ 684 evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port); 685 isrc->xi_cpu = to_cpu; 686 evtchn_cpu_unmask_port(isrc->xi_cpu, isrc->xi_port); 687 goto out; 688 } 689 690 bind_vcpu.port = isrc->xi_port; 691 bind_vcpu.vcpu = vcpu_id; 692 693 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu); 694 if (isrc->xi_cpu != to_cpu) { 695 if (error == 0) { 696 /* Commit to new binding by removing the old one. */ 697 evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port); 698 isrc->xi_cpu = to_cpu; 699 evtchn_cpu_unmask_port(isrc->xi_cpu, isrc->xi_port); 700 } 701 } 702 703out: 704 if (masked == 0) 705 evtchn_unmask_port(isrc->xi_port); 706 mtx_unlock(&xen_intr_isrc_lock); 707 return (0); 708#else 709 return (EOPNOTSUPP); 710#endif 711} 712 713/*------------------- Virtual Interrupt Source PIC Functions -----------------*/ 714/* 715 * Mask a level triggered interrupt source. 716 * 717 * \param isrc The interrupt source to mask (if necessary). 718 */ 719void 720xen_intr_disable_source(struct xenisrc *isrc) 721{ 722 723 /* 724 * NB: checking if the event channel is already masked is 725 * needed because the event channel user-space device 726 * masks event channels on its filter as part of its 727 * normal operation, and those shouldn't be automatically 728 * unmasked by the generic interrupt code. The event channel 729 * device will unmask them when needed. 730 */ 731 if (__predict_true(is_valid_evtchn(isrc->xi_port))) 732 isrc->xi_masked = !!evtchn_test_and_set_mask(isrc->xi_port); 733} 734 735/* 736 * Unmask a level triggered interrupt source. 737 * 738 * \param isrc The interrupt source to unmask (if necessary). 739 */ 740void 741xen_intr_enable_source(struct xenisrc *isrc) 742{ 743 744 if (isrc->xi_masked == 0) 745 evtchn_unmask_port(isrc->xi_port); 746} 747 748/* 749 * Enable and unmask the interrupt source. 750 * 751 * \param isrc The interrupt source to enable. 752 */ 753void 754xen_intr_enable_intr(struct xenisrc *isrc) 755{ 756 757 evtchn_unmask_port(isrc->xi_port); 758} 759 760/*--------------------------- Public Functions -------------------------------*/ 761/*------- API comments for these methods can be found in xen/xenintr.h -------*/ 762int 763xen_intr_bind_local_port(device_t dev, evtchn_port_t local_port, 764 driver_filter_t filter, driver_intr_t handler, void *arg, 765 enum intr_type flags, xen_intr_handle_t *port_handlep) 766{ 767 struct xenisrc *isrc; 768 int error; 769 770 error = xen_intr_bind_isrc(&isrc, local_port, EVTCHN_TYPE_PORT, 771 device_get_nameunit(dev), filter, handler, arg, flags, 772 port_handlep); 773 if (error != 0) 774 return (error); 775 776 /* 777 * The Event Channel API didn't open this port, so it is not 778 * responsible for closing it automatically on unbind. 779 */ 780 isrc->xi_close = 0; 781 return (0); 782} 783 784int 785xen_intr_alloc_and_bind_local_port(device_t dev, u_int remote_domain, 786 driver_filter_t filter, driver_intr_t handler, void *arg, 787 enum intr_type flags, xen_intr_handle_t *port_handlep) 788{ 789 struct xenisrc *isrc; 790 struct evtchn_alloc_unbound alloc_unbound; 791 int error; 792 793 alloc_unbound.dom = DOMID_SELF; 794 alloc_unbound.remote_dom = remote_domain; 795 error = HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound, 796 &alloc_unbound); 797 if (error != 0) { 798 /* 799 * XXX Trap Hypercall error code Linuxisms in 800 * the HYPERCALL layer. 801 */ 802 return (-error); 803 } 804 805 error = xen_intr_bind_isrc(&isrc, alloc_unbound.port, EVTCHN_TYPE_PORT, 806 device_get_nameunit(dev), filter, handler, arg, flags, 807 port_handlep); 808 if (error != 0) { 809 evtchn_close_t close = { .port = alloc_unbound.port }; 810 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) 811 panic("EVTCHNOP_close failed"); 812 return (error); 813 } 814 815 isrc->xi_close = 1; 816 return (0); 817} 818 819int 820xen_intr_bind_remote_port(device_t dev, u_int remote_domain, 821 u_int remote_port, driver_filter_t filter, driver_intr_t handler, 822 void *arg, enum intr_type flags, xen_intr_handle_t *port_handlep) 823{ 824 struct xenisrc *isrc; 825 struct evtchn_bind_interdomain bind_interdomain; 826 int error; 827 828 bind_interdomain.remote_dom = remote_domain; 829 bind_interdomain.remote_port = remote_port; 830 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain, 831 &bind_interdomain); 832 if (error != 0) { 833 /* 834 * XXX Trap Hypercall error code Linuxisms in 835 * the HYPERCALL layer. 836 */ 837 return (-error); 838 } 839 840 error = xen_intr_bind_isrc(&isrc, bind_interdomain.local_port, 841 EVTCHN_TYPE_PORT, device_get_nameunit(dev), filter, handler, arg, 842 flags, port_handlep); 843 if (error) { 844 evtchn_close_t close = { .port = bind_interdomain.local_port }; 845 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) 846 panic("EVTCHNOP_close failed"); 847 return (error); 848 } 849 850 /* 851 * The Event Channel API opened this port, so it is 852 * responsible for closing it automatically on unbind. 853 */ 854 isrc->xi_close = 1; 855 return (0); 856} 857 858int 859xen_intr_bind_virq(device_t dev, u_int virq, u_int cpu, 860 driver_filter_t filter, driver_intr_t handler, void *arg, 861 enum intr_type flags, xen_intr_handle_t *port_handlep) 862{ 863 u_int vcpu_id = XEN_CPUID_TO_VCPUID(cpu); 864 struct xenisrc *isrc; 865 struct evtchn_bind_virq bind_virq = { .virq = virq, .vcpu = vcpu_id }; 866 int error; 867 868 isrc = NULL; 869 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, &bind_virq); 870 if (error != 0) { 871 /* 872 * XXX Trap Hypercall error code Linuxisms in 873 * the HYPERCALL layer. 874 */ 875 return (-error); 876 } 877 878 error = xen_intr_bind_isrc(&isrc, bind_virq.port, EVTCHN_TYPE_VIRQ, 879 device_get_nameunit(dev), filter, handler, arg, flags, 880 port_handlep); 881 882#ifdef SMP 883 if (error == 0) 884 error = xen_arch_intr_event_bind(isrc, cpu); 885#endif 886 887 if (error != 0) { 888 evtchn_close_t close = { .port = bind_virq.port }; 889 890 xen_intr_unbind(port_handlep); 891 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) 892 panic("EVTCHNOP_close failed"); 893 return (error); 894 } 895 896#ifdef SMP 897 if (isrc->xi_cpu != cpu) { 898 /* 899 * Too early in the boot process for the generic interrupt 900 * code to perform the binding. Update our event channel 901 * masks manually so events can't fire on the wrong cpu 902 * during AP startup. 903 */ 904 xen_intr_assign_cpu(isrc, cpu); 905 } 906#endif 907 908 /* 909 * The Event Channel API opened this port, so it is 910 * responsible for closing it automatically on unbind. 911 */ 912 isrc->xi_close = 1; 913 isrc->xi_virq = virq; 914 915 return (0); 916} 917 918int 919xen_intr_alloc_and_bind_ipi(u_int cpu, driver_filter_t filter, 920 enum intr_type flags, xen_intr_handle_t *port_handlep) 921{ 922#ifdef SMP 923 u_int vcpu_id = XEN_CPUID_TO_VCPUID(cpu); 924 struct xenisrc *isrc; 925 struct evtchn_bind_ipi bind_ipi = { .vcpu = vcpu_id }; 926 /* Same size as the one used by intr_handler->ih_name. */ 927 char name[MAXCOMLEN + 1]; 928 int error; 929 930 isrc = NULL; 931 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &bind_ipi); 932 if (error != 0) { 933 /* 934 * XXX Trap Hypercall error code Linuxisms in 935 * the HYPERCALL layer. 936 */ 937 return (-error); 938 } 939 940 snprintf(name, sizeof(name), "cpu%u", cpu); 941 942 error = xen_intr_bind_isrc(&isrc, bind_ipi.port, EVTCHN_TYPE_IPI, 943 name, filter, NULL, NULL, flags, port_handlep); 944 if (error != 0) { 945 evtchn_close_t close = { .port = bind_ipi.port }; 946 947 xen_intr_unbind(port_handlep); 948 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) 949 panic("EVTCHNOP_close failed"); 950 return (error); 951 } 952 953 if (isrc->xi_cpu != cpu) { 954 /* 955 * Too early in the boot process for the generic interrupt 956 * code to perform the binding. Update our event channel 957 * masks manually so events can't fire on the wrong cpu 958 * during AP startup. 959 */ 960 xen_intr_assign_cpu(isrc, cpu); 961 } 962 963 /* 964 * The Event Channel API opened this port, so it is 965 * responsible for closing it automatically on unbind. 966 */ 967 isrc->xi_close = 1; 968 return (0); 969#else 970 return (EOPNOTSUPP); 971#endif 972} 973 974int 975xen_intr_describe(xen_intr_handle_t port_handle, const char *fmt, ...) 976{ 977 char descr[MAXCOMLEN + 1]; 978 struct xenisrc *isrc; 979 va_list ap; 980 981 isrc = xen_intr_isrc_from_handle(port_handle); 982 if (isrc == NULL) 983 return (EINVAL); 984 985 va_start(ap, fmt); 986 vsnprintf(descr, sizeof(descr), fmt, ap); 987 va_end(ap); 988 return (xen_arch_intr_describe(isrc, isrc->xi_cookie, descr)); 989} 990 991void 992xen_intr_unbind(xen_intr_handle_t *port_handlep) 993{ 994 struct xenisrc *isrc; 995 996 KASSERT(port_handlep != NULL, 997 ("NULL xen_intr_handle_t passed to %s", __func__)); 998 999 isrc = xen_intr_isrc_from_handle(*port_handlep); 1000 *port_handlep = NULL; 1001 if (isrc == NULL) 1002 return; 1003 1004 mtx_lock(&xen_intr_isrc_lock); 1005 if (refcount_release(&isrc->xi_refcount) == 0) { 1006 mtx_unlock(&xen_intr_isrc_lock); 1007 return; 1008 } 1009 mtx_unlock(&xen_intr_isrc_lock); 1010 1011 if (isrc->xi_cookie != NULL) 1012 xen_arch_intr_remove_handler(isrc, isrc->xi_cookie); 1013 xen_intr_release_isrc(isrc); 1014} 1015 1016void 1017xen_intr_signal(xen_intr_handle_t handle) 1018{ 1019 struct xenisrc *isrc; 1020 1021 isrc = xen_intr_isrc_from_handle(handle); 1022 if (isrc != NULL) { 1023 KASSERT(isrc->xi_type == EVTCHN_TYPE_PORT || 1024 isrc->xi_type == EVTCHN_TYPE_IPI, 1025 ("evtchn_signal on something other than a local port")); 1026 struct evtchn_send send = { .port = isrc->xi_port }; 1027 (void)HYPERVISOR_event_channel_op(EVTCHNOP_send, &send); 1028 } 1029} 1030 1031evtchn_port_t 1032xen_intr_port(xen_intr_handle_t handle) 1033{ 1034 struct xenisrc *isrc; 1035 1036 isrc = xen_intr_isrc_from_handle(handle); 1037 if (isrc == NULL) 1038 return (0); 1039 1040 return (isrc->xi_port); 1041} 1042 1043int 1044xen_intr_add_handler(const char *name, driver_filter_t filter, 1045 driver_intr_t handler, void *arg, enum intr_type flags, 1046 xen_intr_handle_t handle) 1047{ 1048 struct xenisrc *isrc; 1049 int error; 1050 1051 isrc = xen_intr_isrc_from_handle(handle); 1052 if (isrc == NULL || isrc->xi_cookie != NULL) 1053 return (EINVAL); 1054 1055 error = xen_arch_intr_add_handler(name, filter, handler, arg, 1056 flags | INTR_EXCL, isrc, &isrc->xi_cookie); 1057 if (error != 0) 1058 printf("%s: %s: add handler failed: %d\n", name, __func__, 1059 error); 1060 1061 return (error); 1062} 1063 1064int 1065xen_intr_get_evtchn_from_port(evtchn_port_t port, xen_intr_handle_t *handlep) 1066{ 1067 1068 if (!is_valid_evtchn(port)) 1069 return (EINVAL); 1070 1071 if (handlep == NULL) { 1072 return (EINVAL); 1073 } 1074 1075 mtx_lock(&xen_intr_isrc_lock); 1076 if (xen_intr_port_to_isrc[port] == NULL) { 1077 mtx_unlock(&xen_intr_isrc_lock); 1078 return (EINVAL); 1079 } 1080 refcount_acquire(&xen_intr_port_to_isrc[port]->xi_refcount); 1081 mtx_unlock(&xen_intr_isrc_lock); 1082 1083 /* Assign the opaque handler */ 1084 *handlep = xen_intr_handle_from_isrc(xen_intr_port_to_isrc[port]); 1085 1086 return (0); 1087} 1088 1089#ifdef DDB 1090static const char * 1091xen_intr_print_type(enum evtchn_type type) 1092{ 1093 static const char *evtchn_type_to_string[EVTCHN_TYPE_COUNT] = { 1094 [EVTCHN_TYPE_UNBOUND] = "UNBOUND", 1095 [EVTCHN_TYPE_VIRQ] = "VIRQ", 1096 [EVTCHN_TYPE_IPI] = "IPI", 1097 [EVTCHN_TYPE_PORT] = "PORT", 1098 }; 1099 1100 if (type >= EVTCHN_TYPE_COUNT) 1101 return ("UNKNOWN"); 1102 1103 return (evtchn_type_to_string[type]); 1104} 1105 1106static void 1107xen_intr_dump_port(struct xenisrc *isrc) 1108{ 1109 struct xen_intr_pcpu_data *pcpu; 1110 shared_info_t *s = HYPERVISOR_shared_info; 1111 u_int i; 1112 1113 db_printf("Port %d Type: %s\n", 1114 isrc->xi_port, xen_intr_print_type(isrc->xi_type)); 1115 if (isrc->xi_type == EVTCHN_TYPE_VIRQ) 1116 db_printf("\tVirq: %u\n", isrc->xi_virq); 1117 1118 db_printf("\tMasked: %d Pending: %d\n", 1119 !!xen_test_bit(isrc->xi_port, &s->evtchn_mask[0]), 1120 !!xen_test_bit(isrc->xi_port, &s->evtchn_pending[0])); 1121 1122 db_printf("\tPer-CPU Masks: "); 1123 CPU_FOREACH(i) { 1124 pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu); 1125 db_printf("cpu#%u: %d ", i, 1126 !!xen_test_bit(isrc->xi_port, pcpu->evtchn_enabled)); 1127 } 1128 db_printf("\n"); 1129} 1130 1131DB_SHOW_COMMAND(xen_evtchn, db_show_xen_evtchn) 1132{ 1133 u_int i; 1134 1135 if (!xen_domain()) { 1136 db_printf("Only available on Xen guests\n"); 1137 return; 1138 } 1139 1140 for (i = 0; i < NR_EVENT_CHANNELS; i++) { 1141 struct xenisrc *isrc; 1142 1143 isrc = xen_intr_port_to_isrc[i]; 1144 if (isrc == NULL) 1145 continue; 1146 1147 xen_intr_dump_port(isrc); 1148 } 1149} 1150#endif /* DDB */ 1151