xen_intr.c revision 291645
1/****************************************************************************** 2 * xen_intr.c 3 * 4 * Xen event and interrupt services for x86 PV and HVM guests. 5 * 6 * Copyright (c) 2002-2005, K A Fraser 7 * Copyright (c) 2005, Intel Corporation <xiaofeng.ling@intel.com> 8 * Copyright (c) 2012, Spectra Logic Corporation 9 * 10 * This file may be distributed separately from the Linux kernel, or 11 * incorporated into other software packages, subject to the following license: 12 * 13 * Permission is hereby granted, free of charge, to any person obtaining a copy 14 * of this source file (the "Software"), to deal in the Software without 15 * restriction, including without limitation the rights to use, copy, modify, 16 * merge, publish, distribute, sublicense, and/or sell copies of the Software, 17 * and to permit persons to whom the Software is furnished to do so, subject to 18 * the following conditions: 19 * 20 * The above copyright notice and this permission notice shall be included in 21 * all copies or substantial portions of the Software. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 24 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 25 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 26 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 27 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 28 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 29 * IN THE SOFTWARE. 30 */ 31 32#include <sys/cdefs.h> 33__FBSDID("$FreeBSD: stable/10/sys/x86/xen/xen_intr.c 291645 2015-12-02 10:26:34Z royger $"); 34 35#include <sys/param.h> 36#include <sys/systm.h> 37#include <sys/bus.h> 38#include <sys/malloc.h> 39#include <sys/kernel.h> 40#include <sys/limits.h> 41#include <sys/lock.h> 42#include <sys/mutex.h> 43#include <sys/interrupt.h> 44#include <sys/pcpu.h> 45#include <sys/smp.h> 46 47#include <vm/vm.h> 48#include <vm/pmap.h> 49 50#include <machine/intr_machdep.h> 51#include <machine/apicvar.h> 52#include <machine/smp.h> 53#include <machine/stdarg.h> 54 55#include <machine/xen/synch_bitops.h> 56#include <machine/xen/xen-os.h> 57#include <machine/xen/xenvar.h> 58 59#include <xen/hypervisor.h> 60#include <xen/xen_intr.h> 61#include <xen/evtchn/evtchnvar.h> 62 63#include <dev/xen/xenpci/xenpcivar.h> 64 65static MALLOC_DEFINE(M_XENINTR, "xen_intr", "Xen Interrupt Services"); 66 67/** 68 * Per-cpu event channel processing state. 69 */ 70struct xen_intr_pcpu_data { 71 /** 72 * The last event channel bitmap section (level one bit) processed. 73 * This is used to ensure we scan all ports before 74 * servicing an already servied port again. 75 */ 76 u_int last_processed_l1i; 77 78 /** 79 * The last event channel processed within the event channel 80 * bitmap being scanned. 81 */ 82 u_int last_processed_l2i; 83 84 /** Pointer to this CPU's interrupt statistic counter. */ 85 u_long *evtchn_intrcnt; 86 87 /** 88 * A bitmap of ports that can be serviced from this CPU. 89 * A set bit means interrupt handling is enabled. 90 */ 91 u_long evtchn_enabled[sizeof(u_long) * 8]; 92}; 93 94/* 95 * Start the scan at port 0 by initializing the last scanned 96 * location as the highest numbered event channel port. 97 */ 98DPCPU_DEFINE(struct xen_intr_pcpu_data, xen_intr_pcpu) = { 99 .last_processed_l1i = LONG_BIT - 1, 100 .last_processed_l2i = LONG_BIT - 1 101}; 102 103DPCPU_DECLARE(struct vcpu_info *, vcpu_info); 104 105#define is_valid_evtchn(x) ((x) != 0) 106 107struct xenisrc { 108 struct intsrc xi_intsrc; 109 enum evtchn_type xi_type; 110 int xi_cpu; /* VCPU for delivery. */ 111 int xi_vector; /* Global isrc vector number. */ 112 evtchn_port_t xi_port; 113 int xi_pirq; 114 int xi_virq; 115 u_int xi_close:1; /* close on unbind? */ 116 u_int xi_needs_eoi:1; 117 u_int xi_shared:1; /* Shared with other domains. */ 118}; 119 120#define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0])) 121 122static void xen_intr_suspend(struct pic *); 123static void xen_intr_resume(struct pic *, bool suspend_cancelled); 124static void xen_intr_enable_source(struct intsrc *isrc); 125static void xen_intr_disable_source(struct intsrc *isrc, int eoi); 126static void xen_intr_eoi_source(struct intsrc *isrc); 127static void xen_intr_enable_intr(struct intsrc *isrc); 128static void xen_intr_disable_intr(struct intsrc *isrc); 129static int xen_intr_vector(struct intsrc *isrc); 130static int xen_intr_source_pending(struct intsrc *isrc); 131static int xen_intr_config_intr(struct intsrc *isrc, 132 enum intr_trigger trig, enum intr_polarity pol); 133static int xen_intr_assign_cpu(struct intsrc *isrc, u_int apic_id); 134 135static void xen_intr_pirq_enable_source(struct intsrc *isrc); 136static void xen_intr_pirq_disable_source(struct intsrc *isrc, int eoi); 137static void xen_intr_pirq_eoi_source(struct intsrc *isrc); 138static void xen_intr_pirq_enable_intr(struct intsrc *isrc); 139 140/** 141 * PIC interface for all event channel port types except physical IRQs. 142 */ 143struct pic xen_intr_pic = { 144 .pic_enable_source = xen_intr_enable_source, 145 .pic_disable_source = xen_intr_disable_source, 146 .pic_eoi_source = xen_intr_eoi_source, 147 .pic_enable_intr = xen_intr_enable_intr, 148 .pic_disable_intr = xen_intr_disable_intr, 149 .pic_vector = xen_intr_vector, 150 .pic_source_pending = xen_intr_source_pending, 151 .pic_suspend = xen_intr_suspend, 152 .pic_resume = xen_intr_resume, 153 .pic_config_intr = xen_intr_config_intr, 154 .pic_assign_cpu = xen_intr_assign_cpu 155}; 156 157/** 158 * PIC interface for all event channel representing 159 * physical interrupt sources. 160 */ 161struct pic xen_intr_pirq_pic = { 162 .pic_enable_source = xen_intr_pirq_enable_source, 163 .pic_disable_source = xen_intr_pirq_disable_source, 164 .pic_eoi_source = xen_intr_pirq_eoi_source, 165 .pic_enable_intr = xen_intr_pirq_enable_intr, 166 .pic_disable_intr = xen_intr_disable_intr, 167 .pic_vector = xen_intr_vector, 168 .pic_source_pending = xen_intr_source_pending, 169 .pic_suspend = xen_intr_suspend, 170 .pic_resume = xen_intr_resume, 171 .pic_config_intr = xen_intr_config_intr, 172 .pic_assign_cpu = xen_intr_assign_cpu 173}; 174 175static struct mtx xen_intr_isrc_lock; 176static int xen_intr_isrc_count; 177static struct xenisrc *xen_intr_port_to_isrc[NR_EVENT_CHANNELS]; 178 179/*------------------------- Private Functions --------------------------------*/ 180/** 181 * Disable signal delivery for an event channel port on the 182 * specified CPU. 183 * 184 * \param port The event channel port to mask. 185 * 186 * This API is used to manage the port<=>CPU binding of event 187 * channel handlers. 188 * 189 * \note This operation does not preclude reception of an event 190 * for this event channel on another CPU. To mask the 191 * event channel globally, use evtchn_mask(). 192 */ 193static inline void 194evtchn_cpu_mask_port(u_int cpu, evtchn_port_t port) 195{ 196 struct xen_intr_pcpu_data *pcpu; 197 198 pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu); 199 clear_bit(port, pcpu->evtchn_enabled); 200} 201 202/** 203 * Enable signal delivery for an event channel port on the 204 * specified CPU. 205 * 206 * \param port The event channel port to unmask. 207 * 208 * This API is used to manage the port<=>CPU binding of event 209 * channel handlers. 210 * 211 * \note This operation does not guarantee that event delivery 212 * is enabled for this event channel port. The port must 213 * also be globally enabled. See evtchn_unmask(). 214 */ 215static inline void 216evtchn_cpu_unmask_port(u_int cpu, evtchn_port_t port) 217{ 218 struct xen_intr_pcpu_data *pcpu; 219 220 pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu); 221 set_bit(port, pcpu->evtchn_enabled); 222} 223 224/** 225 * Allocate and register a per-cpu Xen upcall interrupt counter. 226 * 227 * \param cpu The cpu for which to register this interrupt count. 228 */ 229static void 230xen_intr_intrcnt_add(u_int cpu) 231{ 232 char buf[MAXCOMLEN + 1]; 233 struct xen_intr_pcpu_data *pcpu; 234 235 pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu); 236 if (pcpu->evtchn_intrcnt != NULL) 237 return; 238 239 snprintf(buf, sizeof(buf), "cpu%d:xen", cpu); 240 intrcnt_add(buf, &pcpu->evtchn_intrcnt); 241} 242 243/** 244 * Search for an already allocated but currently unused Xen interrupt 245 * source object. 246 * 247 * \param type Restrict the search to interrupt sources of the given 248 * type. 249 * 250 * \return A pointer to a free Xen interrupt source object or NULL. 251 */ 252static struct xenisrc * 253xen_intr_find_unused_isrc(enum evtchn_type type) 254{ 255 int isrc_idx; 256 257 KASSERT(mtx_owned(&xen_intr_isrc_lock), ("Evtchn isrc lock not held")); 258 259 for (isrc_idx = 0; isrc_idx < xen_intr_isrc_count; isrc_idx ++) { 260 struct xenisrc *isrc; 261 u_int vector; 262 263 vector = FIRST_EVTCHN_INT + isrc_idx; 264 isrc = (struct xenisrc *)intr_lookup_source(vector); 265 if (isrc != NULL 266 && isrc->xi_type == EVTCHN_TYPE_UNBOUND) { 267 KASSERT(isrc->xi_intsrc.is_handlers == 0, 268 ("Free evtchn still has handlers")); 269 isrc->xi_type = type; 270 return (isrc); 271 } 272 } 273 return (NULL); 274} 275 276/** 277 * Allocate a Xen interrupt source object. 278 * 279 * \param type The type of interrupt source to create. 280 * 281 * \return A pointer to a newly allocated Xen interrupt source 282 * object or NULL. 283 */ 284static struct xenisrc * 285xen_intr_alloc_isrc(enum evtchn_type type) 286{ 287 static int warned; 288 struct xenisrc *isrc; 289 int vector; 290 291 KASSERT(mtx_owned(&xen_intr_isrc_lock), ("Evtchn alloc lock not held")); 292 293 if (xen_intr_isrc_count > NR_EVENT_CHANNELS) { 294 if (!warned) { 295 warned = 1; 296 printf("xen_intr_alloc: Event channels exhausted.\n"); 297 } 298 return (NULL); 299 } 300 vector = FIRST_EVTCHN_INT + xen_intr_isrc_count; 301 xen_intr_isrc_count++; 302 303 mtx_unlock(&xen_intr_isrc_lock); 304 isrc = malloc(sizeof(*isrc), M_XENINTR, M_WAITOK | M_ZERO); 305 isrc->xi_intsrc.is_pic = &xen_intr_pic; 306 isrc->xi_vector = vector; 307 isrc->xi_type = type; 308 intr_register_source(&isrc->xi_intsrc); 309 mtx_lock(&xen_intr_isrc_lock); 310 311 return (isrc); 312} 313 314/** 315 * Attempt to free an active Xen interrupt source object. 316 * 317 * \param isrc The interrupt source object to release. 318 * 319 * \returns EBUSY if the source is still in use, otherwise 0. 320 */ 321static int 322xen_intr_release_isrc(struct xenisrc *isrc) 323{ 324 325 mtx_lock(&xen_intr_isrc_lock); 326 if (isrc->xi_intsrc.is_handlers != 0) { 327 mtx_unlock(&xen_intr_isrc_lock); 328 return (EBUSY); 329 } 330 evtchn_mask_port(isrc->xi_port); 331 evtchn_clear_port(isrc->xi_port); 332 333 /* Rebind port to CPU 0. */ 334 evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port); 335 evtchn_cpu_unmask_port(0, isrc->xi_port); 336 337 if (isrc->xi_close != 0 && is_valid_evtchn(isrc->xi_port)) { 338 struct evtchn_close close = { .port = isrc->xi_port }; 339 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) 340 panic("EVTCHNOP_close failed"); 341 } 342 343 xen_intr_port_to_isrc[isrc->xi_port] = NULL; 344 isrc->xi_cpu = 0; 345 isrc->xi_type = EVTCHN_TYPE_UNBOUND; 346 isrc->xi_port = 0; 347 mtx_unlock(&xen_intr_isrc_lock); 348 return (0); 349} 350 351/** 352 * Associate an interrupt handler with an already allocated local Xen 353 * event channel port. 354 * 355 * \param isrcp The returned Xen interrupt object associated with 356 * the specified local port. 357 * \param local_port The event channel to bind. 358 * \param type The event channel type of local_port. 359 * \param intr_owner The device making this bind request. 360 * \param filter An interrupt filter handler. Specify NULL 361 * to always dispatch to the ithread handler. 362 * \param handler An interrupt ithread handler. Optional (can 363 * specify NULL) if all necessary event actions 364 * are performed by filter. 365 * \param arg Argument to present to both filter and handler. 366 * \param irqflags Interrupt handler flags. See sys/bus.h. 367 * \param handlep Pointer to an opaque handle used to manage this 368 * registration. 369 * 370 * \returns 0 on success, otherwise an errno. 371 */ 372static int 373xen_intr_bind_isrc(struct xenisrc **isrcp, evtchn_port_t local_port, 374 enum evtchn_type type, device_t intr_owner, driver_filter_t filter, 375 driver_intr_t handler, void *arg, enum intr_type flags, 376 xen_intr_handle_t *port_handlep) 377{ 378 struct xenisrc *isrc; 379 int error; 380 381 *isrcp = NULL; 382 if (port_handlep == NULL) { 383 device_printf(intr_owner, 384 "xen_intr_bind_isrc: Bad event handle\n"); 385 return (EINVAL); 386 } 387 388 mtx_lock(&xen_intr_isrc_lock); 389 isrc = xen_intr_find_unused_isrc(type); 390 if (isrc == NULL) { 391 isrc = xen_intr_alloc_isrc(type); 392 if (isrc == NULL) { 393 mtx_unlock(&xen_intr_isrc_lock); 394 return (ENOSPC); 395 } 396 } 397 isrc->xi_port = local_port; 398 xen_intr_port_to_isrc[local_port] = isrc; 399 mtx_unlock(&xen_intr_isrc_lock); 400 401 error = intr_add_handler(device_get_nameunit(intr_owner), 402 isrc->xi_vector, filter, handler, arg, 403 flags|INTR_EXCL, port_handlep); 404 if (error != 0) { 405 device_printf(intr_owner, 406 "xen_intr_bind_irq: intr_add_handler failed\n"); 407 xen_intr_release_isrc(isrc); 408 return (error); 409 } 410 *isrcp = isrc; 411 evtchn_unmask_port(local_port); 412 return (0); 413} 414 415/** 416 * Lookup a Xen interrupt source object given an interrupt binding handle. 417 * 418 * \param handle A handle initialized by a previous call to 419 * xen_intr_bind_isrc(). 420 * 421 * \returns A pointer to the Xen interrupt source object associated 422 * with the given interrupt handle. NULL if no association 423 * currently exists. 424 */ 425static struct xenisrc * 426xen_intr_isrc(xen_intr_handle_t handle) 427{ 428 struct intr_handler *ih; 429 430 ih = handle; 431 if (ih == NULL || ih->ih_event == NULL) 432 return (NULL); 433 434 return (ih->ih_event->ie_source); 435} 436 437/** 438 * Determine the event channel ports at the given section of the 439 * event port bitmap which have pending events for the given cpu. 440 * 441 * \param pcpu The Xen interrupt pcpu data for the cpu being querried. 442 * \param sh The Xen shared info area. 443 * \param idx The index of the section of the event channel bitmap to 444 * inspect. 445 * 446 * \returns A u_long with bits set for every event channel with pending 447 * events. 448 */ 449static inline u_long 450xen_intr_active_ports(struct xen_intr_pcpu_data *pcpu, shared_info_t *sh, 451 u_int idx) 452{ 453 return (sh->evtchn_pending[idx] 454 & ~sh->evtchn_mask[idx] 455 & pcpu->evtchn_enabled[idx]); 456} 457 458/** 459 * Interrupt handler for processing all Xen event channel events. 460 * 461 * \param trap_frame The trap frame context for the current interrupt. 462 */ 463void 464xen_intr_handle_upcall(struct trapframe *trap_frame) 465{ 466 u_int l1i, l2i, port, cpu; 467 u_long masked_l1, masked_l2; 468 struct xenisrc *isrc; 469 shared_info_t *s; 470 vcpu_info_t *v; 471 struct xen_intr_pcpu_data *pc; 472 u_long l1, l2; 473 474 /* 475 * Disable preemption in order to always check and fire events 476 * on the right vCPU 477 */ 478 critical_enter(); 479 480 cpu = PCPU_GET(cpuid); 481 pc = DPCPU_PTR(xen_intr_pcpu); 482 s = HYPERVISOR_shared_info; 483 v = DPCPU_GET(vcpu_info); 484 485 if (xen_hvm_domain() && !xen_vector_callback_enabled) { 486 KASSERT((cpu == 0), ("Fired PCI event callback on wrong CPU")); 487 } 488 489 v->evtchn_upcall_pending = 0; 490 491#if 0 492#ifndef CONFIG_X86 /* No need for a barrier -- XCHG is a barrier on x86. */ 493 /* Clear master flag /before/ clearing selector flag. */ 494 wmb(); 495#endif 496#endif 497 498 l1 = atomic_readandclear_long(&v->evtchn_pending_sel); 499 500 l1i = pc->last_processed_l1i; 501 l2i = pc->last_processed_l2i; 502 (*pc->evtchn_intrcnt)++; 503 504 while (l1 != 0) { 505 506 l1i = (l1i + 1) % LONG_BIT; 507 masked_l1 = l1 & ((~0UL) << l1i); 508 509 if (masked_l1 == 0) { 510 /* 511 * if we masked out all events, wrap around 512 * to the beginning. 513 */ 514 l1i = LONG_BIT - 1; 515 l2i = LONG_BIT - 1; 516 continue; 517 } 518 l1i = ffsl(masked_l1) - 1; 519 520 do { 521 l2 = xen_intr_active_ports(pc, s, l1i); 522 523 l2i = (l2i + 1) % LONG_BIT; 524 masked_l2 = l2 & ((~0UL) << l2i); 525 526 if (masked_l2 == 0) { 527 /* if we masked out all events, move on */ 528 l2i = LONG_BIT - 1; 529 break; 530 } 531 l2i = ffsl(masked_l2) - 1; 532 533 /* process port */ 534 port = (l1i * LONG_BIT) + l2i; 535 synch_clear_bit(port, &s->evtchn_pending[0]); 536 537 isrc = xen_intr_port_to_isrc[port]; 538 if (__predict_false(isrc == NULL)) 539 continue; 540 541 /* Make sure we are firing on the right vCPU */ 542 KASSERT((isrc->xi_cpu == PCPU_GET(cpuid)), 543 ("Received unexpected event on vCPU#%d, event bound to vCPU#%d", 544 PCPU_GET(cpuid), isrc->xi_cpu)); 545 546 intr_execute_handlers(&isrc->xi_intsrc, trap_frame); 547 548 /* 549 * If this is the final port processed, 550 * we'll pick up here+1 next time. 551 */ 552 pc->last_processed_l1i = l1i; 553 pc->last_processed_l2i = l2i; 554 555 } while (l2i != LONG_BIT - 1); 556 557 l2 = xen_intr_active_ports(pc, s, l1i); 558 if (l2 == 0) { 559 /* 560 * We handled all ports, so we can clear the 561 * selector bit. 562 */ 563 l1 &= ~(1UL << l1i); 564 } 565 } 566 critical_exit(); 567} 568 569static int 570xen_intr_init(void *dummy __unused) 571{ 572 struct xen_intr_pcpu_data *pcpu; 573 int i; 574 575 if (!xen_domain()) 576 return (0); 577 578 mtx_init(&xen_intr_isrc_lock, "xen-irq-lock", NULL, MTX_DEF); 579 580 /* 581 * Register interrupt count manually as we aren't 582 * guaranteed to see a call to xen_intr_assign_cpu() 583 * before our first interrupt. Also set the per-cpu 584 * mask of CPU#0 to enable all, since by default 585 * all event channels are bound to CPU#0. 586 */ 587 CPU_FOREACH(i) { 588 pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu); 589 memset(pcpu->evtchn_enabled, i == 0 ? ~0 : 0, 590 sizeof(pcpu->evtchn_enabled)); 591 xen_intr_intrcnt_add(i); 592 } 593 594 intr_register_pic(&xen_intr_pic); 595 596 return (0); 597} 598SYSINIT(xen_intr_init, SI_SUB_INTR, SI_ORDER_MIDDLE, xen_intr_init, NULL); 599 600/*--------------------------- Common PIC Functions ---------------------------*/ 601/** 602 * Prepare this PIC for system suspension. 603 */ 604static void 605xen_intr_suspend(struct pic *unused) 606{ 607} 608 609static void 610xen_rebind_ipi(struct xenisrc *isrc) 611{ 612#ifdef SMP 613 int cpu = isrc->xi_cpu; 614 int vcpu_id = pcpu_find(cpu)->pc_vcpu_id; 615 int error; 616 struct evtchn_bind_ipi bind_ipi = { .vcpu = vcpu_id }; 617 618 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, 619 &bind_ipi); 620 if (error != 0) 621 panic("unable to rebind xen IPI: %d", error); 622 623 isrc->xi_port = bind_ipi.port; 624 isrc->xi_cpu = 0; 625 xen_intr_port_to_isrc[bind_ipi.port] = isrc; 626 627 error = xen_intr_assign_cpu(&isrc->xi_intsrc, 628 cpu_apic_ids[cpu]); 629 if (error) 630 panic("unable to bind xen IPI to CPU#%d: %d", 631 cpu, error); 632 633 evtchn_unmask_port(bind_ipi.port); 634#else 635 panic("Resume IPI event channel on UP"); 636#endif 637} 638 639static void 640xen_rebind_virq(struct xenisrc *isrc) 641{ 642 int cpu = isrc->xi_cpu; 643 int vcpu_id = pcpu_find(cpu)->pc_vcpu_id; 644 int error; 645 struct evtchn_bind_virq bind_virq = { .virq = isrc->xi_virq, 646 .vcpu = vcpu_id }; 647 648 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, 649 &bind_virq); 650 if (error != 0) 651 panic("unable to rebind xen VIRQ#%d: %d", isrc->xi_virq, error); 652 653 isrc->xi_port = bind_virq.port; 654 isrc->xi_cpu = 0; 655 xen_intr_port_to_isrc[bind_virq.port] = isrc; 656 657#ifdef SMP 658 error = xen_intr_assign_cpu(&isrc->xi_intsrc, 659 cpu_apic_ids[cpu]); 660 if (error) 661 panic("unable to bind xen VIRQ#%d to CPU#%d: %d", 662 isrc->xi_virq, cpu, error); 663#endif 664 665 evtchn_unmask_port(bind_virq.port); 666} 667 668/** 669 * Return this PIC to service after being suspended. 670 */ 671static void 672xen_intr_resume(struct pic *unused, bool suspend_cancelled) 673{ 674 shared_info_t *s = HYPERVISOR_shared_info; 675 struct xenisrc *isrc; 676 u_int isrc_idx; 677 int i; 678 679 if (suspend_cancelled) 680 return; 681 682 /* Reset the per-CPU masks */ 683 CPU_FOREACH(i) { 684 struct xen_intr_pcpu_data *pcpu; 685 686 pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu); 687 memset(pcpu->evtchn_enabled, 688 i == 0 ? ~0 : 0, sizeof(pcpu->evtchn_enabled)); 689 } 690 691 /* Mask all event channels. */ 692 for (i = 0; i < nitems(s->evtchn_mask); i++) 693 atomic_store_rel_long(&s->evtchn_mask[i], ~0); 694 695 /* Remove port -> isrc mappings */ 696 memset(xen_intr_port_to_isrc, 0, sizeof(xen_intr_port_to_isrc)); 697 698 /* Free unused isrcs and rebind VIRQs and IPIs */ 699 for (isrc_idx = 0; isrc_idx < xen_intr_isrc_count; isrc_idx++) { 700 u_int vector; 701 702 vector = FIRST_EVTCHN_INT + isrc_idx; 703 isrc = (struct xenisrc *)intr_lookup_source(vector); 704 if (isrc != NULL) { 705 isrc->xi_port = 0; 706 switch (isrc->xi_type) { 707 case EVTCHN_TYPE_IPI: 708 xen_rebind_ipi(isrc); 709 break; 710 case EVTCHN_TYPE_VIRQ: 711 xen_rebind_virq(isrc); 712 break; 713 default: 714 intr_remove_handler(isrc->xi_cookie); 715 isrc->xi_cpu = 0; 716 isrc->xi_type = EVTCHN_TYPE_UNBOUND; 717 isrc->xi_cookie = NULL; 718 break; 719 } 720 } 721 } 722} 723 724/** 725 * Disable a Xen interrupt source. 726 * 727 * \param isrc The interrupt source to disable. 728 */ 729static void 730xen_intr_disable_intr(struct intsrc *base_isrc) 731{ 732 struct xenisrc *isrc = (struct xenisrc *)base_isrc; 733 734 evtchn_mask_port(isrc->xi_port); 735} 736 737/** 738 * Determine the global interrupt vector number for 739 * a Xen interrupt source. 740 * 741 * \param isrc The interrupt source to query. 742 * 743 * \return The vector number corresponding to the given interrupt source. 744 */ 745static int 746xen_intr_vector(struct intsrc *base_isrc) 747{ 748 struct xenisrc *isrc = (struct xenisrc *)base_isrc; 749 750 return (isrc->xi_vector); 751} 752 753/** 754 * Determine whether or not interrupt events are pending on the 755 * the given interrupt source. 756 * 757 * \param isrc The interrupt source to query. 758 * 759 * \returns 0 if no events are pending, otherwise non-zero. 760 */ 761static int 762xen_intr_source_pending(struct intsrc *isrc) 763{ 764 /* 765 * EventChannels are edge triggered and never masked. 766 * There can be no pending events. 767 */ 768 return (0); 769} 770 771/** 772 * Perform configuration of an interrupt source. 773 * 774 * \param isrc The interrupt source to configure. 775 * \param trig Edge or level. 776 * \param pol Active high or low. 777 * 778 * \returns 0 if no events are pending, otherwise non-zero. 779 */ 780static int 781xen_intr_config_intr(struct intsrc *isrc, enum intr_trigger trig, 782 enum intr_polarity pol) 783{ 784 /* Configuration is only possible via the evtchn apis. */ 785 return (ENODEV); 786} 787 788/** 789 * Configure CPU affinity for interrupt source event delivery. 790 * 791 * \param isrc The interrupt source to configure. 792 * \param apic_id The apic id of the CPU for handling future events. 793 * 794 * \returns 0 if successful, otherwise an errno. 795 */ 796static int 797xen_intr_assign_cpu(struct intsrc *base_isrc, u_int apic_id) 798{ 799#ifdef SMP 800 struct evtchn_bind_vcpu bind_vcpu; 801 struct xenisrc *isrc; 802 u_int to_cpu, vcpu_id; 803 int error; 804 805#ifdef XENHVM 806 if (xen_vector_callback_enabled == 0) 807 return (EOPNOTSUPP); 808#endif 809 810 to_cpu = apic_cpuid(apic_id); 811 vcpu_id = pcpu_find(to_cpu)->pc_vcpu_id; 812 xen_intr_intrcnt_add(to_cpu); 813 814 mtx_lock(&xen_intr_isrc_lock); 815 isrc = (struct xenisrc *)base_isrc; 816 if (!is_valid_evtchn(isrc->xi_port)) { 817 mtx_unlock(&xen_intr_isrc_lock); 818 return (EINVAL); 819 } 820 821 if ((isrc->xi_type == EVTCHN_TYPE_VIRQ) || 822 (isrc->xi_type == EVTCHN_TYPE_IPI)) { 823 /* 824 * Virtual IRQs are associated with a cpu by 825 * the Hypervisor at evtchn_bind_virq time, so 826 * all we need to do is update the per-CPU masks. 827 */ 828 evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port); 829 isrc->xi_cpu = to_cpu; 830 evtchn_cpu_unmask_port(isrc->xi_cpu, isrc->xi_port); 831 mtx_unlock(&xen_intr_isrc_lock); 832 return (0); 833 } 834 835 bind_vcpu.port = isrc->xi_port; 836 bind_vcpu.vcpu = vcpu_id; 837 838 /* 839 * Allow interrupts to be fielded on the new VCPU before 840 * we ask the hypervisor to deliver them there. 841 */ 842 evtchn_cpu_unmask_port(to_cpu, isrc->xi_port); 843 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu); 844 if (isrc->xi_cpu != to_cpu) { 845 if (error == 0) { 846 /* Commit to new binding by removing the old one. */ 847 evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port); 848 isrc->xi_cpu = to_cpu; 849 } else { 850 /* Roll-back to previous binding. */ 851 evtchn_cpu_mask_port(to_cpu, isrc->xi_port); 852 } 853 } 854 mtx_unlock(&xen_intr_isrc_lock); 855 return (0); 856#else 857 return (EOPNOTSUPP); 858#endif 859} 860 861/*------------------- Virtual Interrupt Source PIC Functions -----------------*/ 862/* 863 * Mask a level triggered interrupt source. 864 * 865 * \param isrc The interrupt source to mask (if necessary). 866 * \param eoi If non-zero, perform any necessary end-of-interrupt 867 * acknowledgements. 868 */ 869static void 870xen_intr_disable_source(struct intsrc *isrc, int eoi) 871{ 872} 873 874/* 875 * Unmask a level triggered interrupt source. 876 * 877 * \param isrc The interrupt source to unmask (if necessary). 878 */ 879static void 880xen_intr_enable_source(struct intsrc *isrc) 881{ 882} 883 884/* 885 * Perform any necessary end-of-interrupt acknowledgements. 886 * 887 * \param isrc The interrupt source to EOI. 888 */ 889static void 890xen_intr_eoi_source(struct intsrc *isrc) 891{ 892} 893 894/* 895 * Enable and unmask the interrupt source. 896 * 897 * \param isrc The interrupt source to enable. 898 */ 899static void 900xen_intr_enable_intr(struct intsrc *base_isrc) 901{ 902 struct xenisrc *isrc = (struct xenisrc *)base_isrc; 903 904 evtchn_unmask_port(isrc->xi_port); 905} 906 907/*------------------ Physical Interrupt Source PIC Functions -----------------*/ 908/* 909 * Mask a level triggered interrupt source. 910 * 911 * \param isrc The interrupt source to mask (if necessary). 912 * \param eoi If non-zero, perform any necessary end-of-interrupt 913 * acknowledgements. 914 */ 915static void 916xen_intr_pirq_disable_source(struct intsrc *base_isrc, int eoi) 917{ 918 struct xenisrc *isrc; 919 920 isrc = (struct xenisrc *)base_isrc; 921 evtchn_mask_port(isrc->xi_port); 922} 923 924/* 925 * Unmask a level triggered interrupt source. 926 * 927 * \param isrc The interrupt source to unmask (if necessary). 928 */ 929static void 930xen_intr_pirq_enable_source(struct intsrc *base_isrc) 931{ 932 struct xenisrc *isrc; 933 934 isrc = (struct xenisrc *)base_isrc; 935 evtchn_unmask_port(isrc->xi_port); 936} 937 938/* 939 * Perform any necessary end-of-interrupt acknowledgements. 940 * 941 * \param isrc The interrupt source to EOI. 942 */ 943static void 944xen_intr_pirq_eoi_source(struct intsrc *base_isrc) 945{ 946 struct xenisrc *isrc; 947 948 /* XXX Use shared page of flags for this. */ 949 isrc = (struct xenisrc *)base_isrc; 950 if (isrc->xi_needs_eoi != 0) { 951 struct physdev_eoi eoi = { .irq = isrc->xi_pirq }; 952 953 (void)HYPERVISOR_physdev_op(PHYSDEVOP_eoi, &eoi); 954 } 955} 956 957/* 958 * Enable and unmask the interrupt source. 959 * 960 * \param isrc The interrupt source to enable. 961 */ 962static void 963xen_intr_pirq_enable_intr(struct intsrc *isrc) 964{ 965} 966 967/*--------------------------- Public Functions -------------------------------*/ 968/*------- API comments for these methods can be found in xen/xenintr.h -------*/ 969int 970xen_intr_bind_local_port(device_t dev, evtchn_port_t local_port, 971 driver_filter_t filter, driver_intr_t handler, void *arg, 972 enum intr_type flags, xen_intr_handle_t *port_handlep) 973{ 974 struct xenisrc *isrc; 975 int error; 976 977 error = xen_intr_bind_isrc(&isrc, local_port, EVTCHN_TYPE_PORT, dev, 978 filter, handler, arg, flags, port_handlep); 979 if (error != 0) 980 return (error); 981 982 /* 983 * The Event Channel API didn't open this port, so it is not 984 * responsible for closing it automatically on unbind. 985 */ 986 isrc->xi_close = 0; 987 return (0); 988} 989 990int 991xen_intr_alloc_and_bind_local_port(device_t dev, u_int remote_domain, 992 driver_filter_t filter, driver_intr_t handler, void *arg, 993 enum intr_type flags, xen_intr_handle_t *port_handlep) 994{ 995 struct xenisrc *isrc; 996 struct evtchn_alloc_unbound alloc_unbound; 997 int error; 998 999 alloc_unbound.dom = DOMID_SELF; 1000 alloc_unbound.remote_dom = remote_domain; 1001 error = HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound, 1002 &alloc_unbound); 1003 if (error != 0) { 1004 /* 1005 * XXX Trap Hypercall error code Linuxisms in 1006 * the HYPERCALL layer. 1007 */ 1008 return (-error); 1009 } 1010 1011 error = xen_intr_bind_isrc(&isrc, alloc_unbound.port, EVTCHN_TYPE_PORT, 1012 dev, filter, handler, arg, flags, 1013 port_handlep); 1014 if (error != 0) { 1015 evtchn_close_t close = { .port = alloc_unbound.port }; 1016 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) 1017 panic("EVTCHNOP_close failed"); 1018 return (error); 1019 } 1020 1021 isrc->xi_close = 1; 1022 return (0); 1023} 1024 1025int 1026xen_intr_bind_remote_port(device_t dev, u_int remote_domain, 1027 u_int remote_port, driver_filter_t filter, driver_intr_t handler, 1028 void *arg, enum intr_type flags, xen_intr_handle_t *port_handlep) 1029{ 1030 struct xenisrc *isrc; 1031 struct evtchn_bind_interdomain bind_interdomain; 1032 int error; 1033 1034 bind_interdomain.remote_dom = remote_domain; 1035 bind_interdomain.remote_port = remote_port; 1036 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain, 1037 &bind_interdomain); 1038 if (error != 0) { 1039 /* 1040 * XXX Trap Hypercall error code Linuxisms in 1041 * the HYPERCALL layer. 1042 */ 1043 return (-error); 1044 } 1045 1046 error = xen_intr_bind_isrc(&isrc, bind_interdomain.local_port, 1047 EVTCHN_TYPE_PORT, dev, filter, handler, 1048 arg, flags, port_handlep); 1049 if (error) { 1050 evtchn_close_t close = { .port = bind_interdomain.local_port }; 1051 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) 1052 panic("EVTCHNOP_close failed"); 1053 return (error); 1054 } 1055 1056 /* 1057 * The Event Channel API opened this port, so it is 1058 * responsible for closing it automatically on unbind. 1059 */ 1060 isrc->xi_close = 1; 1061 return (0); 1062} 1063 1064int 1065xen_intr_bind_virq(device_t dev, u_int virq, u_int cpu, 1066 driver_filter_t filter, driver_intr_t handler, void *arg, 1067 enum intr_type flags, xen_intr_handle_t *port_handlep) 1068{ 1069 int vcpu_id = pcpu_find(cpu)->pc_vcpu_id; 1070 struct xenisrc *isrc; 1071 struct evtchn_bind_virq bind_virq = { .virq = virq, .vcpu = vcpu_id }; 1072 int error; 1073 1074 /* Ensure the target CPU is ready to handle evtchn interrupts. */ 1075 xen_intr_intrcnt_add(cpu); 1076 1077 isrc = NULL; 1078 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, &bind_virq); 1079 if (error != 0) { 1080 /* 1081 * XXX Trap Hypercall error code Linuxisms in 1082 * the HYPERCALL layer. 1083 */ 1084 return (-error); 1085 } 1086 1087 error = xen_intr_bind_isrc(&isrc, bind_virq.port, EVTCHN_TYPE_VIRQ, dev, 1088 filter, handler, arg, flags, port_handlep); 1089 1090#ifdef SMP 1091 if (error == 0) 1092 error = intr_event_bind(isrc->xi_intsrc.is_event, cpu); 1093#endif 1094 1095 if (error != 0) { 1096 evtchn_close_t close = { .port = bind_virq.port }; 1097 1098 xen_intr_unbind(*port_handlep); 1099 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) 1100 panic("EVTCHNOP_close failed"); 1101 return (error); 1102 } 1103 1104#ifdef SMP 1105 if (isrc->xi_cpu != cpu) { 1106 /* 1107 * Too early in the boot process for the generic interrupt 1108 * code to perform the binding. Update our event channel 1109 * masks manually so events can't fire on the wrong cpu 1110 * during AP startup. 1111 */ 1112 xen_intr_assign_cpu(&isrc->xi_intsrc, cpu_apic_ids[cpu]); 1113 } 1114#endif 1115 1116 /* 1117 * The Event Channel API opened this port, so it is 1118 * responsible for closing it automatically on unbind. 1119 */ 1120 isrc->xi_close = 1; 1121 isrc->xi_virq = virq; 1122 1123 return (0); 1124} 1125 1126int 1127xen_intr_alloc_and_bind_ipi(device_t dev, u_int cpu, 1128 driver_filter_t filter, enum intr_type flags, 1129 xen_intr_handle_t *port_handlep) 1130{ 1131#ifdef SMP 1132 int vcpu_id = pcpu_find(cpu)->pc_vcpu_id; 1133 struct xenisrc *isrc; 1134 struct evtchn_bind_ipi bind_ipi = { .vcpu = vcpu_id }; 1135 int error; 1136 1137 /* Ensure the target CPU is ready to handle evtchn interrupts. */ 1138 xen_intr_intrcnt_add(cpu); 1139 1140 isrc = NULL; 1141 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &bind_ipi); 1142 if (error != 0) { 1143 /* 1144 * XXX Trap Hypercall error code Linuxisms in 1145 * the HYPERCALL layer. 1146 */ 1147 return (-error); 1148 } 1149 1150 error = xen_intr_bind_isrc(&isrc, bind_ipi.port, EVTCHN_TYPE_IPI, 1151 dev, filter, NULL, NULL, flags, 1152 port_handlep); 1153 if (error == 0) 1154 error = intr_event_bind(isrc->xi_intsrc.is_event, cpu); 1155 1156 if (error != 0) { 1157 evtchn_close_t close = { .port = bind_ipi.port }; 1158 1159 xen_intr_unbind(*port_handlep); 1160 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) 1161 panic("EVTCHNOP_close failed"); 1162 return (error); 1163 } 1164 1165 if (isrc->xi_cpu != cpu) { 1166 /* 1167 * Too early in the boot process for the generic interrupt 1168 * code to perform the binding. Update our event channel 1169 * masks manually so events can't fire on the wrong cpu 1170 * during AP startup. 1171 */ 1172 xen_intr_assign_cpu(&isrc->xi_intsrc, cpu_apic_ids[cpu]); 1173 } 1174 1175 /* 1176 * The Event Channel API opened this port, so it is 1177 * responsible for closing it automatically on unbind. 1178 */ 1179 isrc->xi_close = 1; 1180 return (0); 1181#else 1182 return (EOPNOTSUPP); 1183#endif 1184} 1185 1186int 1187xen_intr_describe(xen_intr_handle_t port_handle, const char *fmt, ...) 1188{ 1189 char descr[MAXCOMLEN + 1]; 1190 struct xenisrc *isrc; 1191 va_list ap; 1192 1193 isrc = xen_intr_isrc(port_handle); 1194 if (isrc == NULL) 1195 return (EINVAL); 1196 1197 va_start(ap, fmt); 1198 vsnprintf(descr, sizeof(descr), fmt, ap); 1199 va_end(ap); 1200 return (intr_describe(isrc->xi_vector, port_handle, descr)); 1201} 1202 1203void 1204xen_intr_unbind(xen_intr_handle_t *port_handlep) 1205{ 1206 struct intr_handler *handler; 1207 struct xenisrc *isrc; 1208 1209 handler = *port_handlep; 1210 *port_handlep = NULL; 1211 isrc = xen_intr_isrc(handler); 1212 if (isrc == NULL) 1213 return; 1214 1215 intr_remove_handler(handler); 1216 xen_intr_release_isrc(isrc); 1217} 1218 1219void 1220xen_intr_signal(xen_intr_handle_t handle) 1221{ 1222 struct xenisrc *isrc; 1223 1224 isrc = xen_intr_isrc(handle); 1225 if (isrc != NULL) { 1226 KASSERT(isrc->xi_type == EVTCHN_TYPE_PORT || 1227 isrc->xi_type == EVTCHN_TYPE_IPI, 1228 ("evtchn_signal on something other than a local port")); 1229 struct evtchn_send send = { .port = isrc->xi_port }; 1230 (void)HYPERVISOR_event_channel_op(EVTCHNOP_send, &send); 1231 } 1232} 1233 1234evtchn_port_t 1235xen_intr_port(xen_intr_handle_t handle) 1236{ 1237 struct xenisrc *isrc; 1238 1239 isrc = xen_intr_isrc(handle); 1240 if (isrc == NULL) 1241 return (0); 1242 1243 return (isrc->xi_port); 1244} 1245