1// SPDX-License-Identifier: GPL-2.0 2 3#define pr_fmt(fmt) "DMAR-IR: " fmt 4 5#include <linux/interrupt.h> 6#include <linux/dmar.h> 7#include <linux/spinlock.h> 8#include <linux/slab.h> 9#include <linux/jiffies.h> 10#include <linux/hpet.h> 11#include <linux/pci.h> 12#include <linux/irq.h> 13#include <linux/acpi.h> 14#include <linux/irqdomain.h> 15#include <linux/crash_dump.h> 16#include <asm/io_apic.h> 17#include <asm/apic.h> 18#include <asm/smp.h> 19#include <asm/cpu.h> 20#include <asm/irq_remapping.h> 21#include <asm/pci-direct.h> 22#include <asm/posted_intr.h> 23 24#include "iommu.h" 25#include "../irq_remapping.h" 26#include "../iommu-pages.h" 27#include "cap_audit.h" 28 29enum irq_mode { 30 IRQ_REMAPPING, 31 IRQ_POSTING, 32}; 33 34struct ioapic_scope { 35 struct intel_iommu *iommu; 36 unsigned int id; 37 unsigned int bus; /* PCI bus number */ 38 unsigned int devfn; /* PCI devfn number */ 39}; 40 41struct hpet_scope { 42 struct intel_iommu *iommu; 43 u8 id; 44 unsigned int bus; 45 unsigned int devfn; 46}; 47 48struct irq_2_iommu { 49 struct intel_iommu *iommu; 50 u16 irte_index; 51 u16 sub_handle; 52 u8 irte_mask; 53 enum irq_mode mode; 54 bool posted_msi; 55}; 56 57struct intel_ir_data { 58 struct irq_2_iommu irq_2_iommu; 59 struct irte irte_entry; 60 union { 61 struct msi_msg msi_entry; 62 }; 63}; 64 65#define IR_X2APIC_MODE(mode) (mode ? (1 << 11) : 0) 66#define IRTE_DEST(dest) ((eim_mode) ? dest : dest << 8) 67 68static int __read_mostly eim_mode; 69static struct ioapic_scope ir_ioapic[MAX_IO_APICS]; 70static struct hpet_scope ir_hpet[MAX_HPET_TBS]; 71 72/* 73 * Lock ordering: 74 * ->dmar_global_lock 75 * ->irq_2_ir_lock 76 * ->qi->q_lock 77 * ->iommu->register_lock 78 * Note: 79 * intel_irq_remap_ops.{supported,prepare,enable,disable,reenable} are called 80 * in single-threaded environment with interrupt disabled, so no need to tabke 81 * the dmar_global_lock. 82 */ 83DEFINE_RAW_SPINLOCK(irq_2_ir_lock); 84static const struct irq_domain_ops intel_ir_domain_ops; 85 86static void iommu_disable_irq_remapping(struct intel_iommu *iommu); 87static int __init parse_ioapics_under_ir(void); 88static const struct msi_parent_ops dmar_msi_parent_ops; 89 90static bool ir_pre_enabled(struct intel_iommu *iommu) 91{ 92 return (iommu->flags & VTD_FLAG_IRQ_REMAP_PRE_ENABLED); 93} 94 95static void clear_ir_pre_enabled(struct intel_iommu *iommu) 96{ 97 iommu->flags &= ~VTD_FLAG_IRQ_REMAP_PRE_ENABLED; 98} 99 100static void init_ir_status(struct intel_iommu *iommu) 101{ 102 u32 gsts; 103 104 gsts = readl(iommu->reg + DMAR_GSTS_REG); 105 if (gsts & DMA_GSTS_IRES) 106 iommu->flags |= VTD_FLAG_IRQ_REMAP_PRE_ENABLED; 107} 108 109static int alloc_irte(struct intel_iommu *iommu, 110 struct irq_2_iommu *irq_iommu, u16 count) 111{ 112 struct ir_table *table = iommu->ir_table; 113 unsigned int mask = 0; 114 unsigned long flags; 115 int index; 116 117 if (!count || !irq_iommu) 118 return -1; 119 120 if (count > 1) { 121 count = __roundup_pow_of_two(count); 122 mask = ilog2(count); 123 } 124 125 if (mask > ecap_max_handle_mask(iommu->ecap)) { 126 pr_err("Requested mask %x exceeds the max invalidation handle" 127 " mask value %Lx\n", mask, 128 ecap_max_handle_mask(iommu->ecap)); 129 return -1; 130 } 131 132 raw_spin_lock_irqsave(&irq_2_ir_lock, flags); 133 index = bitmap_find_free_region(table->bitmap, 134 INTR_REMAP_TABLE_ENTRIES, mask); 135 if (index < 0) { 136 pr_warn("IR%d: can't allocate an IRTE\n", iommu->seq_id); 137 } else { 138 irq_iommu->iommu = iommu; 139 irq_iommu->irte_index = index; 140 irq_iommu->sub_handle = 0; 141 irq_iommu->irte_mask = mask; 142 irq_iommu->mode = IRQ_REMAPPING; 143 } 144 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); 145 146 return index; 147} 148 149static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask) 150{ 151 struct qi_desc desc; 152 153 desc.qw0 = QI_IEC_IIDEX(index) | QI_IEC_TYPE | QI_IEC_IM(mask) 154 | QI_IEC_SELECTIVE; 155 desc.qw1 = 0; 156 desc.qw2 = 0; 157 desc.qw3 = 0; 158 159 return qi_submit_sync(iommu, &desc, 1, 0); 160} 161 162static int modify_irte(struct irq_2_iommu *irq_iommu, 163 struct irte *irte_modified) 164{ 165 struct intel_iommu *iommu; 166 unsigned long flags; 167 struct irte *irte; 168 int rc, index; 169 170 if (!irq_iommu) 171 return -1; 172 173 raw_spin_lock_irqsave(&irq_2_ir_lock, flags); 174 175 iommu = irq_iommu->iommu; 176 177 index = irq_iommu->irte_index + irq_iommu->sub_handle; 178 irte = &iommu->ir_table->base[index]; 179 180 if ((irte->pst == 1) || (irte_modified->pst == 1)) { 181 /* 182 * We use cmpxchg16 to atomically update the 128-bit IRTE, 183 * and it cannot be updated by the hardware or other processors 184 * behind us, so the return value of cmpxchg16 should be the 185 * same as the old value. 186 */ 187 u128 old = irte->irte; 188 WARN_ON(!try_cmpxchg128(&irte->irte, &old, irte_modified->irte)); 189 } else { 190 WRITE_ONCE(irte->low, irte_modified->low); 191 WRITE_ONCE(irte->high, irte_modified->high); 192 } 193 __iommu_flush_cache(iommu, irte, sizeof(*irte)); 194 195 rc = qi_flush_iec(iommu, index, 0); 196 197 /* Update iommu mode according to the IRTE mode */ 198 irq_iommu->mode = irte->pst ? IRQ_POSTING : IRQ_REMAPPING; 199 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); 200 201 return rc; 202} 203 204static struct intel_iommu *map_hpet_to_iommu(u8 hpet_id) 205{ 206 int i; 207 208 for (i = 0; i < MAX_HPET_TBS; i++) { 209 if (ir_hpet[i].id == hpet_id && ir_hpet[i].iommu) 210 return ir_hpet[i].iommu; 211 } 212 return NULL; 213} 214 215static struct intel_iommu *map_ioapic_to_iommu(int apic) 216{ 217 int i; 218 219 for (i = 0; i < MAX_IO_APICS; i++) { 220 if (ir_ioapic[i].id == apic && ir_ioapic[i].iommu) 221 return ir_ioapic[i].iommu; 222 } 223 return NULL; 224} 225 226static struct irq_domain *map_dev_to_ir(struct pci_dev *dev) 227{ 228 struct dmar_drhd_unit *drhd = dmar_find_matched_drhd_unit(dev); 229 230 return drhd ? drhd->iommu->ir_domain : NULL; 231} 232 233static int clear_entries(struct irq_2_iommu *irq_iommu) 234{ 235 struct irte *start, *entry, *end; 236 struct intel_iommu *iommu; 237 int index; 238 239 if (irq_iommu->sub_handle) 240 return 0; 241 242 iommu = irq_iommu->iommu; 243 index = irq_iommu->irte_index; 244 245 start = iommu->ir_table->base + index; 246 end = start + (1 << irq_iommu->irte_mask); 247 248 for (entry = start; entry < end; entry++) { 249 WRITE_ONCE(entry->low, 0); 250 WRITE_ONCE(entry->high, 0); 251 } 252 bitmap_release_region(iommu->ir_table->bitmap, index, 253 irq_iommu->irte_mask); 254 255 return qi_flush_iec(iommu, index, irq_iommu->irte_mask); 256} 257 258/* 259 * source validation type 260 */ 261#define SVT_NO_VERIFY 0x0 /* no verification is required */ 262#define SVT_VERIFY_SID_SQ 0x1 /* verify using SID and SQ fields */ 263#define SVT_VERIFY_BUS 0x2 /* verify bus of request-id */ 264 265/* 266 * source-id qualifier 267 */ 268#define SQ_ALL_16 0x0 /* verify all 16 bits of request-id */ 269#define SQ_13_IGNORE_1 0x1 /* verify most significant 13 bits, ignore 270 * the third least significant bit 271 */ 272#define SQ_13_IGNORE_2 0x2 /* verify most significant 13 bits, ignore 273 * the second and third least significant bits 274 */ 275#define SQ_13_IGNORE_3 0x3 /* verify most significant 13 bits, ignore 276 * the least three significant bits 277 */ 278 279/* 280 * set SVT, SQ and SID fields of irte to verify 281 * source ids of interrupt requests 282 */ 283static void set_irte_sid(struct irte *irte, unsigned int svt, 284 unsigned int sq, unsigned int sid) 285{ 286 if (disable_sourceid_checking) 287 svt = SVT_NO_VERIFY; 288 irte->svt = svt; 289 irte->sq = sq; 290 irte->sid = sid; 291} 292 293/* 294 * Set an IRTE to match only the bus number. Interrupt requests that reference 295 * this IRTE must have a requester-id whose bus number is between or equal 296 * to the start_bus and end_bus arguments. 297 */ 298static void set_irte_verify_bus(struct irte *irte, unsigned int start_bus, 299 unsigned int end_bus) 300{ 301 set_irte_sid(irte, SVT_VERIFY_BUS, SQ_ALL_16, 302 (start_bus << 8) | end_bus); 303} 304 305static int set_ioapic_sid(struct irte *irte, int apic) 306{ 307 int i; 308 u16 sid = 0; 309 310 if (!irte) 311 return -1; 312 313 for (i = 0; i < MAX_IO_APICS; i++) { 314 if (ir_ioapic[i].iommu && ir_ioapic[i].id == apic) { 315 sid = (ir_ioapic[i].bus << 8) | ir_ioapic[i].devfn; 316 break; 317 } 318 } 319 320 if (sid == 0) { 321 pr_warn("Failed to set source-id of IOAPIC (%d)\n", apic); 322 return -1; 323 } 324 325 set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, sid); 326 327 return 0; 328} 329 330static int set_hpet_sid(struct irte *irte, u8 id) 331{ 332 int i; 333 u16 sid = 0; 334 335 if (!irte) 336 return -1; 337 338 for (i = 0; i < MAX_HPET_TBS; i++) { 339 if (ir_hpet[i].iommu && ir_hpet[i].id == id) { 340 sid = (ir_hpet[i].bus << 8) | ir_hpet[i].devfn; 341 break; 342 } 343 } 344 345 if (sid == 0) { 346 pr_warn("Failed to set source-id of HPET block (%d)\n", id); 347 return -1; 348 } 349 350 /* 351 * Should really use SQ_ALL_16. Some platforms are broken. 352 * While we figure out the right quirks for these broken platforms, use 353 * SQ_13_IGNORE_3 for now. 354 */ 355 set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_13_IGNORE_3, sid); 356 357 return 0; 358} 359 360struct set_msi_sid_data { 361 struct pci_dev *pdev; 362 u16 alias; 363 int count; 364 int busmatch_count; 365}; 366 367static int set_msi_sid_cb(struct pci_dev *pdev, u16 alias, void *opaque) 368{ 369 struct set_msi_sid_data *data = opaque; 370 371 if (data->count == 0 || PCI_BUS_NUM(alias) == PCI_BUS_NUM(data->alias)) 372 data->busmatch_count++; 373 374 data->pdev = pdev; 375 data->alias = alias; 376 data->count++; 377 378 return 0; 379} 380 381static int set_msi_sid(struct irte *irte, struct pci_dev *dev) 382{ 383 struct set_msi_sid_data data; 384 385 if (!irte || !dev) 386 return -1; 387 388 data.count = 0; 389 data.busmatch_count = 0; 390 pci_for_each_dma_alias(dev, set_msi_sid_cb, &data); 391 392 /* 393 * DMA alias provides us with a PCI device and alias. The only case 394 * where the it will return an alias on a different bus than the 395 * device is the case of a PCIe-to-PCI bridge, where the alias is for 396 * the subordinate bus. In this case we can only verify the bus. 397 * 398 * If there are multiple aliases, all with the same bus number, 399 * then all we can do is verify the bus. This is typical in NTB 400 * hardware which use proxy IDs where the device will generate traffic 401 * from multiple devfn numbers on the same bus. 402 * 403 * If the alias device is on a different bus than our source device 404 * then we have a topology based alias, use it. 405 * 406 * Otherwise, the alias is for a device DMA quirk and we cannot 407 * assume that MSI uses the same requester ID. Therefore use the 408 * original device. 409 */ 410 if (PCI_BUS_NUM(data.alias) != data.pdev->bus->number) 411 set_irte_verify_bus(irte, PCI_BUS_NUM(data.alias), 412 dev->bus->number); 413 else if (data.count >= 2 && data.busmatch_count == data.count) 414 set_irte_verify_bus(irte, dev->bus->number, dev->bus->number); 415 else if (data.pdev->bus->number != dev->bus->number) 416 set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, data.alias); 417 else 418 set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, 419 pci_dev_id(dev)); 420 421 return 0; 422} 423 424static int iommu_load_old_irte(struct intel_iommu *iommu) 425{ 426 struct irte *old_ir_table; 427 phys_addr_t irt_phys; 428 unsigned int i; 429 size_t size; 430 u64 irta; 431 432 /* Check whether the old ir-table has the same size as ours */ 433 irta = dmar_readq(iommu->reg + DMAR_IRTA_REG); 434 if ((irta & INTR_REMAP_TABLE_REG_SIZE_MASK) 435 != INTR_REMAP_TABLE_REG_SIZE) 436 return -EINVAL; 437 438 irt_phys = irta & VTD_PAGE_MASK; 439 size = INTR_REMAP_TABLE_ENTRIES*sizeof(struct irte); 440 441 /* Map the old IR table */ 442 old_ir_table = memremap(irt_phys, size, MEMREMAP_WB); 443 if (!old_ir_table) 444 return -ENOMEM; 445 446 /* Copy data over */ 447 memcpy(iommu->ir_table->base, old_ir_table, size); 448 449 __iommu_flush_cache(iommu, iommu->ir_table->base, size); 450 451 /* 452 * Now check the table for used entries and mark those as 453 * allocated in the bitmap 454 */ 455 for (i = 0; i < INTR_REMAP_TABLE_ENTRIES; i++) { 456 if (iommu->ir_table->base[i].present) 457 bitmap_set(iommu->ir_table->bitmap, i, 1); 458 } 459 460 memunmap(old_ir_table); 461 462 return 0; 463} 464 465 466static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode) 467{ 468 unsigned long flags; 469 u64 addr; 470 u32 sts; 471 472 addr = virt_to_phys((void *)iommu->ir_table->base); 473 474 raw_spin_lock_irqsave(&iommu->register_lock, flags); 475 476 dmar_writeq(iommu->reg + DMAR_IRTA_REG, 477 (addr) | IR_X2APIC_MODE(mode) | INTR_REMAP_TABLE_REG_SIZE); 478 479 /* Set interrupt-remapping table pointer */ 480 writel(iommu->gcmd | DMA_GCMD_SIRTP, iommu->reg + DMAR_GCMD_REG); 481 482 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, 483 readl, (sts & DMA_GSTS_IRTPS), sts); 484 raw_spin_unlock_irqrestore(&iommu->register_lock, flags); 485 486 /* 487 * Global invalidation of interrupt entry cache to make sure the 488 * hardware uses the new irq remapping table. 489 */ 490 if (!cap_esirtps(iommu->cap)) 491 qi_global_iec(iommu); 492} 493 494static void iommu_enable_irq_remapping(struct intel_iommu *iommu) 495{ 496 unsigned long flags; 497 u32 sts; 498 499 raw_spin_lock_irqsave(&iommu->register_lock, flags); 500 501 /* Enable interrupt-remapping */ 502 iommu->gcmd |= DMA_GCMD_IRE; 503 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); 504 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, 505 readl, (sts & DMA_GSTS_IRES), sts); 506 507 /* Block compatibility-format MSIs */ 508 if (sts & DMA_GSTS_CFIS) { 509 iommu->gcmd &= ~DMA_GCMD_CFI; 510 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); 511 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, 512 readl, !(sts & DMA_GSTS_CFIS), sts); 513 } 514 515 /* 516 * With CFI clear in the Global Command register, we should be 517 * protected from dangerous (i.e. compatibility) interrupts 518 * regardless of x2apic status. Check just to be sure. 519 */ 520 if (sts & DMA_GSTS_CFIS) 521 WARN(1, KERN_WARNING 522 "Compatibility-format IRQs enabled despite intr remapping;\n" 523 "you are vulnerable to IRQ injection.\n"); 524 525 raw_spin_unlock_irqrestore(&iommu->register_lock, flags); 526} 527 528static int intel_setup_irq_remapping(struct intel_iommu *iommu) 529{ 530 struct ir_table *ir_table; 531 struct fwnode_handle *fn; 532 unsigned long *bitmap; 533 void *ir_table_base; 534 535 if (iommu->ir_table) 536 return 0; 537 538 ir_table = kzalloc(sizeof(struct ir_table), GFP_KERNEL); 539 if (!ir_table) 540 return -ENOMEM; 541 542 ir_table_base = iommu_alloc_pages_node(iommu->node, GFP_KERNEL, 543 INTR_REMAP_PAGE_ORDER); 544 if (!ir_table_base) { 545 pr_err("IR%d: failed to allocate pages of order %d\n", 546 iommu->seq_id, INTR_REMAP_PAGE_ORDER); 547 goto out_free_table; 548 } 549 550 bitmap = bitmap_zalloc(INTR_REMAP_TABLE_ENTRIES, GFP_KERNEL); 551 if (bitmap == NULL) { 552 pr_err("IR%d: failed to allocate bitmap\n", iommu->seq_id); 553 goto out_free_pages; 554 } 555 556 fn = irq_domain_alloc_named_id_fwnode("INTEL-IR", iommu->seq_id); 557 if (!fn) 558 goto out_free_bitmap; 559 560 iommu->ir_domain = 561 irq_domain_create_hierarchy(arch_get_ir_parent_domain(), 562 0, INTR_REMAP_TABLE_ENTRIES, 563 fn, &intel_ir_domain_ops, 564 iommu); 565 if (!iommu->ir_domain) { 566 pr_err("IR%d: failed to allocate irqdomain\n", iommu->seq_id); 567 goto out_free_fwnode; 568 } 569 570 irq_domain_update_bus_token(iommu->ir_domain, DOMAIN_BUS_DMAR); 571 iommu->ir_domain->flags |= IRQ_DOMAIN_FLAG_MSI_PARENT | 572 IRQ_DOMAIN_FLAG_ISOLATED_MSI; 573 iommu->ir_domain->msi_parent_ops = &dmar_msi_parent_ops; 574 575 ir_table->base = ir_table_base; 576 ir_table->bitmap = bitmap; 577 iommu->ir_table = ir_table; 578 579 /* 580 * If the queued invalidation is already initialized, 581 * shouldn't disable it. 582 */ 583 if (!iommu->qi) { 584 /* 585 * Clear previous faults. 586 */ 587 dmar_fault(-1, iommu); 588 dmar_disable_qi(iommu); 589 590 if (dmar_enable_qi(iommu)) { 591 pr_err("Failed to enable queued invalidation\n"); 592 goto out_free_ir_domain; 593 } 594 } 595 596 init_ir_status(iommu); 597 598 if (ir_pre_enabled(iommu)) { 599 if (!is_kdump_kernel()) { 600 pr_warn("IRQ remapping was enabled on %s but we are not in kdump mode\n", 601 iommu->name); 602 clear_ir_pre_enabled(iommu); 603 iommu_disable_irq_remapping(iommu); 604 } else if (iommu_load_old_irte(iommu)) 605 pr_err("Failed to copy IR table for %s from previous kernel\n", 606 iommu->name); 607 else 608 pr_info("Copied IR table for %s from previous kernel\n", 609 iommu->name); 610 } 611 612 iommu_set_irq_remapping(iommu, eim_mode); 613 614 return 0; 615 616out_free_ir_domain: 617 irq_domain_remove(iommu->ir_domain); 618 iommu->ir_domain = NULL; 619out_free_fwnode: 620 irq_domain_free_fwnode(fn); 621out_free_bitmap: 622 bitmap_free(bitmap); 623out_free_pages: 624 iommu_free_pages(ir_table_base, INTR_REMAP_PAGE_ORDER); 625out_free_table: 626 kfree(ir_table); 627 628 iommu->ir_table = NULL; 629 630 return -ENOMEM; 631} 632 633static void intel_teardown_irq_remapping(struct intel_iommu *iommu) 634{ 635 struct fwnode_handle *fn; 636 637 if (iommu && iommu->ir_table) { 638 if (iommu->ir_domain) { 639 fn = iommu->ir_domain->fwnode; 640 641 irq_domain_remove(iommu->ir_domain); 642 irq_domain_free_fwnode(fn); 643 iommu->ir_domain = NULL; 644 } 645 iommu_free_pages(iommu->ir_table->base, INTR_REMAP_PAGE_ORDER); 646 bitmap_free(iommu->ir_table->bitmap); 647 kfree(iommu->ir_table); 648 iommu->ir_table = NULL; 649 } 650} 651 652/* 653 * Disable Interrupt Remapping. 654 */ 655static void iommu_disable_irq_remapping(struct intel_iommu *iommu) 656{ 657 unsigned long flags; 658 u32 sts; 659 660 if (!ecap_ir_support(iommu->ecap)) 661 return; 662 663 /* 664 * global invalidation of interrupt entry cache before disabling 665 * interrupt-remapping. 666 */ 667 if (!cap_esirtps(iommu->cap)) 668 qi_global_iec(iommu); 669 670 raw_spin_lock_irqsave(&iommu->register_lock, flags); 671 672 sts = readl(iommu->reg + DMAR_GSTS_REG); 673 if (!(sts & DMA_GSTS_IRES)) 674 goto end; 675 676 iommu->gcmd &= ~DMA_GCMD_IRE; 677 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); 678 679 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, 680 readl, !(sts & DMA_GSTS_IRES), sts); 681 682end: 683 raw_spin_unlock_irqrestore(&iommu->register_lock, flags); 684} 685 686static int __init dmar_x2apic_optout(void) 687{ 688 struct acpi_table_dmar *dmar; 689 dmar = (struct acpi_table_dmar *)dmar_tbl; 690 if (!dmar || no_x2apic_optout) 691 return 0; 692 return dmar->flags & DMAR_X2APIC_OPT_OUT; 693} 694 695static void __init intel_cleanup_irq_remapping(void) 696{ 697 struct dmar_drhd_unit *drhd; 698 struct intel_iommu *iommu; 699 700 for_each_iommu(iommu, drhd) { 701 if (ecap_ir_support(iommu->ecap)) { 702 iommu_disable_irq_remapping(iommu); 703 intel_teardown_irq_remapping(iommu); 704 } 705 } 706 707 if (x2apic_supported()) 708 pr_warn("Failed to enable irq remapping. You are vulnerable to irq-injection attacks.\n"); 709} 710 711static int __init intel_prepare_irq_remapping(void) 712{ 713 struct dmar_drhd_unit *drhd; 714 struct intel_iommu *iommu; 715 int eim = 0; 716 717 if (irq_remap_broken) { 718 pr_warn("This system BIOS has enabled interrupt remapping\n" 719 "on a chipset that contains an erratum making that\n" 720 "feature unstable. To maintain system stability\n" 721 "interrupt remapping is being disabled. Please\n" 722 "contact your BIOS vendor for an update\n"); 723 add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK); 724 return -ENODEV; 725 } 726 727 if (dmar_table_init() < 0) 728 return -ENODEV; 729 730 if (intel_cap_audit(CAP_AUDIT_STATIC_IRQR, NULL)) 731 return -ENODEV; 732 733 if (!dmar_ir_support()) 734 return -ENODEV; 735 736 if (parse_ioapics_under_ir()) { 737 pr_info("Not enabling interrupt remapping\n"); 738 goto error; 739 } 740 741 /* First make sure all IOMMUs support IRQ remapping */ 742 for_each_iommu(iommu, drhd) 743 if (!ecap_ir_support(iommu->ecap)) 744 goto error; 745 746 /* Detect remapping mode: lapic or x2apic */ 747 if (x2apic_supported()) { 748 eim = !dmar_x2apic_optout(); 749 if (!eim) { 750 pr_info("x2apic is disabled because BIOS sets x2apic opt out bit."); 751 pr_info("Use 'intremap=no_x2apic_optout' to override the BIOS setting.\n"); 752 } 753 } 754 755 for_each_iommu(iommu, drhd) { 756 if (eim && !ecap_eim_support(iommu->ecap)) { 757 pr_info("%s does not support EIM\n", iommu->name); 758 eim = 0; 759 } 760 } 761 762 eim_mode = eim; 763 if (eim) 764 pr_info("Queued invalidation will be enabled to support x2apic and Intr-remapping.\n"); 765 766 /* Do the initializations early */ 767 for_each_iommu(iommu, drhd) { 768 if (intel_setup_irq_remapping(iommu)) { 769 pr_err("Failed to setup irq remapping for %s\n", 770 iommu->name); 771 goto error; 772 } 773 } 774 775 return 0; 776 777error: 778 intel_cleanup_irq_remapping(); 779 return -ENODEV; 780} 781 782/* 783 * Set Posted-Interrupts capability. 784 */ 785static inline void set_irq_posting_cap(void) 786{ 787 struct dmar_drhd_unit *drhd; 788 struct intel_iommu *iommu; 789 790 if (!disable_irq_post) { 791 /* 792 * If IRTE is in posted format, the 'pda' field goes across the 793 * 64-bit boundary, we need use cmpxchg16b to atomically update 794 * it. We only expose posted-interrupt when X86_FEATURE_CX16 795 * is supported. Actually, hardware platforms supporting PI 796 * should have X86_FEATURE_CX16 support, this has been confirmed 797 * with Intel hardware guys. 798 */ 799 if (boot_cpu_has(X86_FEATURE_CX16)) 800 intel_irq_remap_ops.capability |= 1 << IRQ_POSTING_CAP; 801 802 for_each_iommu(iommu, drhd) 803 if (!cap_pi_support(iommu->cap)) { 804 intel_irq_remap_ops.capability &= 805 ~(1 << IRQ_POSTING_CAP); 806 break; 807 } 808 } 809} 810 811static int __init intel_enable_irq_remapping(void) 812{ 813 struct dmar_drhd_unit *drhd; 814 struct intel_iommu *iommu; 815 bool setup = false; 816 817 /* 818 * Setup Interrupt-remapping for all the DRHD's now. 819 */ 820 for_each_iommu(iommu, drhd) { 821 if (!ir_pre_enabled(iommu)) 822 iommu_enable_irq_remapping(iommu); 823 setup = true; 824 } 825 826 if (!setup) 827 goto error; 828 829 irq_remapping_enabled = 1; 830 831 set_irq_posting_cap(); 832 833 pr_info("Enabled IRQ remapping in %s mode\n", eim_mode ? "x2apic" : "xapic"); 834 835 return eim_mode ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE; 836 837error: 838 intel_cleanup_irq_remapping(); 839 return -1; 840} 841 842static int ir_parse_one_hpet_scope(struct acpi_dmar_device_scope *scope, 843 struct intel_iommu *iommu, 844 struct acpi_dmar_hardware_unit *drhd) 845{ 846 struct acpi_dmar_pci_path *path; 847 u8 bus; 848 int count, free = -1; 849 850 bus = scope->bus; 851 path = (struct acpi_dmar_pci_path *)(scope + 1); 852 count = (scope->length - sizeof(struct acpi_dmar_device_scope)) 853 / sizeof(struct acpi_dmar_pci_path); 854 855 while (--count > 0) { 856 /* 857 * Access PCI directly due to the PCI 858 * subsystem isn't initialized yet. 859 */ 860 bus = read_pci_config_byte(bus, path->device, path->function, 861 PCI_SECONDARY_BUS); 862 path++; 863 } 864 865 for (count = 0; count < MAX_HPET_TBS; count++) { 866 if (ir_hpet[count].iommu == iommu && 867 ir_hpet[count].id == scope->enumeration_id) 868 return 0; 869 else if (ir_hpet[count].iommu == NULL && free == -1) 870 free = count; 871 } 872 if (free == -1) { 873 pr_warn("Exceeded Max HPET blocks\n"); 874 return -ENOSPC; 875 } 876 877 ir_hpet[free].iommu = iommu; 878 ir_hpet[free].id = scope->enumeration_id; 879 ir_hpet[free].bus = bus; 880 ir_hpet[free].devfn = PCI_DEVFN(path->device, path->function); 881 pr_info("HPET id %d under DRHD base 0x%Lx\n", 882 scope->enumeration_id, drhd->address); 883 884 return 0; 885} 886 887static int ir_parse_one_ioapic_scope(struct acpi_dmar_device_scope *scope, 888 struct intel_iommu *iommu, 889 struct acpi_dmar_hardware_unit *drhd) 890{ 891 struct acpi_dmar_pci_path *path; 892 u8 bus; 893 int count, free = -1; 894 895 bus = scope->bus; 896 path = (struct acpi_dmar_pci_path *)(scope + 1); 897 count = (scope->length - sizeof(struct acpi_dmar_device_scope)) 898 / sizeof(struct acpi_dmar_pci_path); 899 900 while (--count > 0) { 901 /* 902 * Access PCI directly due to the PCI 903 * subsystem isn't initialized yet. 904 */ 905 bus = read_pci_config_byte(bus, path->device, path->function, 906 PCI_SECONDARY_BUS); 907 path++; 908 } 909 910 for (count = 0; count < MAX_IO_APICS; count++) { 911 if (ir_ioapic[count].iommu == iommu && 912 ir_ioapic[count].id == scope->enumeration_id) 913 return 0; 914 else if (ir_ioapic[count].iommu == NULL && free == -1) 915 free = count; 916 } 917 if (free == -1) { 918 pr_warn("Exceeded Max IO APICS\n"); 919 return -ENOSPC; 920 } 921 922 ir_ioapic[free].bus = bus; 923 ir_ioapic[free].devfn = PCI_DEVFN(path->device, path->function); 924 ir_ioapic[free].iommu = iommu; 925 ir_ioapic[free].id = scope->enumeration_id; 926 pr_info("IOAPIC id %d under DRHD base 0x%Lx IOMMU %d\n", 927 scope->enumeration_id, drhd->address, iommu->seq_id); 928 929 return 0; 930} 931 932static int ir_parse_ioapic_hpet_scope(struct acpi_dmar_header *header, 933 struct intel_iommu *iommu) 934{ 935 int ret = 0; 936 struct acpi_dmar_hardware_unit *drhd; 937 struct acpi_dmar_device_scope *scope; 938 void *start, *end; 939 940 drhd = (struct acpi_dmar_hardware_unit *)header; 941 start = (void *)(drhd + 1); 942 end = ((void *)drhd) + header->length; 943 944 while (start < end && ret == 0) { 945 scope = start; 946 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_IOAPIC) 947 ret = ir_parse_one_ioapic_scope(scope, iommu, drhd); 948 else if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_HPET) 949 ret = ir_parse_one_hpet_scope(scope, iommu, drhd); 950 start += scope->length; 951 } 952 953 return ret; 954} 955 956static void ir_remove_ioapic_hpet_scope(struct intel_iommu *iommu) 957{ 958 int i; 959 960 for (i = 0; i < MAX_HPET_TBS; i++) 961 if (ir_hpet[i].iommu == iommu) 962 ir_hpet[i].iommu = NULL; 963 964 for (i = 0; i < MAX_IO_APICS; i++) 965 if (ir_ioapic[i].iommu == iommu) 966 ir_ioapic[i].iommu = NULL; 967} 968 969/* 970 * Finds the assocaition between IOAPIC's and its Interrupt-remapping 971 * hardware unit. 972 */ 973static int __init parse_ioapics_under_ir(void) 974{ 975 struct dmar_drhd_unit *drhd; 976 struct intel_iommu *iommu; 977 bool ir_supported = false; 978 int ioapic_idx; 979 980 for_each_iommu(iommu, drhd) { 981 int ret; 982 983 if (!ecap_ir_support(iommu->ecap)) 984 continue; 985 986 ret = ir_parse_ioapic_hpet_scope(drhd->hdr, iommu); 987 if (ret) 988 return ret; 989 990 ir_supported = true; 991 } 992 993 if (!ir_supported) 994 return -ENODEV; 995 996 for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++) { 997 int ioapic_id = mpc_ioapic_id(ioapic_idx); 998 if (!map_ioapic_to_iommu(ioapic_id)) { 999 pr_err(FW_BUG "ioapic %d has no mapping iommu, " 1000 "interrupt remapping will be disabled\n", 1001 ioapic_id); 1002 return -1; 1003 } 1004 } 1005 1006 return 0; 1007} 1008 1009static int __init ir_dev_scope_init(void) 1010{ 1011 int ret; 1012 1013 if (!irq_remapping_enabled) 1014 return 0; 1015 1016 down_write(&dmar_global_lock); 1017 ret = dmar_dev_scope_init(); 1018 up_write(&dmar_global_lock); 1019 1020 return ret; 1021} 1022rootfs_initcall(ir_dev_scope_init); 1023 1024static void disable_irq_remapping(void) 1025{ 1026 struct dmar_drhd_unit *drhd; 1027 struct intel_iommu *iommu = NULL; 1028 1029 /* 1030 * Disable Interrupt-remapping for all the DRHD's now. 1031 */ 1032 for_each_iommu(iommu, drhd) { 1033 if (!ecap_ir_support(iommu->ecap)) 1034 continue; 1035 1036 iommu_disable_irq_remapping(iommu); 1037 } 1038 1039 /* 1040 * Clear Posted-Interrupts capability. 1041 */ 1042 if (!disable_irq_post) 1043 intel_irq_remap_ops.capability &= ~(1 << IRQ_POSTING_CAP); 1044} 1045 1046static int reenable_irq_remapping(int eim) 1047{ 1048 struct dmar_drhd_unit *drhd; 1049 bool setup = false; 1050 struct intel_iommu *iommu = NULL; 1051 1052 for_each_iommu(iommu, drhd) 1053 if (iommu->qi) 1054 dmar_reenable_qi(iommu); 1055 1056 /* 1057 * Setup Interrupt-remapping for all the DRHD's now. 1058 */ 1059 for_each_iommu(iommu, drhd) { 1060 if (!ecap_ir_support(iommu->ecap)) 1061 continue; 1062 1063 /* Set up interrupt remapping for iommu.*/ 1064 iommu_set_irq_remapping(iommu, eim); 1065 iommu_enable_irq_remapping(iommu); 1066 setup = true; 1067 } 1068 1069 if (!setup) 1070 goto error; 1071 1072 set_irq_posting_cap(); 1073 1074 return 0; 1075 1076error: 1077 /* 1078 * handle error condition gracefully here! 1079 */ 1080 return -1; 1081} 1082 1083/* 1084 * Store the MSI remapping domain pointer in the device if enabled. 1085 * 1086 * This is called from dmar_pci_bus_add_dev() so it works even when DMA 1087 * remapping is disabled. Only update the pointer if the device is not 1088 * already handled by a non default PCI/MSI interrupt domain. This protects 1089 * e.g. VMD devices. 1090 */ 1091void intel_irq_remap_add_device(struct dmar_pci_notify_info *info) 1092{ 1093 if (!irq_remapping_enabled || !pci_dev_has_default_msi_parent_domain(info->dev)) 1094 return; 1095 1096 dev_set_msi_domain(&info->dev->dev, map_dev_to_ir(info->dev)); 1097} 1098 1099static void prepare_irte(struct irte *irte, int vector, unsigned int dest) 1100{ 1101 memset(irte, 0, sizeof(*irte)); 1102 1103 irte->present = 1; 1104 irte->dst_mode = apic->dest_mode_logical; 1105 /* 1106 * Trigger mode in the IRTE will always be edge, and for IO-APIC, the 1107 * actual level or edge trigger will be setup in the IO-APIC 1108 * RTE. This will help simplify level triggered irq migration. 1109 * For more details, see the comments (in io_apic.c) explainig IO-APIC 1110 * irq migration in the presence of interrupt-remapping. 1111 */ 1112 irte->trigger_mode = 0; 1113 irte->dlvry_mode = APIC_DELIVERY_MODE_FIXED; 1114 irte->vector = vector; 1115 irte->dest_id = IRTE_DEST(dest); 1116 irte->redir_hint = 1; 1117} 1118 1119static void prepare_irte_posted(struct irte *irte) 1120{ 1121 memset(irte, 0, sizeof(*irte)); 1122 1123 irte->present = 1; 1124 irte->p_pst = 1; 1125} 1126 1127struct irq_remap_ops intel_irq_remap_ops = { 1128 .prepare = intel_prepare_irq_remapping, 1129 .enable = intel_enable_irq_remapping, 1130 .disable = disable_irq_remapping, 1131 .reenable = reenable_irq_remapping, 1132 .enable_faulting = enable_drhd_fault_handling, 1133}; 1134 1135#ifdef CONFIG_X86_POSTED_MSI 1136 1137static phys_addr_t get_pi_desc_addr(struct irq_data *irqd) 1138{ 1139 int cpu = cpumask_first(irq_data_get_effective_affinity_mask(irqd)); 1140 1141 if (WARN_ON(cpu >= nr_cpu_ids)) 1142 return 0; 1143 1144 return __pa(per_cpu_ptr(&posted_msi_pi_desc, cpu)); 1145} 1146 1147static void intel_ir_reconfigure_irte_posted(struct irq_data *irqd) 1148{ 1149 struct intel_ir_data *ir_data = irqd->chip_data; 1150 struct irte *irte = &ir_data->irte_entry; 1151 struct irte irte_pi; 1152 u64 pid_addr; 1153 1154 pid_addr = get_pi_desc_addr(irqd); 1155 1156 if (!pid_addr) { 1157 pr_warn("Failed to setup IRQ %d for posted mode", irqd->irq); 1158 return; 1159 } 1160 1161 memset(&irte_pi, 0, sizeof(irte_pi)); 1162 1163 /* The shared IRTE already be set up as posted during alloc_irte */ 1164 dmar_copy_shared_irte(&irte_pi, irte); 1165 1166 irte_pi.pda_l = (pid_addr >> (32 - PDA_LOW_BIT)) & ~(-1UL << PDA_LOW_BIT); 1167 irte_pi.pda_h = (pid_addr >> 32) & ~(-1UL << PDA_HIGH_BIT); 1168 1169 modify_irte(&ir_data->irq_2_iommu, &irte_pi); 1170} 1171 1172#else 1173static inline void intel_ir_reconfigure_irte_posted(struct irq_data *irqd) {} 1174#endif 1175 1176static void intel_ir_reconfigure_irte(struct irq_data *irqd, bool force) 1177{ 1178 struct intel_ir_data *ir_data = irqd->chip_data; 1179 struct irte *irte = &ir_data->irte_entry; 1180 struct irq_cfg *cfg = irqd_cfg(irqd); 1181 1182 /* 1183 * Atomically updates the IRTE with the new destination, vector 1184 * and flushes the interrupt entry cache. 1185 */ 1186 irte->vector = cfg->vector; 1187 irte->dest_id = IRTE_DEST(cfg->dest_apicid); 1188 1189 if (ir_data->irq_2_iommu.posted_msi) 1190 intel_ir_reconfigure_irte_posted(irqd); 1191 else if (force || ir_data->irq_2_iommu.mode == IRQ_REMAPPING) 1192 modify_irte(&ir_data->irq_2_iommu, irte); 1193} 1194 1195/* 1196 * Migrate the IO-APIC irq in the presence of intr-remapping. 1197 * 1198 * For both level and edge triggered, irq migration is a simple atomic 1199 * update(of vector and cpu destination) of IRTE and flush the hardware cache. 1200 * 1201 * For level triggered, we eliminate the io-apic RTE modification (with the 1202 * updated vector information), by using a virtual vector (io-apic pin number). 1203 * Real vector that is used for interrupting cpu will be coming from 1204 * the interrupt-remapping table entry. 1205 * 1206 * As the migration is a simple atomic update of IRTE, the same mechanism 1207 * is used to migrate MSI irq's in the presence of interrupt-remapping. 1208 */ 1209static int 1210intel_ir_set_affinity(struct irq_data *data, const struct cpumask *mask, 1211 bool force) 1212{ 1213 struct irq_data *parent = data->parent_data; 1214 struct irq_cfg *cfg = irqd_cfg(data); 1215 int ret; 1216 1217 ret = parent->chip->irq_set_affinity(parent, mask, force); 1218 if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE) 1219 return ret; 1220 1221 intel_ir_reconfigure_irte(data, false); 1222 /* 1223 * After this point, all the interrupts will start arriving 1224 * at the new destination. So, time to cleanup the previous 1225 * vector allocation. 1226 */ 1227 vector_schedule_cleanup(cfg); 1228 1229 return IRQ_SET_MASK_OK_DONE; 1230} 1231 1232static void intel_ir_compose_msi_msg(struct irq_data *irq_data, 1233 struct msi_msg *msg) 1234{ 1235 struct intel_ir_data *ir_data = irq_data->chip_data; 1236 1237 *msg = ir_data->msi_entry; 1238} 1239 1240static int intel_ir_set_vcpu_affinity(struct irq_data *data, void *info) 1241{ 1242 struct intel_ir_data *ir_data = data->chip_data; 1243 struct vcpu_data *vcpu_pi_info = info; 1244 1245 /* stop posting interrupts, back to the default mode */ 1246 if (!vcpu_pi_info) { 1247 modify_irte(&ir_data->irq_2_iommu, &ir_data->irte_entry); 1248 } else { 1249 struct irte irte_pi; 1250 1251 /* 1252 * We are not caching the posted interrupt entry. We 1253 * copy the data from the remapped entry and modify 1254 * the fields which are relevant for posted mode. The 1255 * cached remapped entry is used for switching back to 1256 * remapped mode. 1257 */ 1258 memset(&irte_pi, 0, sizeof(irte_pi)); 1259 dmar_copy_shared_irte(&irte_pi, &ir_data->irte_entry); 1260 1261 /* Update the posted mode fields */ 1262 irte_pi.p_pst = 1; 1263 irte_pi.p_urgent = 0; 1264 irte_pi.p_vector = vcpu_pi_info->vector; 1265 irte_pi.pda_l = (vcpu_pi_info->pi_desc_addr >> 1266 (32 - PDA_LOW_BIT)) & ~(-1UL << PDA_LOW_BIT); 1267 irte_pi.pda_h = (vcpu_pi_info->pi_desc_addr >> 32) & 1268 ~(-1UL << PDA_HIGH_BIT); 1269 1270 modify_irte(&ir_data->irq_2_iommu, &irte_pi); 1271 } 1272 1273 return 0; 1274} 1275 1276static struct irq_chip intel_ir_chip = { 1277 .name = "INTEL-IR", 1278 .irq_ack = apic_ack_irq, 1279 .irq_set_affinity = intel_ir_set_affinity, 1280 .irq_compose_msi_msg = intel_ir_compose_msi_msg, 1281 .irq_set_vcpu_affinity = intel_ir_set_vcpu_affinity, 1282}; 1283 1284/* 1285 * With posted MSIs, all vectors are multiplexed into a single notification 1286 * vector. Devices MSIs are then dispatched in a demux loop where 1287 * EOIs can be coalesced as well. 1288 * 1289 * "INTEL-IR-POST" IRQ chip does not do EOI on ACK, thus the dummy irq_ack() 1290 * function. Instead EOI is performed by the posted interrupt notification 1291 * handler. 1292 * 1293 * For the example below, 3 MSIs are coalesced into one CPU notification. Only 1294 * one apic_eoi() is needed. 1295 * 1296 * __sysvec_posted_msi_notification() 1297 * irq_enter(); 1298 * handle_edge_irq() 1299 * irq_chip_ack_parent() 1300 * dummy(); // No EOI 1301 * handle_irq_event() 1302 * driver_handler() 1303 * handle_edge_irq() 1304 * irq_chip_ack_parent() 1305 * dummy(); // No EOI 1306 * handle_irq_event() 1307 * driver_handler() 1308 * handle_edge_irq() 1309 * irq_chip_ack_parent() 1310 * dummy(); // No EOI 1311 * handle_irq_event() 1312 * driver_handler() 1313 * apic_eoi() 1314 * irq_exit() 1315 */ 1316 1317static void dummy_ack(struct irq_data *d) { } 1318 1319static struct irq_chip intel_ir_chip_post_msi = { 1320 .name = "INTEL-IR-POST", 1321 .irq_ack = dummy_ack, 1322 .irq_set_affinity = intel_ir_set_affinity, 1323 .irq_compose_msi_msg = intel_ir_compose_msi_msg, 1324 .irq_set_vcpu_affinity = intel_ir_set_vcpu_affinity, 1325}; 1326 1327static void fill_msi_msg(struct msi_msg *msg, u32 index, u32 subhandle) 1328{ 1329 memset(msg, 0, sizeof(*msg)); 1330 1331 msg->arch_addr_lo.dmar_base_address = X86_MSI_BASE_ADDRESS_LOW; 1332 msg->arch_addr_lo.dmar_subhandle_valid = true; 1333 msg->arch_addr_lo.dmar_format = true; 1334 msg->arch_addr_lo.dmar_index_0_14 = index & 0x7FFF; 1335 msg->arch_addr_lo.dmar_index_15 = !!(index & 0x8000); 1336 1337 msg->address_hi = X86_MSI_BASE_ADDRESS_HIGH; 1338 1339 msg->arch_data.dmar_subhandle = subhandle; 1340} 1341 1342static void intel_irq_remapping_prepare_irte(struct intel_ir_data *data, 1343 struct irq_cfg *irq_cfg, 1344 struct irq_alloc_info *info, 1345 int index, int sub_handle) 1346{ 1347 struct irte *irte = &data->irte_entry; 1348 1349 prepare_irte(irte, irq_cfg->vector, irq_cfg->dest_apicid); 1350 1351 switch (info->type) { 1352 case X86_IRQ_ALLOC_TYPE_IOAPIC: 1353 /* Set source-id of interrupt request */ 1354 set_ioapic_sid(irte, info->devid); 1355 apic_printk(APIC_VERBOSE, KERN_DEBUG "IOAPIC[%d]: Set IRTE entry (P:%d FPD:%d Dst_Mode:%d Redir_hint:%d Trig_Mode:%d Dlvry_Mode:%X Avail:%X Vector:%02X Dest:%08X SID:%04X SQ:%X SVT:%X)\n", 1356 info->devid, irte->present, irte->fpd, 1357 irte->dst_mode, irte->redir_hint, 1358 irte->trigger_mode, irte->dlvry_mode, 1359 irte->avail, irte->vector, irte->dest_id, 1360 irte->sid, irte->sq, irte->svt); 1361 sub_handle = info->ioapic.pin; 1362 break; 1363 case X86_IRQ_ALLOC_TYPE_HPET: 1364 set_hpet_sid(irte, info->devid); 1365 break; 1366 case X86_IRQ_ALLOC_TYPE_PCI_MSI: 1367 case X86_IRQ_ALLOC_TYPE_PCI_MSIX: 1368 if (posted_msi_supported()) { 1369 prepare_irte_posted(irte); 1370 data->irq_2_iommu.posted_msi = 1; 1371 } 1372 1373 set_msi_sid(irte, 1374 pci_real_dma_dev(msi_desc_to_pci_dev(info->desc))); 1375 break; 1376 default: 1377 BUG_ON(1); 1378 break; 1379 } 1380 fill_msi_msg(&data->msi_entry, index, sub_handle); 1381} 1382 1383static void intel_free_irq_resources(struct irq_domain *domain, 1384 unsigned int virq, unsigned int nr_irqs) 1385{ 1386 struct irq_data *irq_data; 1387 struct intel_ir_data *data; 1388 struct irq_2_iommu *irq_iommu; 1389 unsigned long flags; 1390 int i; 1391 for (i = 0; i < nr_irqs; i++) { 1392 irq_data = irq_domain_get_irq_data(domain, virq + i); 1393 if (irq_data && irq_data->chip_data) { 1394 data = irq_data->chip_data; 1395 irq_iommu = &data->irq_2_iommu; 1396 raw_spin_lock_irqsave(&irq_2_ir_lock, flags); 1397 clear_entries(irq_iommu); 1398 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); 1399 irq_domain_reset_irq_data(irq_data); 1400 kfree(data); 1401 } 1402 } 1403} 1404 1405static int intel_irq_remapping_alloc(struct irq_domain *domain, 1406 unsigned int virq, unsigned int nr_irqs, 1407 void *arg) 1408{ 1409 struct intel_iommu *iommu = domain->host_data; 1410 struct irq_alloc_info *info = arg; 1411 struct intel_ir_data *data, *ird; 1412 struct irq_data *irq_data; 1413 struct irq_cfg *irq_cfg; 1414 int i, ret, index; 1415 1416 if (!info || !iommu) 1417 return -EINVAL; 1418 if (nr_irqs > 1 && info->type != X86_IRQ_ALLOC_TYPE_PCI_MSI) 1419 return -EINVAL; 1420 1421 ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg); 1422 if (ret < 0) 1423 return ret; 1424 1425 ret = -ENOMEM; 1426 data = kzalloc(sizeof(*data), GFP_KERNEL); 1427 if (!data) 1428 goto out_free_parent; 1429 1430 index = alloc_irte(iommu, &data->irq_2_iommu, nr_irqs); 1431 if (index < 0) { 1432 pr_warn("Failed to allocate IRTE\n"); 1433 kfree(data); 1434 goto out_free_parent; 1435 } 1436 1437 for (i = 0; i < nr_irqs; i++) { 1438 irq_data = irq_domain_get_irq_data(domain, virq + i); 1439 irq_cfg = irqd_cfg(irq_data); 1440 if (!irq_data || !irq_cfg) { 1441 if (!i) 1442 kfree(data); 1443 ret = -EINVAL; 1444 goto out_free_data; 1445 } 1446 1447 if (i > 0) { 1448 ird = kzalloc(sizeof(*ird), GFP_KERNEL); 1449 if (!ird) 1450 goto out_free_data; 1451 /* Initialize the common data */ 1452 ird->irq_2_iommu = data->irq_2_iommu; 1453 ird->irq_2_iommu.sub_handle = i; 1454 } else { 1455 ird = data; 1456 } 1457 1458 irq_data->hwirq = (index << 16) + i; 1459 irq_data->chip_data = ird; 1460 if (posted_msi_supported() && 1461 ((info->type == X86_IRQ_ALLOC_TYPE_PCI_MSI) || 1462 (info->type == X86_IRQ_ALLOC_TYPE_PCI_MSIX))) 1463 irq_data->chip = &intel_ir_chip_post_msi; 1464 else 1465 irq_data->chip = &intel_ir_chip; 1466 intel_irq_remapping_prepare_irte(ird, irq_cfg, info, index, i); 1467 irq_set_status_flags(virq + i, IRQ_MOVE_PCNTXT); 1468 } 1469 return 0; 1470 1471out_free_data: 1472 intel_free_irq_resources(domain, virq, i); 1473out_free_parent: 1474 irq_domain_free_irqs_common(domain, virq, nr_irqs); 1475 return ret; 1476} 1477 1478static void intel_irq_remapping_free(struct irq_domain *domain, 1479 unsigned int virq, unsigned int nr_irqs) 1480{ 1481 intel_free_irq_resources(domain, virq, nr_irqs); 1482 irq_domain_free_irqs_common(domain, virq, nr_irqs); 1483} 1484 1485static int intel_irq_remapping_activate(struct irq_domain *domain, 1486 struct irq_data *irq_data, bool reserve) 1487{ 1488 intel_ir_reconfigure_irte(irq_data, true); 1489 return 0; 1490} 1491 1492static void intel_irq_remapping_deactivate(struct irq_domain *domain, 1493 struct irq_data *irq_data) 1494{ 1495 struct intel_ir_data *data = irq_data->chip_data; 1496 struct irte entry; 1497 1498 memset(&entry, 0, sizeof(entry)); 1499 modify_irte(&data->irq_2_iommu, &entry); 1500} 1501 1502static int intel_irq_remapping_select(struct irq_domain *d, 1503 struct irq_fwspec *fwspec, 1504 enum irq_domain_bus_token bus_token) 1505{ 1506 struct intel_iommu *iommu = NULL; 1507 1508 if (x86_fwspec_is_ioapic(fwspec)) 1509 iommu = map_ioapic_to_iommu(fwspec->param[0]); 1510 else if (x86_fwspec_is_hpet(fwspec)) 1511 iommu = map_hpet_to_iommu(fwspec->param[0]); 1512 1513 return iommu && d == iommu->ir_domain; 1514} 1515 1516static const struct irq_domain_ops intel_ir_domain_ops = { 1517 .select = intel_irq_remapping_select, 1518 .alloc = intel_irq_remapping_alloc, 1519 .free = intel_irq_remapping_free, 1520 .activate = intel_irq_remapping_activate, 1521 .deactivate = intel_irq_remapping_deactivate, 1522}; 1523 1524static const struct msi_parent_ops dmar_msi_parent_ops = { 1525 .supported_flags = X86_VECTOR_MSI_FLAGS_SUPPORTED | MSI_FLAG_MULTI_PCI_MSI, 1526 .prefix = "IR-", 1527 .init_dev_msi_info = msi_parent_init_dev_msi_info, 1528}; 1529 1530/* 1531 * Support of Interrupt Remapping Unit Hotplug 1532 */ 1533static int dmar_ir_add(struct dmar_drhd_unit *dmaru, struct intel_iommu *iommu) 1534{ 1535 int ret; 1536 int eim = x2apic_enabled(); 1537 1538 ret = intel_cap_audit(CAP_AUDIT_HOTPLUG_IRQR, iommu); 1539 if (ret) 1540 return ret; 1541 1542 if (eim && !ecap_eim_support(iommu->ecap)) { 1543 pr_info("DRHD %Lx: EIM not supported by DRHD, ecap %Lx\n", 1544 iommu->reg_phys, iommu->ecap); 1545 return -ENODEV; 1546 } 1547 1548 if (ir_parse_ioapic_hpet_scope(dmaru->hdr, iommu)) { 1549 pr_warn("DRHD %Lx: failed to parse managed IOAPIC/HPET\n", 1550 iommu->reg_phys); 1551 return -ENODEV; 1552 } 1553 1554 /* TODO: check all IOAPICs are covered by IOMMU */ 1555 1556 /* Setup Interrupt-remapping now. */ 1557 ret = intel_setup_irq_remapping(iommu); 1558 if (ret) { 1559 pr_err("Failed to setup irq remapping for %s\n", 1560 iommu->name); 1561 intel_teardown_irq_remapping(iommu); 1562 ir_remove_ioapic_hpet_scope(iommu); 1563 } else { 1564 iommu_enable_irq_remapping(iommu); 1565 } 1566 1567 return ret; 1568} 1569 1570int dmar_ir_hotplug(struct dmar_drhd_unit *dmaru, bool insert) 1571{ 1572 int ret = 0; 1573 struct intel_iommu *iommu = dmaru->iommu; 1574 1575 if (!irq_remapping_enabled) 1576 return 0; 1577 if (iommu == NULL) 1578 return -EINVAL; 1579 if (!ecap_ir_support(iommu->ecap)) 1580 return 0; 1581 if (irq_remapping_cap(IRQ_POSTING_CAP) && 1582 !cap_pi_support(iommu->cap)) 1583 return -EBUSY; 1584 1585 if (insert) { 1586 if (!iommu->ir_table) 1587 ret = dmar_ir_add(dmaru, iommu); 1588 } else { 1589 if (iommu->ir_table) { 1590 if (!bitmap_empty(iommu->ir_table->bitmap, 1591 INTR_REMAP_TABLE_ENTRIES)) { 1592 ret = -EBUSY; 1593 } else { 1594 iommu_disable_irq_remapping(iommu); 1595 intel_teardown_irq_remapping(iommu); 1596 ir_remove_ioapic_hpet_scope(iommu); 1597 } 1598 } 1599 } 1600 1601 return ret; 1602} 1603