1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar 4 * Copyright (C) 2005-2006, Thomas Gleixner, Russell King 5 * 6 * This file contains the interrupt descriptor management code. Detailed 7 * information is available in Documentation/core-api/genericirq.rst 8 * 9 */ 10#include <linux/irq.h> 11#include <linux/slab.h> 12#include <linux/export.h> 13#include <linux/interrupt.h> 14#include <linux/kernel_stat.h> 15#include <linux/maple_tree.h> 16#include <linux/irqdomain.h> 17#include <linux/sysfs.h> 18 19#include "internals.h" 20 21/* 22 * lockdep: we want to handle all irq_desc locks as a single lock-class: 23 */ 24static struct lock_class_key irq_desc_lock_class; 25 26#if defined(CONFIG_SMP) 27static int __init irq_affinity_setup(char *str) 28{ 29 alloc_bootmem_cpumask_var(&irq_default_affinity); 30 cpulist_parse(str, irq_default_affinity); 31 /* 32 * Set at least the boot cpu. We don't want to end up with 33 * bugreports caused by random commandline masks 34 */ 35 cpumask_set_cpu(smp_processor_id(), irq_default_affinity); 36 return 1; 37} 38__setup("irqaffinity=", irq_affinity_setup); 39 40static void __init init_irq_default_affinity(void) 41{ 42 if (!cpumask_available(irq_default_affinity)) 43 zalloc_cpumask_var(&irq_default_affinity, GFP_NOWAIT); 44 if (cpumask_empty(irq_default_affinity)) 45 cpumask_setall(irq_default_affinity); 46} 47#else 48static void __init init_irq_default_affinity(void) 49{ 50} 51#endif 52 53#ifdef CONFIG_SMP 54static int alloc_masks(struct irq_desc *desc, int node) 55{ 56 if (!zalloc_cpumask_var_node(&desc->irq_common_data.affinity, 57 GFP_KERNEL, node)) 58 return -ENOMEM; 59 60#ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK 61 if (!zalloc_cpumask_var_node(&desc->irq_common_data.effective_affinity, 62 GFP_KERNEL, node)) { 63 free_cpumask_var(desc->irq_common_data.affinity); 64 return -ENOMEM; 65 } 66#endif 67 68#ifdef CONFIG_GENERIC_PENDING_IRQ 69 if (!zalloc_cpumask_var_node(&desc->pending_mask, GFP_KERNEL, node)) { 70#ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK 71 free_cpumask_var(desc->irq_common_data.effective_affinity); 72#endif 73 free_cpumask_var(desc->irq_common_data.affinity); 74 return -ENOMEM; 75 } 76#endif 77 return 0; 78} 79 80static void desc_smp_init(struct irq_desc *desc, int node, 81 const struct cpumask *affinity) 82{ 83 if (!affinity) 84 affinity = irq_default_affinity; 85 cpumask_copy(desc->irq_common_data.affinity, affinity); 86 87#ifdef CONFIG_GENERIC_PENDING_IRQ 88 cpumask_clear(desc->pending_mask); 89#endif 90#ifdef CONFIG_NUMA 91 desc->irq_common_data.node = node; 92#endif 93} 94 95static void free_masks(struct irq_desc *desc) 96{ 97#ifdef CONFIG_GENERIC_PENDING_IRQ 98 free_cpumask_var(desc->pending_mask); 99#endif 100 free_cpumask_var(desc->irq_common_data.affinity); 101#ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK 102 free_cpumask_var(desc->irq_common_data.effective_affinity); 103#endif 104} 105 106#else 107static inline int 108alloc_masks(struct irq_desc *desc, int node) { return 0; } 109static inline void 110desc_smp_init(struct irq_desc *desc, int node, const struct cpumask *affinity) { } 111static inline void free_masks(struct irq_desc *desc) { } 112#endif 113 114static void desc_set_defaults(unsigned int irq, struct irq_desc *desc, int node, 115 const struct cpumask *affinity, struct module *owner) 116{ 117 int cpu; 118 119 desc->irq_common_data.handler_data = NULL; 120 desc->irq_common_data.msi_desc = NULL; 121 122 desc->irq_data.common = &desc->irq_common_data; 123 desc->irq_data.irq = irq; 124 desc->irq_data.chip = &no_irq_chip; 125 desc->irq_data.chip_data = NULL; 126 irq_settings_clr_and_set(desc, ~0, _IRQ_DEFAULT_INIT_FLAGS); 127 irqd_set(&desc->irq_data, IRQD_IRQ_DISABLED); 128 irqd_set(&desc->irq_data, IRQD_IRQ_MASKED); 129 desc->handle_irq = handle_bad_irq; 130 desc->depth = 1; 131 desc->irq_count = 0; 132 desc->irqs_unhandled = 0; 133 desc->tot_count = 0; 134 desc->name = NULL; 135 desc->owner = owner; 136 for_each_possible_cpu(cpu) 137 *per_cpu_ptr(desc->kstat_irqs, cpu) = (struct irqstat) { }; 138 desc_smp_init(desc, node, affinity); 139} 140 141int nr_irqs = NR_IRQS; 142EXPORT_SYMBOL_GPL(nr_irqs); 143 144static DEFINE_MUTEX(sparse_irq_lock); 145static struct maple_tree sparse_irqs = MTREE_INIT_EXT(sparse_irqs, 146 MT_FLAGS_ALLOC_RANGE | 147 MT_FLAGS_LOCK_EXTERN | 148 MT_FLAGS_USE_RCU, 149 sparse_irq_lock); 150 151static int irq_find_free_area(unsigned int from, unsigned int cnt) 152{ 153 MA_STATE(mas, &sparse_irqs, 0, 0); 154 155 if (mas_empty_area(&mas, from, MAX_SPARSE_IRQS, cnt)) 156 return -ENOSPC; 157 return mas.index; 158} 159 160static unsigned int irq_find_at_or_after(unsigned int offset) 161{ 162 unsigned long index = offset; 163 struct irq_desc *desc; 164 165 guard(rcu)(); 166 desc = mt_find(&sparse_irqs, &index, nr_irqs); 167 168 return desc ? irq_desc_get_irq(desc) : nr_irqs; 169} 170 171static void irq_insert_desc(unsigned int irq, struct irq_desc *desc) 172{ 173 MA_STATE(mas, &sparse_irqs, irq, irq); 174 WARN_ON(mas_store_gfp(&mas, desc, GFP_KERNEL) != 0); 175} 176 177static void delete_irq_desc(unsigned int irq) 178{ 179 MA_STATE(mas, &sparse_irqs, irq, irq); 180 mas_erase(&mas); 181} 182 183#ifdef CONFIG_SPARSE_IRQ 184static const struct kobj_type irq_kobj_type; 185#endif 186 187static int init_desc(struct irq_desc *desc, int irq, int node, 188 unsigned int flags, 189 const struct cpumask *affinity, 190 struct module *owner) 191{ 192 desc->kstat_irqs = alloc_percpu(struct irqstat); 193 if (!desc->kstat_irqs) 194 return -ENOMEM; 195 196 if (alloc_masks(desc, node)) { 197 free_percpu(desc->kstat_irqs); 198 return -ENOMEM; 199 } 200 201 raw_spin_lock_init(&desc->lock); 202 lockdep_set_class(&desc->lock, &irq_desc_lock_class); 203 mutex_init(&desc->request_mutex); 204 init_waitqueue_head(&desc->wait_for_threads); 205 desc_set_defaults(irq, desc, node, affinity, owner); 206 irqd_set(&desc->irq_data, flags); 207 irq_resend_init(desc); 208#ifdef CONFIG_SPARSE_IRQ 209 kobject_init(&desc->kobj, &irq_kobj_type); 210 init_rcu_head(&desc->rcu); 211#endif 212 213 return 0; 214} 215 216#ifdef CONFIG_SPARSE_IRQ 217 218static void irq_kobj_release(struct kobject *kobj); 219 220#ifdef CONFIG_SYSFS 221static struct kobject *irq_kobj_base; 222 223#define IRQ_ATTR_RO(_name) \ 224static struct kobj_attribute _name##_attr = __ATTR_RO(_name) 225 226static ssize_t per_cpu_count_show(struct kobject *kobj, 227 struct kobj_attribute *attr, char *buf) 228{ 229 struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj); 230 ssize_t ret = 0; 231 char *p = ""; 232 int cpu; 233 234 for_each_possible_cpu(cpu) { 235 unsigned int c = irq_desc_kstat_cpu(desc, cpu); 236 237 ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s%u", p, c); 238 p = ","; 239 } 240 241 ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n"); 242 return ret; 243} 244IRQ_ATTR_RO(per_cpu_count); 245 246static ssize_t chip_name_show(struct kobject *kobj, 247 struct kobj_attribute *attr, char *buf) 248{ 249 struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj); 250 ssize_t ret = 0; 251 252 raw_spin_lock_irq(&desc->lock); 253 if (desc->irq_data.chip && desc->irq_data.chip->name) { 254 ret = scnprintf(buf, PAGE_SIZE, "%s\n", 255 desc->irq_data.chip->name); 256 } 257 raw_spin_unlock_irq(&desc->lock); 258 259 return ret; 260} 261IRQ_ATTR_RO(chip_name); 262 263static ssize_t hwirq_show(struct kobject *kobj, 264 struct kobj_attribute *attr, char *buf) 265{ 266 struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj); 267 ssize_t ret = 0; 268 269 raw_spin_lock_irq(&desc->lock); 270 if (desc->irq_data.domain) 271 ret = sprintf(buf, "%lu\n", desc->irq_data.hwirq); 272 raw_spin_unlock_irq(&desc->lock); 273 274 return ret; 275} 276IRQ_ATTR_RO(hwirq); 277 278static ssize_t type_show(struct kobject *kobj, 279 struct kobj_attribute *attr, char *buf) 280{ 281 struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj); 282 ssize_t ret = 0; 283 284 raw_spin_lock_irq(&desc->lock); 285 ret = sprintf(buf, "%s\n", 286 irqd_is_level_type(&desc->irq_data) ? "level" : "edge"); 287 raw_spin_unlock_irq(&desc->lock); 288 289 return ret; 290 291} 292IRQ_ATTR_RO(type); 293 294static ssize_t wakeup_show(struct kobject *kobj, 295 struct kobj_attribute *attr, char *buf) 296{ 297 struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj); 298 ssize_t ret = 0; 299 300 raw_spin_lock_irq(&desc->lock); 301 ret = sprintf(buf, "%s\n", 302 irqd_is_wakeup_set(&desc->irq_data) ? "enabled" : "disabled"); 303 raw_spin_unlock_irq(&desc->lock); 304 305 return ret; 306 307} 308IRQ_ATTR_RO(wakeup); 309 310static ssize_t name_show(struct kobject *kobj, 311 struct kobj_attribute *attr, char *buf) 312{ 313 struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj); 314 ssize_t ret = 0; 315 316 raw_spin_lock_irq(&desc->lock); 317 if (desc->name) 318 ret = scnprintf(buf, PAGE_SIZE, "%s\n", desc->name); 319 raw_spin_unlock_irq(&desc->lock); 320 321 return ret; 322} 323IRQ_ATTR_RO(name); 324 325static ssize_t actions_show(struct kobject *kobj, 326 struct kobj_attribute *attr, char *buf) 327{ 328 struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj); 329 struct irqaction *action; 330 ssize_t ret = 0; 331 char *p = ""; 332 333 raw_spin_lock_irq(&desc->lock); 334 for_each_action_of_desc(desc, action) { 335 ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s%s", 336 p, action->name); 337 p = ","; 338 } 339 raw_spin_unlock_irq(&desc->lock); 340 341 if (ret) 342 ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n"); 343 344 return ret; 345} 346IRQ_ATTR_RO(actions); 347 348static struct attribute *irq_attrs[] = { 349 &per_cpu_count_attr.attr, 350 &chip_name_attr.attr, 351 &hwirq_attr.attr, 352 &type_attr.attr, 353 &wakeup_attr.attr, 354 &name_attr.attr, 355 &actions_attr.attr, 356 NULL 357}; 358ATTRIBUTE_GROUPS(irq); 359 360static const struct kobj_type irq_kobj_type = { 361 .release = irq_kobj_release, 362 .sysfs_ops = &kobj_sysfs_ops, 363 .default_groups = irq_groups, 364}; 365 366static void irq_sysfs_add(int irq, struct irq_desc *desc) 367{ 368 if (irq_kobj_base) { 369 /* 370 * Continue even in case of failure as this is nothing 371 * crucial and failures in the late irq_sysfs_init() 372 * cannot be rolled back. 373 */ 374 if (kobject_add(&desc->kobj, irq_kobj_base, "%d", irq)) 375 pr_warn("Failed to add kobject for irq %d\n", irq); 376 else 377 desc->istate |= IRQS_SYSFS; 378 } 379} 380 381static void irq_sysfs_del(struct irq_desc *desc) 382{ 383 /* 384 * Only invoke kobject_del() when kobject_add() was successfully 385 * invoked for the descriptor. This covers both early boot, where 386 * sysfs is not initialized yet, and the case of a failed 387 * kobject_add() invocation. 388 */ 389 if (desc->istate & IRQS_SYSFS) 390 kobject_del(&desc->kobj); 391} 392 393static int __init irq_sysfs_init(void) 394{ 395 struct irq_desc *desc; 396 int irq; 397 398 /* Prevent concurrent irq alloc/free */ 399 irq_lock_sparse(); 400 401 irq_kobj_base = kobject_create_and_add("irq", kernel_kobj); 402 if (!irq_kobj_base) { 403 irq_unlock_sparse(); 404 return -ENOMEM; 405 } 406 407 /* Add the already allocated interrupts */ 408 for_each_irq_desc(irq, desc) 409 irq_sysfs_add(irq, desc); 410 irq_unlock_sparse(); 411 412 return 0; 413} 414postcore_initcall(irq_sysfs_init); 415 416#else /* !CONFIG_SYSFS */ 417 418static const struct kobj_type irq_kobj_type = { 419 .release = irq_kobj_release, 420}; 421 422static void irq_sysfs_add(int irq, struct irq_desc *desc) {} 423static void irq_sysfs_del(struct irq_desc *desc) {} 424 425#endif /* CONFIG_SYSFS */ 426 427struct irq_desc *irq_to_desc(unsigned int irq) 428{ 429 return mtree_load(&sparse_irqs, irq); 430} 431#ifdef CONFIG_KVM_BOOK3S_64_HV_MODULE 432EXPORT_SYMBOL_GPL(irq_to_desc); 433#endif 434 435void irq_lock_sparse(void) 436{ 437 mutex_lock(&sparse_irq_lock); 438} 439 440void irq_unlock_sparse(void) 441{ 442 mutex_unlock(&sparse_irq_lock); 443} 444 445static struct irq_desc *alloc_desc(int irq, int node, unsigned int flags, 446 const struct cpumask *affinity, 447 struct module *owner) 448{ 449 struct irq_desc *desc; 450 int ret; 451 452 desc = kzalloc_node(sizeof(*desc), GFP_KERNEL, node); 453 if (!desc) 454 return NULL; 455 456 ret = init_desc(desc, irq, node, flags, affinity, owner); 457 if (unlikely(ret)) { 458 kfree(desc); 459 return NULL; 460 } 461 462 return desc; 463} 464 465static void irq_kobj_release(struct kobject *kobj) 466{ 467 struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj); 468 469 free_masks(desc); 470 free_percpu(desc->kstat_irqs); 471 kfree(desc); 472} 473 474static void delayed_free_desc(struct rcu_head *rhp) 475{ 476 struct irq_desc *desc = container_of(rhp, struct irq_desc, rcu); 477 478 kobject_put(&desc->kobj); 479} 480 481static void free_desc(unsigned int irq) 482{ 483 struct irq_desc *desc = irq_to_desc(irq); 484 485 irq_remove_debugfs_entry(desc); 486 unregister_irq_proc(irq, desc); 487 488 /* 489 * sparse_irq_lock protects also show_interrupts() and 490 * kstat_irq_usr(). Once we deleted the descriptor from the 491 * sparse tree we can free it. Access in proc will fail to 492 * lookup the descriptor. 493 * 494 * The sysfs entry must be serialized against a concurrent 495 * irq_sysfs_init() as well. 496 */ 497 irq_sysfs_del(desc); 498 delete_irq_desc(irq); 499 500 /* 501 * We free the descriptor, masks and stat fields via RCU. That 502 * allows demultiplex interrupts to do rcu based management of 503 * the child interrupts. 504 * This also allows us to use rcu in kstat_irqs_usr(). 505 */ 506 call_rcu(&desc->rcu, delayed_free_desc); 507} 508 509static int alloc_descs(unsigned int start, unsigned int cnt, int node, 510 const struct irq_affinity_desc *affinity, 511 struct module *owner) 512{ 513 struct irq_desc *desc; 514 int i; 515 516 /* Validate affinity mask(s) */ 517 if (affinity) { 518 for (i = 0; i < cnt; i++) { 519 if (cpumask_empty(&affinity[i].mask)) 520 return -EINVAL; 521 } 522 } 523 524 for (i = 0; i < cnt; i++) { 525 const struct cpumask *mask = NULL; 526 unsigned int flags = 0; 527 528 if (affinity) { 529 if (affinity->is_managed) { 530 flags = IRQD_AFFINITY_MANAGED | 531 IRQD_MANAGED_SHUTDOWN; 532 } 533 mask = &affinity->mask; 534 node = cpu_to_node(cpumask_first(mask)); 535 affinity++; 536 } 537 538 desc = alloc_desc(start + i, node, flags, mask, owner); 539 if (!desc) 540 goto err; 541 irq_insert_desc(start + i, desc); 542 irq_sysfs_add(start + i, desc); 543 irq_add_debugfs_entry(start + i, desc); 544 } 545 return start; 546 547err: 548 for (i--; i >= 0; i--) 549 free_desc(start + i); 550 return -ENOMEM; 551} 552 553static int irq_expand_nr_irqs(unsigned int nr) 554{ 555 if (nr > MAX_SPARSE_IRQS) 556 return -ENOMEM; 557 nr_irqs = nr; 558 return 0; 559} 560 561int __init early_irq_init(void) 562{ 563 int i, initcnt, node = first_online_node; 564 struct irq_desc *desc; 565 566 init_irq_default_affinity(); 567 568 /* Let arch update nr_irqs and return the nr of preallocated irqs */ 569 initcnt = arch_probe_nr_irqs(); 570 printk(KERN_INFO "NR_IRQS: %d, nr_irqs: %d, preallocated irqs: %d\n", 571 NR_IRQS, nr_irqs, initcnt); 572 573 if (WARN_ON(nr_irqs > MAX_SPARSE_IRQS)) 574 nr_irqs = MAX_SPARSE_IRQS; 575 576 if (WARN_ON(initcnt > MAX_SPARSE_IRQS)) 577 initcnt = MAX_SPARSE_IRQS; 578 579 if (initcnt > nr_irqs) 580 nr_irqs = initcnt; 581 582 for (i = 0; i < initcnt; i++) { 583 desc = alloc_desc(i, node, 0, NULL, NULL); 584 irq_insert_desc(i, desc); 585 } 586 return arch_early_irq_init(); 587} 588 589#else /* !CONFIG_SPARSE_IRQ */ 590 591struct irq_desc irq_desc[NR_IRQS] __cacheline_aligned_in_smp = { 592 [0 ... NR_IRQS-1] = { 593 .handle_irq = handle_bad_irq, 594 .depth = 1, 595 .lock = __RAW_SPIN_LOCK_UNLOCKED(irq_desc->lock), 596 } 597}; 598 599int __init early_irq_init(void) 600{ 601 int count, i, node = first_online_node; 602 int ret; 603 604 init_irq_default_affinity(); 605 606 printk(KERN_INFO "NR_IRQS: %d\n", NR_IRQS); 607 608 count = ARRAY_SIZE(irq_desc); 609 610 for (i = 0; i < count; i++) { 611 ret = init_desc(irq_desc + i, i, node, 0, NULL, NULL); 612 if (unlikely(ret)) 613 goto __free_desc_res; 614 } 615 616 return arch_early_irq_init(); 617 618__free_desc_res: 619 while (--i >= 0) { 620 free_masks(irq_desc + i); 621 free_percpu(irq_desc[i].kstat_irqs); 622 } 623 624 return ret; 625} 626 627struct irq_desc *irq_to_desc(unsigned int irq) 628{ 629 return (irq < NR_IRQS) ? irq_desc + irq : NULL; 630} 631EXPORT_SYMBOL(irq_to_desc); 632 633static void free_desc(unsigned int irq) 634{ 635 struct irq_desc *desc = irq_to_desc(irq); 636 unsigned long flags; 637 638 raw_spin_lock_irqsave(&desc->lock, flags); 639 desc_set_defaults(irq, desc, irq_desc_get_node(desc), NULL, NULL); 640 raw_spin_unlock_irqrestore(&desc->lock, flags); 641 delete_irq_desc(irq); 642} 643 644static inline int alloc_descs(unsigned int start, unsigned int cnt, int node, 645 const struct irq_affinity_desc *affinity, 646 struct module *owner) 647{ 648 u32 i; 649 650 for (i = 0; i < cnt; i++) { 651 struct irq_desc *desc = irq_to_desc(start + i); 652 653 desc->owner = owner; 654 irq_insert_desc(start + i, desc); 655 } 656 return start; 657} 658 659static int irq_expand_nr_irqs(unsigned int nr) 660{ 661 return -ENOMEM; 662} 663 664void irq_mark_irq(unsigned int irq) 665{ 666 mutex_lock(&sparse_irq_lock); 667 irq_insert_desc(irq, irq_desc + irq); 668 mutex_unlock(&sparse_irq_lock); 669} 670 671#ifdef CONFIG_GENERIC_IRQ_LEGACY 672void irq_init_desc(unsigned int irq) 673{ 674 free_desc(irq); 675} 676#endif 677 678#endif /* !CONFIG_SPARSE_IRQ */ 679 680int handle_irq_desc(struct irq_desc *desc) 681{ 682 struct irq_data *data; 683 684 if (!desc) 685 return -EINVAL; 686 687 data = irq_desc_get_irq_data(desc); 688 if (WARN_ON_ONCE(!in_hardirq() && handle_enforce_irqctx(data))) 689 return -EPERM; 690 691 generic_handle_irq_desc(desc); 692 return 0; 693} 694 695/** 696 * generic_handle_irq - Invoke the handler for a particular irq 697 * @irq: The irq number to handle 698 * 699 * Returns: 0 on success, or -EINVAL if conversion has failed 700 * 701 * This function must be called from an IRQ context with irq regs 702 * initialized. 703 */ 704int generic_handle_irq(unsigned int irq) 705{ 706 return handle_irq_desc(irq_to_desc(irq)); 707} 708EXPORT_SYMBOL_GPL(generic_handle_irq); 709 710/** 711 * generic_handle_irq_safe - Invoke the handler for a particular irq from any 712 * context. 713 * @irq: The irq number to handle 714 * 715 * Returns: 0 on success, a negative value on error. 716 * 717 * This function can be called from any context (IRQ or process context). It 718 * will report an error if not invoked from IRQ context and the irq has been 719 * marked to enforce IRQ-context only. 720 */ 721int generic_handle_irq_safe(unsigned int irq) 722{ 723 unsigned long flags; 724 int ret; 725 726 local_irq_save(flags); 727 ret = handle_irq_desc(irq_to_desc(irq)); 728 local_irq_restore(flags); 729 return ret; 730} 731EXPORT_SYMBOL_GPL(generic_handle_irq_safe); 732 733#ifdef CONFIG_IRQ_DOMAIN 734/** 735 * generic_handle_domain_irq - Invoke the handler for a HW irq belonging 736 * to a domain. 737 * @domain: The domain where to perform the lookup 738 * @hwirq: The HW irq number to convert to a logical one 739 * 740 * Returns: 0 on success, or -EINVAL if conversion has failed 741 * 742 * This function must be called from an IRQ context with irq regs 743 * initialized. 744 */ 745int generic_handle_domain_irq(struct irq_domain *domain, unsigned int hwirq) 746{ 747 return handle_irq_desc(irq_resolve_mapping(domain, hwirq)); 748} 749EXPORT_SYMBOL_GPL(generic_handle_domain_irq); 750 751 /** 752 * generic_handle_irq_safe - Invoke the handler for a HW irq belonging 753 * to a domain from any context. 754 * @domain: The domain where to perform the lookup 755 * @hwirq: The HW irq number to convert to a logical one 756 * 757 * Returns: 0 on success, a negative value on error. 758 * 759 * This function can be called from any context (IRQ or process 760 * context). If the interrupt is marked as 'enforce IRQ-context only' then 761 * the function must be invoked from hard interrupt context. 762 */ 763int generic_handle_domain_irq_safe(struct irq_domain *domain, unsigned int hwirq) 764{ 765 unsigned long flags; 766 int ret; 767 768 local_irq_save(flags); 769 ret = handle_irq_desc(irq_resolve_mapping(domain, hwirq)); 770 local_irq_restore(flags); 771 return ret; 772} 773EXPORT_SYMBOL_GPL(generic_handle_domain_irq_safe); 774 775/** 776 * generic_handle_domain_nmi - Invoke the handler for a HW nmi belonging 777 * to a domain. 778 * @domain: The domain where to perform the lookup 779 * @hwirq: The HW irq number to convert to a logical one 780 * 781 * Returns: 0 on success, or -EINVAL if conversion has failed 782 * 783 * This function must be called from an NMI context with irq regs 784 * initialized. 785 **/ 786int generic_handle_domain_nmi(struct irq_domain *domain, unsigned int hwirq) 787{ 788 WARN_ON_ONCE(!in_nmi()); 789 return handle_irq_desc(irq_resolve_mapping(domain, hwirq)); 790} 791#endif 792 793/* Dynamic interrupt handling */ 794 795/** 796 * irq_free_descs - free irq descriptors 797 * @from: Start of descriptor range 798 * @cnt: Number of consecutive irqs to free 799 */ 800void irq_free_descs(unsigned int from, unsigned int cnt) 801{ 802 int i; 803 804 if (from >= nr_irqs || (from + cnt) > nr_irqs) 805 return; 806 807 mutex_lock(&sparse_irq_lock); 808 for (i = 0; i < cnt; i++) 809 free_desc(from + i); 810 811 mutex_unlock(&sparse_irq_lock); 812} 813EXPORT_SYMBOL_GPL(irq_free_descs); 814 815/** 816 * __irq_alloc_descs - allocate and initialize a range of irq descriptors 817 * @irq: Allocate for specific irq number if irq >= 0 818 * @from: Start the search from this irq number 819 * @cnt: Number of consecutive irqs to allocate. 820 * @node: Preferred node on which the irq descriptor should be allocated 821 * @owner: Owning module (can be NULL) 822 * @affinity: Optional pointer to an affinity mask array of size @cnt which 823 * hints where the irq descriptors should be allocated and which 824 * default affinities to use 825 * 826 * Returns the first irq number or error code 827 */ 828int __ref 829__irq_alloc_descs(int irq, unsigned int from, unsigned int cnt, int node, 830 struct module *owner, const struct irq_affinity_desc *affinity) 831{ 832 int start, ret; 833 834 if (!cnt) 835 return -EINVAL; 836 837 if (irq >= 0) { 838 if (from > irq) 839 return -EINVAL; 840 from = irq; 841 } else { 842 /* 843 * For interrupts which are freely allocated the 844 * architecture can force a lower bound to the @from 845 * argument. x86 uses this to exclude the GSI space. 846 */ 847 from = arch_dynirq_lower_bound(from); 848 } 849 850 mutex_lock(&sparse_irq_lock); 851 852 start = irq_find_free_area(from, cnt); 853 ret = -EEXIST; 854 if (irq >=0 && start != irq) 855 goto unlock; 856 857 if (start + cnt > nr_irqs) { 858 ret = irq_expand_nr_irqs(start + cnt); 859 if (ret) 860 goto unlock; 861 } 862 ret = alloc_descs(start, cnt, node, affinity, owner); 863unlock: 864 mutex_unlock(&sparse_irq_lock); 865 return ret; 866} 867EXPORT_SYMBOL_GPL(__irq_alloc_descs); 868 869/** 870 * irq_get_next_irq - get next allocated irq number 871 * @offset: where to start the search 872 * 873 * Returns next irq number after offset or nr_irqs if none is found. 874 */ 875unsigned int irq_get_next_irq(unsigned int offset) 876{ 877 return irq_find_at_or_after(offset); 878} 879 880struct irq_desc * 881__irq_get_desc_lock(unsigned int irq, unsigned long *flags, bool bus, 882 unsigned int check) 883{ 884 struct irq_desc *desc = irq_to_desc(irq); 885 886 if (desc) { 887 if (check & _IRQ_DESC_CHECK) { 888 if ((check & _IRQ_DESC_PERCPU) && 889 !irq_settings_is_per_cpu_devid(desc)) 890 return NULL; 891 892 if (!(check & _IRQ_DESC_PERCPU) && 893 irq_settings_is_per_cpu_devid(desc)) 894 return NULL; 895 } 896 897 if (bus) 898 chip_bus_lock(desc); 899 raw_spin_lock_irqsave(&desc->lock, *flags); 900 } 901 return desc; 902} 903 904void __irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags, bool bus) 905 __releases(&desc->lock) 906{ 907 raw_spin_unlock_irqrestore(&desc->lock, flags); 908 if (bus) 909 chip_bus_sync_unlock(desc); 910} 911 912int irq_set_percpu_devid_partition(unsigned int irq, 913 const struct cpumask *affinity) 914{ 915 struct irq_desc *desc = irq_to_desc(irq); 916 917 if (!desc || desc->percpu_enabled) 918 return -EINVAL; 919 920 desc->percpu_enabled = kzalloc(sizeof(*desc->percpu_enabled), GFP_KERNEL); 921 922 if (!desc->percpu_enabled) 923 return -ENOMEM; 924 925 desc->percpu_affinity = affinity ? : cpu_possible_mask; 926 927 irq_set_percpu_devid_flags(irq); 928 return 0; 929} 930 931int irq_set_percpu_devid(unsigned int irq) 932{ 933 return irq_set_percpu_devid_partition(irq, NULL); 934} 935 936int irq_get_percpu_devid_partition(unsigned int irq, struct cpumask *affinity) 937{ 938 struct irq_desc *desc = irq_to_desc(irq); 939 940 if (!desc || !desc->percpu_enabled) 941 return -EINVAL; 942 943 if (affinity) 944 cpumask_copy(affinity, desc->percpu_affinity); 945 946 return 0; 947} 948EXPORT_SYMBOL_GPL(irq_get_percpu_devid_partition); 949 950void kstat_incr_irq_this_cpu(unsigned int irq) 951{ 952 kstat_incr_irqs_this_cpu(irq_to_desc(irq)); 953} 954 955/** 956 * kstat_irqs_cpu - Get the statistics for an interrupt on a cpu 957 * @irq: The interrupt number 958 * @cpu: The cpu number 959 * 960 * Returns the sum of interrupt counts on @cpu since boot for 961 * @irq. The caller must ensure that the interrupt is not removed 962 * concurrently. 963 */ 964unsigned int kstat_irqs_cpu(unsigned int irq, int cpu) 965{ 966 struct irq_desc *desc = irq_to_desc(irq); 967 968 return desc && desc->kstat_irqs ? per_cpu(desc->kstat_irqs->cnt, cpu) : 0; 969} 970 971unsigned int kstat_irqs_desc(struct irq_desc *desc, const struct cpumask *cpumask) 972{ 973 unsigned int sum = 0; 974 int cpu; 975 976 if (!irq_settings_is_per_cpu_devid(desc) && 977 !irq_settings_is_per_cpu(desc) && 978 !irq_is_nmi(desc)) 979 return data_race(desc->tot_count); 980 981 for_each_cpu(cpu, cpumask) 982 sum += data_race(per_cpu(desc->kstat_irqs->cnt, cpu)); 983 return sum; 984} 985 986static unsigned int kstat_irqs(unsigned int irq) 987{ 988 struct irq_desc *desc = irq_to_desc(irq); 989 990 if (!desc || !desc->kstat_irqs) 991 return 0; 992 return kstat_irqs_desc(desc, cpu_possible_mask); 993} 994 995#ifdef CONFIG_GENERIC_IRQ_STAT_SNAPSHOT 996 997void kstat_snapshot_irqs(void) 998{ 999 struct irq_desc *desc; 1000 unsigned int irq; 1001 1002 for_each_irq_desc(irq, desc) { 1003 if (!desc->kstat_irqs) 1004 continue; 1005 this_cpu_write(desc->kstat_irqs->ref, this_cpu_read(desc->kstat_irqs->cnt)); 1006 } 1007} 1008 1009unsigned int kstat_get_irq_since_snapshot(unsigned int irq) 1010{ 1011 struct irq_desc *desc = irq_to_desc(irq); 1012 1013 if (!desc || !desc->kstat_irqs) 1014 return 0; 1015 return this_cpu_read(desc->kstat_irqs->cnt) - this_cpu_read(desc->kstat_irqs->ref); 1016} 1017 1018#endif 1019 1020/** 1021 * kstat_irqs_usr - Get the statistics for an interrupt from thread context 1022 * @irq: The interrupt number 1023 * 1024 * Returns the sum of interrupt counts on all cpus since boot for @irq. 1025 * 1026 * It uses rcu to protect the access since a concurrent removal of an 1027 * interrupt descriptor is observing an rcu grace period before 1028 * delayed_free_desc()/irq_kobj_release(). 1029 */ 1030unsigned int kstat_irqs_usr(unsigned int irq) 1031{ 1032 unsigned int sum; 1033 1034 rcu_read_lock(); 1035 sum = kstat_irqs(irq); 1036 rcu_read_unlock(); 1037 return sum; 1038} 1039 1040#ifdef CONFIG_LOCKDEP 1041void __irq_set_lockdep_class(unsigned int irq, struct lock_class_key *lock_class, 1042 struct lock_class_key *request_class) 1043{ 1044 struct irq_desc *desc = irq_to_desc(irq); 1045 1046 if (desc) { 1047 lockdep_set_class(&desc->lock, lock_class); 1048 lockdep_set_class(&desc->request_mutex, request_class); 1049 } 1050} 1051EXPORT_SYMBOL_GPL(__irq_set_lockdep_class); 1052#endif 1053