intr_machdep.c revision 291644
1/*- 2 * Copyright (c) 2003 John Baldwin <jhb@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD: stable/10/sys/x86/x86/intr_machdep.c 291644 2015-12-02 10:23:54Z royger $ 27 */ 28 29/* 30 * Machine dependent interrupt code for x86. For x86, we have to 31 * deal with different PICs. Thus, we use the passed in vector to lookup 32 * an interrupt source associated with that vector. The interrupt source 33 * describes which PIC the source belongs to and includes methods to handle 34 * that source. 35 */ 36 37#include "opt_atpic.h" 38#include "opt_ddb.h" 39 40#include <sys/param.h> 41#include <sys/bus.h> 42#include <sys/interrupt.h> 43#include <sys/ktr.h> 44#include <sys/kernel.h> 45#include <sys/lock.h> 46#include <sys/mutex.h> 47#include <sys/proc.h> 48#include <sys/smp.h> 49#include <sys/syslog.h> 50#include <sys/systm.h> 51#include <machine/clock.h> 52#include <machine/intr_machdep.h> 53#include <machine/smp.h> 54#ifdef DDB 55#include <ddb/ddb.h> 56#endif 57 58#ifndef DEV_ATPIC 59#include <machine/segments.h> 60#include <machine/frame.h> 61#include <dev/ic/i8259.h> 62#include <x86/isa/icu.h> 63#ifdef PC98 64#include <pc98/cbus/cbus.h> 65#else 66#include <x86/isa/isa.h> 67#endif 68#endif 69 70#define MAX_STRAY_LOG 5 71 72typedef void (*mask_fn)(void *); 73 74static int intrcnt_index; 75static struct intsrc *interrupt_sources[NUM_IO_INTS]; 76static struct mtx intr_table_lock; 77static struct mtx intrcnt_lock; 78static TAILQ_HEAD(pics_head, pic) pics; 79 80#ifdef SMP 81static int assign_cpu; 82#endif 83 84u_long intrcnt[INTRCNT_COUNT]; 85char intrnames[INTRCNT_COUNT * (MAXCOMLEN + 1)]; 86size_t sintrcnt = sizeof(intrcnt); 87size_t sintrnames = sizeof(intrnames); 88 89static int intr_assign_cpu(void *arg, u_char cpu); 90static void intr_disable_src(void *arg); 91static void intr_init(void *__dummy); 92static int intr_pic_registered(struct pic *pic); 93static void intrcnt_setname(const char *name, int index); 94static void intrcnt_updatename(struct intsrc *is); 95static void intrcnt_register(struct intsrc *is); 96 97static int 98intr_pic_registered(struct pic *pic) 99{ 100 struct pic *p; 101 102 TAILQ_FOREACH(p, &pics, pics) { 103 if (p == pic) 104 return (1); 105 } 106 return (0); 107} 108 109/* 110 * Register a new interrupt controller (PIC). This is to support suspend 111 * and resume where we suspend/resume controllers rather than individual 112 * sources. This also allows controllers with no active sources (such as 113 * 8259As in a system using the APICs) to participate in suspend and resume. 114 */ 115int 116intr_register_pic(struct pic *pic) 117{ 118 int error; 119 120 mtx_lock(&intr_table_lock); 121 if (intr_pic_registered(pic)) 122 error = EBUSY; 123 else { 124 TAILQ_INSERT_TAIL(&pics, pic, pics); 125 error = 0; 126 } 127 mtx_unlock(&intr_table_lock); 128 return (error); 129} 130 131/* 132 * Register a new interrupt source with the global interrupt system. 133 * The global interrupts need to be disabled when this function is 134 * called. 135 */ 136int 137intr_register_source(struct intsrc *isrc) 138{ 139 int error, vector; 140 141 KASSERT(intr_pic_registered(isrc->is_pic), ("unregistered PIC")); 142 vector = isrc->is_pic->pic_vector(isrc); 143 if (interrupt_sources[vector] != NULL) 144 return (EEXIST); 145 error = intr_event_create(&isrc->is_event, isrc, 0, vector, 146 intr_disable_src, (mask_fn)isrc->is_pic->pic_enable_source, 147 (mask_fn)isrc->is_pic->pic_eoi_source, intr_assign_cpu, "irq%d:", 148 vector); 149 if (error) 150 return (error); 151 mtx_lock(&intr_table_lock); 152 if (interrupt_sources[vector] != NULL) { 153 mtx_unlock(&intr_table_lock); 154 intr_event_destroy(isrc->is_event); 155 return (EEXIST); 156 } 157 intrcnt_register(isrc); 158 interrupt_sources[vector] = isrc; 159 isrc->is_handlers = 0; 160 mtx_unlock(&intr_table_lock); 161 return (0); 162} 163 164struct intsrc * 165intr_lookup_source(int vector) 166{ 167 168 return (interrupt_sources[vector]); 169} 170 171int 172intr_add_handler(const char *name, int vector, driver_filter_t filter, 173 driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep) 174{ 175 struct intsrc *isrc; 176 int error; 177 178 isrc = intr_lookup_source(vector); 179 if (isrc == NULL) 180 return (EINVAL); 181 error = intr_event_add_handler(isrc->is_event, name, filter, handler, 182 arg, intr_priority(flags), flags, cookiep); 183 if (error == 0) { 184 mtx_lock(&intr_table_lock); 185 intrcnt_updatename(isrc); 186 isrc->is_handlers++; 187 if (isrc->is_handlers == 1) { 188 isrc->is_pic->pic_enable_intr(isrc); 189 isrc->is_pic->pic_enable_source(isrc); 190 } 191 mtx_unlock(&intr_table_lock); 192 } 193 return (error); 194} 195 196int 197intr_remove_handler(void *cookie) 198{ 199 struct intsrc *isrc; 200 int error, mtx_owned; 201 202 isrc = intr_handler_source(cookie); 203 error = intr_event_remove_handler(cookie); 204 if (error == 0) { 205 /* 206 * Recursion is needed here so PICs can remove interrupts 207 * while resuming. It was previously not possible due to 208 * intr_resume holding the intr_table_lock and 209 * intr_remove_handler recursing on it. 210 */ 211 mtx_owned = mtx_owned(&intr_table_lock); 212 if (mtx_owned == 0) 213 mtx_lock(&intr_table_lock); 214 isrc->is_handlers--; 215 if (isrc->is_handlers == 0) { 216 isrc->is_pic->pic_disable_source(isrc, PIC_NO_EOI); 217 isrc->is_pic->pic_disable_intr(isrc); 218 } 219 intrcnt_updatename(isrc); 220 if (mtx_owned == 0) 221 mtx_unlock(&intr_table_lock); 222 } 223 return (error); 224} 225 226int 227intr_config_intr(int vector, enum intr_trigger trig, enum intr_polarity pol) 228{ 229 struct intsrc *isrc; 230 231 isrc = intr_lookup_source(vector); 232 if (isrc == NULL) 233 return (EINVAL); 234 return (isrc->is_pic->pic_config_intr(isrc, trig, pol)); 235} 236 237static void 238intr_disable_src(void *arg) 239{ 240 struct intsrc *isrc; 241 242 isrc = arg; 243 isrc->is_pic->pic_disable_source(isrc, PIC_EOI); 244} 245 246void 247intr_execute_handlers(struct intsrc *isrc, struct trapframe *frame) 248{ 249 struct intr_event *ie; 250 int vector; 251 252 /* 253 * We count software interrupts when we process them. The 254 * code here follows previous practice, but there's an 255 * argument for counting hardware interrupts when they're 256 * processed too. 257 */ 258 (*isrc->is_count)++; 259 PCPU_INC(cnt.v_intr); 260 261 ie = isrc->is_event; 262 263 /* 264 * XXX: We assume that IRQ 0 is only used for the ISA timer 265 * device (clk). 266 */ 267 vector = isrc->is_pic->pic_vector(isrc); 268 if (vector == 0) 269 clkintr_pending = 1; 270 271 /* 272 * For stray interrupts, mask and EOI the source, bump the 273 * stray count, and log the condition. 274 */ 275 if (intr_event_handle(ie, frame) != 0) { 276 isrc->is_pic->pic_disable_source(isrc, PIC_EOI); 277 (*isrc->is_straycount)++; 278 if (*isrc->is_straycount < MAX_STRAY_LOG) 279 log(LOG_ERR, "stray irq%d\n", vector); 280 else if (*isrc->is_straycount == MAX_STRAY_LOG) 281 log(LOG_CRIT, 282 "too many stray irq %d's: not logging anymore\n", 283 vector); 284 } 285} 286 287void 288intr_resume(bool suspend_cancelled) 289{ 290 struct pic *pic; 291 292#ifndef DEV_ATPIC 293 atpic_reset(); 294#endif 295 mtx_lock(&intr_table_lock); 296 TAILQ_FOREACH(pic, &pics, pics) { 297 if (pic->pic_resume != NULL) 298 pic->pic_resume(pic, suspend_cancelled); 299 } 300 mtx_unlock(&intr_table_lock); 301} 302 303void 304intr_suspend(void) 305{ 306 struct pic *pic; 307 308 mtx_lock(&intr_table_lock); 309 TAILQ_FOREACH_REVERSE(pic, &pics, pics_head, pics) { 310 if (pic->pic_suspend != NULL) 311 pic->pic_suspend(pic); 312 } 313 mtx_unlock(&intr_table_lock); 314} 315 316static int 317intr_assign_cpu(void *arg, u_char cpu) 318{ 319#ifdef SMP 320 struct intsrc *isrc; 321 int error; 322 323 /* 324 * Don't do anything during early boot. We will pick up the 325 * assignment once the APs are started. 326 */ 327 if (assign_cpu && cpu != NOCPU) { 328 isrc = arg; 329 mtx_lock(&intr_table_lock); 330 error = isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[cpu]); 331 mtx_unlock(&intr_table_lock); 332 } else 333 error = 0; 334 return (error); 335#else 336 return (EOPNOTSUPP); 337#endif 338} 339 340static void 341intrcnt_setname(const char *name, int index) 342{ 343 344 snprintf(intrnames + (MAXCOMLEN + 1) * index, MAXCOMLEN + 1, "%-*s", 345 MAXCOMLEN, name); 346} 347 348static void 349intrcnt_updatename(struct intsrc *is) 350{ 351 352 intrcnt_setname(is->is_event->ie_fullname, is->is_index); 353} 354 355static void 356intrcnt_register(struct intsrc *is) 357{ 358 char straystr[MAXCOMLEN + 1]; 359 360 KASSERT(is->is_event != NULL, ("%s: isrc with no event", __func__)); 361 mtx_lock_spin(&intrcnt_lock); 362 is->is_index = intrcnt_index; 363 intrcnt_index += 2; 364 snprintf(straystr, MAXCOMLEN + 1, "stray irq%d", 365 is->is_pic->pic_vector(is)); 366 intrcnt_updatename(is); 367 is->is_count = &intrcnt[is->is_index]; 368 intrcnt_setname(straystr, is->is_index + 1); 369 is->is_straycount = &intrcnt[is->is_index + 1]; 370 mtx_unlock_spin(&intrcnt_lock); 371} 372 373void 374intrcnt_add(const char *name, u_long **countp) 375{ 376 377 mtx_lock_spin(&intrcnt_lock); 378 *countp = &intrcnt[intrcnt_index]; 379 intrcnt_setname(name, intrcnt_index); 380 intrcnt_index++; 381 mtx_unlock_spin(&intrcnt_lock); 382} 383 384static void 385intr_init(void *dummy __unused) 386{ 387 388 intrcnt_setname("???", 0); 389 intrcnt_index = 1; 390 TAILQ_INIT(&pics); 391 mtx_init(&intr_table_lock, "intr sources", NULL, MTX_DEF); 392 mtx_init(&intrcnt_lock, "intrcnt", NULL, MTX_SPIN); 393} 394SYSINIT(intr_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_init, NULL); 395 396#ifndef DEV_ATPIC 397/* Initialize the two 8259A's to a known-good shutdown state. */ 398void 399atpic_reset(void) 400{ 401 402 outb(IO_ICU1, ICW1_RESET | ICW1_IC4); 403 outb(IO_ICU1 + ICU_IMR_OFFSET, IDT_IO_INTS); 404 outb(IO_ICU1 + ICU_IMR_OFFSET, IRQ_MASK(ICU_SLAVEID)); 405 outb(IO_ICU1 + ICU_IMR_OFFSET, MASTER_MODE); 406 outb(IO_ICU1 + ICU_IMR_OFFSET, 0xff); 407 outb(IO_ICU1, OCW3_SEL | OCW3_RR); 408 409 outb(IO_ICU2, ICW1_RESET | ICW1_IC4); 410 outb(IO_ICU2 + ICU_IMR_OFFSET, IDT_IO_INTS + 8); 411 outb(IO_ICU2 + ICU_IMR_OFFSET, ICU_SLAVEID); 412 outb(IO_ICU2 + ICU_IMR_OFFSET, SLAVE_MODE); 413 outb(IO_ICU2 + ICU_IMR_OFFSET, 0xff); 414 outb(IO_ICU2, OCW3_SEL | OCW3_RR); 415} 416#endif 417 418/* Add a description to an active interrupt handler. */ 419int 420intr_describe(u_int vector, void *ih, const char *descr) 421{ 422 struct intsrc *isrc; 423 int error; 424 425 isrc = intr_lookup_source(vector); 426 if (isrc == NULL) 427 return (EINVAL); 428 error = intr_event_describe_handler(isrc->is_event, ih, descr); 429 if (error) 430 return (error); 431 intrcnt_updatename(isrc); 432 return (0); 433} 434 435#ifdef DDB 436/* 437 * Dump data about interrupt handlers 438 */ 439DB_SHOW_COMMAND(irqs, db_show_irqs) 440{ 441 struct intsrc **isrc; 442 int i, verbose; 443 444 if (strcmp(modif, "v") == 0) 445 verbose = 1; 446 else 447 verbose = 0; 448 isrc = interrupt_sources; 449 for (i = 0; i < NUM_IO_INTS && !db_pager_quit; i++, isrc++) 450 if (*isrc != NULL) 451 db_dump_intr_event((*isrc)->is_event, verbose); 452} 453#endif 454 455#ifdef SMP 456/* 457 * Support for balancing interrupt sources across CPUs. For now we just 458 * allocate CPUs round-robin. 459 */ 460 461static cpuset_t intr_cpus = CPUSET_T_INITIALIZER(0x1); 462static int current_cpu; 463 464/* 465 * Return the CPU that the next interrupt source should use. For now 466 * this just returns the next local APIC according to round-robin. 467 */ 468u_int 469intr_next_cpu(void) 470{ 471 u_int apic_id; 472 473 /* Leave all interrupts on the BSP during boot. */ 474 if (!assign_cpu) 475 return (PCPU_GET(apic_id)); 476 477 mtx_lock_spin(&icu_lock); 478 apic_id = cpu_apic_ids[current_cpu]; 479 do { 480 current_cpu++; 481 if (current_cpu > mp_maxid) 482 current_cpu = 0; 483 } while (!CPU_ISSET(current_cpu, &intr_cpus)); 484 mtx_unlock_spin(&icu_lock); 485 return (apic_id); 486} 487 488/* Attempt to bind the specified IRQ to the specified CPU. */ 489int 490intr_bind(u_int vector, u_char cpu) 491{ 492 struct intsrc *isrc; 493 494 isrc = intr_lookup_source(vector); 495 if (isrc == NULL) 496 return (EINVAL); 497 return (intr_event_bind(isrc->is_event, cpu)); 498} 499 500/* 501 * Add a CPU to our mask of valid CPUs that can be destinations of 502 * interrupts. 503 */ 504void 505intr_add_cpu(u_int cpu) 506{ 507 508 if (cpu >= MAXCPU) 509 panic("%s: Invalid CPU ID", __func__); 510 if (bootverbose) 511 printf("INTR: Adding local APIC %d as a target\n", 512 cpu_apic_ids[cpu]); 513 514 CPU_SET(cpu, &intr_cpus); 515} 516 517/* 518 * Distribute all the interrupt sources among the available CPUs once the 519 * AP's have been launched. 520 */ 521static void 522intr_shuffle_irqs(void *arg __unused) 523{ 524 struct intsrc *isrc; 525 int i; 526 527#ifdef XEN 528 /* 529 * Doesn't work yet 530 */ 531 return; 532#endif 533 534 /* Don't bother on UP. */ 535 if (mp_ncpus == 1) 536 return; 537 538 /* Round-robin assign a CPU to each enabled source. */ 539 mtx_lock(&intr_table_lock); 540 assign_cpu = 1; 541 for (i = 0; i < NUM_IO_INTS; i++) { 542 isrc = interrupt_sources[i]; 543 if (isrc != NULL && isrc->is_handlers > 0) { 544 /* 545 * If this event is already bound to a CPU, 546 * then assign the source to that CPU instead 547 * of picking one via round-robin. Note that 548 * this is careful to only advance the 549 * round-robin if the CPU assignment succeeds. 550 */ 551 if (isrc->is_event->ie_cpu != NOCPU) 552 (void)isrc->is_pic->pic_assign_cpu(isrc, 553 cpu_apic_ids[isrc->is_event->ie_cpu]); 554 else if (isrc->is_pic->pic_assign_cpu(isrc, 555 cpu_apic_ids[current_cpu]) == 0) 556 (void)intr_next_cpu(); 557 558 } 559 } 560 mtx_unlock(&intr_table_lock); 561} 562SYSINIT(intr_shuffle_irqs, SI_SUB_SMP, SI_ORDER_SECOND, intr_shuffle_irqs, 563 NULL); 564#else 565/* 566 * Always route interrupts to the current processor in the UP case. 567 */ 568u_int 569intr_next_cpu(void) 570{ 571 572 return (PCPU_GET(apic_id)); 573} 574#endif 575