intr_machdep.c revision 307244
1/*- 2 * Copyright (c) 2003 John Baldwin <jhb@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD: stable/10/sys/x86/x86/intr_machdep.c 307244 2016-10-14 02:03:53Z sephe $ 27 */ 28 29/* 30 * Machine dependent interrupt code for x86. For x86, we have to 31 * deal with different PICs. Thus, we use the passed in vector to lookup 32 * an interrupt source associated with that vector. The interrupt source 33 * describes which PIC the source belongs to and includes methods to handle 34 * that source. 35 */ 36 37#include "opt_atpic.h" 38#include "opt_ddb.h" 39 40#include <sys/param.h> 41#include <sys/bus.h> 42#include <sys/interrupt.h> 43#include <sys/ktr.h> 44#include <sys/kernel.h> 45#include <sys/lock.h> 46#include <sys/mutex.h> 47#include <sys/proc.h> 48#include <sys/smp.h> 49#include <sys/sx.h> 50#include <sys/syslog.h> 51#include <sys/systm.h> 52#include <machine/clock.h> 53#include <machine/intr_machdep.h> 54#include <machine/smp.h> 55#ifdef DDB 56#include <ddb/ddb.h> 57#endif 58 59#ifndef DEV_ATPIC 60#include <machine/segments.h> 61#include <machine/frame.h> 62#include <dev/ic/i8259.h> 63#include <x86/isa/icu.h> 64#ifdef PC98 65#include <pc98/cbus/cbus.h> 66#else 67#include <x86/isa/isa.h> 68#endif 69#endif 70 71#define MAX_STRAY_LOG 5 72 73typedef void (*mask_fn)(void *); 74 75static int intrcnt_index; 76static struct intsrc *interrupt_sources[NUM_IO_INTS]; 77static struct sx intrsrc_lock; 78static struct mtx intrpic_lock; 79static struct mtx intrcnt_lock; 80static TAILQ_HEAD(pics_head, pic) pics; 81 82#ifdef SMP 83static int assign_cpu; 84#endif 85 86u_long intrcnt[INTRCNT_COUNT]; 87char intrnames[INTRCNT_COUNT * (MAXCOMLEN + 1)]; 88size_t sintrcnt = sizeof(intrcnt); 89size_t sintrnames = sizeof(intrnames); 90 91static int intr_assign_cpu(void *arg, u_char cpu); 92static void intr_disable_src(void *arg); 93static void intr_init(void *__dummy); 94static int intr_pic_registered(struct pic *pic); 95static void intrcnt_setname(const char *name, int index); 96static void intrcnt_updatename(struct intsrc *is); 97static void intrcnt_register(struct intsrc *is); 98 99static int 100intr_pic_registered(struct pic *pic) 101{ 102 struct pic *p; 103 104 TAILQ_FOREACH(p, &pics, pics) { 105 if (p == pic) 106 return (1); 107 } 108 return (0); 109} 110 111/* 112 * Register a new interrupt controller (PIC). This is to support suspend 113 * and resume where we suspend/resume controllers rather than individual 114 * sources. This also allows controllers with no active sources (such as 115 * 8259As in a system using the APICs) to participate in suspend and resume. 116 */ 117int 118intr_register_pic(struct pic *pic) 119{ 120 int error; 121 122 mtx_lock(&intrpic_lock); 123 if (intr_pic_registered(pic)) 124 error = EBUSY; 125 else { 126 TAILQ_INSERT_TAIL(&pics, pic, pics); 127 error = 0; 128 } 129 mtx_unlock(&intrpic_lock); 130 return (error); 131} 132 133/* 134 * Register a new interrupt source with the global interrupt system. 135 * The global interrupts need to be disabled when this function is 136 * called. 137 */ 138int 139intr_register_source(struct intsrc *isrc) 140{ 141 int error, vector; 142 143 KASSERT(intr_pic_registered(isrc->is_pic), ("unregistered PIC")); 144 vector = isrc->is_pic->pic_vector(isrc); 145 if (interrupt_sources[vector] != NULL) 146 return (EEXIST); 147 error = intr_event_create(&isrc->is_event, isrc, 0, vector, 148 intr_disable_src, (mask_fn)isrc->is_pic->pic_enable_source, 149 (mask_fn)isrc->is_pic->pic_eoi_source, intr_assign_cpu, "irq%d:", 150 vector); 151 if (error) 152 return (error); 153 sx_xlock(&intrsrc_lock); 154 if (interrupt_sources[vector] != NULL) { 155 sx_xunlock(&intrsrc_lock); 156 intr_event_destroy(isrc->is_event); 157 return (EEXIST); 158 } 159 intrcnt_register(isrc); 160 interrupt_sources[vector] = isrc; 161 isrc->is_handlers = 0; 162 sx_xunlock(&intrsrc_lock); 163 return (0); 164} 165 166struct intsrc * 167intr_lookup_source(int vector) 168{ 169 170 return (interrupt_sources[vector]); 171} 172 173int 174intr_add_handler(const char *name, int vector, driver_filter_t filter, 175 driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep) 176{ 177 struct intsrc *isrc; 178 int error; 179 180 isrc = intr_lookup_source(vector); 181 if (isrc == NULL) 182 return (EINVAL); 183 error = intr_event_add_handler(isrc->is_event, name, filter, handler, 184 arg, intr_priority(flags), flags, cookiep); 185 if (error == 0) { 186 sx_xlock(&intrsrc_lock); 187 intrcnt_updatename(isrc); 188 isrc->is_handlers++; 189 if (isrc->is_handlers == 1) { 190 isrc->is_pic->pic_enable_intr(isrc); 191 isrc->is_pic->pic_enable_source(isrc); 192 } 193 sx_xunlock(&intrsrc_lock); 194 } 195 return (error); 196} 197 198int 199intr_remove_handler(void *cookie) 200{ 201 struct intsrc *isrc; 202 int error; 203 204 isrc = intr_handler_source(cookie); 205 error = intr_event_remove_handler(cookie); 206 if (error == 0) { 207 sx_xlock(&intrsrc_lock); 208 isrc->is_handlers--; 209 if (isrc->is_handlers == 0) { 210 isrc->is_pic->pic_disable_source(isrc, PIC_NO_EOI); 211 isrc->is_pic->pic_disable_intr(isrc); 212 } 213 intrcnt_updatename(isrc); 214 sx_xunlock(&intrsrc_lock); 215 } 216 return (error); 217} 218 219int 220intr_config_intr(int vector, enum intr_trigger trig, enum intr_polarity pol) 221{ 222 struct intsrc *isrc; 223 224 isrc = intr_lookup_source(vector); 225 if (isrc == NULL) 226 return (EINVAL); 227 return (isrc->is_pic->pic_config_intr(isrc, trig, pol)); 228} 229 230static void 231intr_disable_src(void *arg) 232{ 233 struct intsrc *isrc; 234 235 isrc = arg; 236 isrc->is_pic->pic_disable_source(isrc, PIC_EOI); 237} 238 239void 240intr_execute_handlers(struct intsrc *isrc, struct trapframe *frame) 241{ 242 struct intr_event *ie; 243 int vector; 244 245 /* 246 * We count software interrupts when we process them. The 247 * code here follows previous practice, but there's an 248 * argument for counting hardware interrupts when they're 249 * processed too. 250 */ 251 (*isrc->is_count)++; 252 PCPU_INC(cnt.v_intr); 253 254 ie = isrc->is_event; 255 256 /* 257 * XXX: We assume that IRQ 0 is only used for the ISA timer 258 * device (clk). 259 */ 260 vector = isrc->is_pic->pic_vector(isrc); 261 if (vector == 0) 262 clkintr_pending = 1; 263 264 /* 265 * For stray interrupts, mask and EOI the source, bump the 266 * stray count, and log the condition. 267 */ 268 if (intr_event_handle(ie, frame) != 0) { 269 isrc->is_pic->pic_disable_source(isrc, PIC_EOI); 270 (*isrc->is_straycount)++; 271 if (*isrc->is_straycount < MAX_STRAY_LOG) 272 log(LOG_ERR, "stray irq%d\n", vector); 273 else if (*isrc->is_straycount == MAX_STRAY_LOG) 274 log(LOG_CRIT, 275 "too many stray irq %d's: not logging anymore\n", 276 vector); 277 } 278} 279 280void 281intr_resume(bool suspend_cancelled) 282{ 283 struct pic *pic; 284 285#ifndef DEV_ATPIC 286 atpic_reset(); 287#endif 288 mtx_lock(&intrpic_lock); 289 TAILQ_FOREACH(pic, &pics, pics) { 290 if (pic->pic_resume != NULL) 291 pic->pic_resume(pic, suspend_cancelled); 292 } 293 mtx_unlock(&intrpic_lock); 294} 295 296void 297intr_suspend(void) 298{ 299 struct pic *pic; 300 301 mtx_lock(&intrpic_lock); 302 TAILQ_FOREACH_REVERSE(pic, &pics, pics_head, pics) { 303 if (pic->pic_suspend != NULL) 304 pic->pic_suspend(pic); 305 } 306 mtx_unlock(&intrpic_lock); 307} 308 309static int 310intr_assign_cpu(void *arg, u_char cpu) 311{ 312#ifdef SMP 313 struct intsrc *isrc; 314 int error; 315 316 /* 317 * Don't do anything during early boot. We will pick up the 318 * assignment once the APs are started. 319 */ 320 if (assign_cpu && cpu != NOCPU) { 321 isrc = arg; 322 sx_xlock(&intrsrc_lock); 323 error = isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[cpu]); 324 sx_xunlock(&intrsrc_lock); 325 } else 326 error = 0; 327 return (error); 328#else 329 return (EOPNOTSUPP); 330#endif 331} 332 333static void 334intrcnt_setname(const char *name, int index) 335{ 336 337 snprintf(intrnames + (MAXCOMLEN + 1) * index, MAXCOMLEN + 1, "%-*s", 338 MAXCOMLEN, name); 339} 340 341static void 342intrcnt_updatename(struct intsrc *is) 343{ 344 345 intrcnt_setname(is->is_event->ie_fullname, is->is_index); 346} 347 348static void 349intrcnt_register(struct intsrc *is) 350{ 351 char straystr[MAXCOMLEN + 1]; 352 353 KASSERT(is->is_event != NULL, ("%s: isrc with no event", __func__)); 354 mtx_lock_spin(&intrcnt_lock); 355 is->is_index = intrcnt_index; 356 intrcnt_index += 2; 357 snprintf(straystr, MAXCOMLEN + 1, "stray irq%d", 358 is->is_pic->pic_vector(is)); 359 intrcnt_updatename(is); 360 is->is_count = &intrcnt[is->is_index]; 361 intrcnt_setname(straystr, is->is_index + 1); 362 is->is_straycount = &intrcnt[is->is_index + 1]; 363 mtx_unlock_spin(&intrcnt_lock); 364} 365 366void 367intrcnt_add(const char *name, u_long **countp) 368{ 369 370 mtx_lock_spin(&intrcnt_lock); 371 *countp = &intrcnt[intrcnt_index]; 372 intrcnt_setname(name, intrcnt_index); 373 intrcnt_index++; 374 mtx_unlock_spin(&intrcnt_lock); 375} 376 377static void 378intr_init(void *dummy __unused) 379{ 380 381 intrcnt_setname("???", 0); 382 intrcnt_index = 1; 383 TAILQ_INIT(&pics); 384 mtx_init(&intrpic_lock, "intrpic", NULL, MTX_DEF); 385 sx_init(&intrsrc_lock, "intrsrc"); 386 mtx_init(&intrcnt_lock, "intrcnt", NULL, MTX_SPIN); 387} 388SYSINIT(intr_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_init, NULL); 389 390#ifndef DEV_ATPIC 391/* Initialize the two 8259A's to a known-good shutdown state. */ 392void 393atpic_reset(void) 394{ 395 396 outb(IO_ICU1, ICW1_RESET | ICW1_IC4); 397 outb(IO_ICU1 + ICU_IMR_OFFSET, IDT_IO_INTS); 398 outb(IO_ICU1 + ICU_IMR_OFFSET, IRQ_MASK(ICU_SLAVEID)); 399 outb(IO_ICU1 + ICU_IMR_OFFSET, MASTER_MODE); 400 outb(IO_ICU1 + ICU_IMR_OFFSET, 0xff); 401 outb(IO_ICU1, OCW3_SEL | OCW3_RR); 402 403 outb(IO_ICU2, ICW1_RESET | ICW1_IC4); 404 outb(IO_ICU2 + ICU_IMR_OFFSET, IDT_IO_INTS + 8); 405 outb(IO_ICU2 + ICU_IMR_OFFSET, ICU_SLAVEID); 406 outb(IO_ICU2 + ICU_IMR_OFFSET, SLAVE_MODE); 407 outb(IO_ICU2 + ICU_IMR_OFFSET, 0xff); 408 outb(IO_ICU2, OCW3_SEL | OCW3_RR); 409} 410#endif 411 412/* Add a description to an active interrupt handler. */ 413int 414intr_describe(u_int vector, void *ih, const char *descr) 415{ 416 struct intsrc *isrc; 417 int error; 418 419 isrc = intr_lookup_source(vector); 420 if (isrc == NULL) 421 return (EINVAL); 422 error = intr_event_describe_handler(isrc->is_event, ih, descr); 423 if (error) 424 return (error); 425 intrcnt_updatename(isrc); 426 return (0); 427} 428 429#ifdef DDB 430/* 431 * Dump data about interrupt handlers 432 */ 433DB_SHOW_COMMAND(irqs, db_show_irqs) 434{ 435 struct intsrc **isrc; 436 int i, verbose; 437 438 if (strcmp(modif, "v") == 0) 439 verbose = 1; 440 else 441 verbose = 0; 442 isrc = interrupt_sources; 443 for (i = 0; i < NUM_IO_INTS && !db_pager_quit; i++, isrc++) 444 if (*isrc != NULL) 445 db_dump_intr_event((*isrc)->is_event, verbose); 446} 447#endif 448 449#ifdef SMP 450/* 451 * Support for balancing interrupt sources across CPUs. For now we just 452 * allocate CPUs round-robin. 453 */ 454 455static cpuset_t intr_cpus = CPUSET_T_INITIALIZER(0x1); 456static int current_cpu; 457 458/* 459 * Return the CPU that the next interrupt source should use. For now 460 * this just returns the next local APIC according to round-robin. 461 */ 462u_int 463intr_next_cpu(void) 464{ 465 u_int apic_id; 466 467 /* Leave all interrupts on the BSP during boot. */ 468 if (!assign_cpu) 469 return (PCPU_GET(apic_id)); 470 471 mtx_lock_spin(&icu_lock); 472 apic_id = cpu_apic_ids[current_cpu]; 473 do { 474 current_cpu++; 475 if (current_cpu > mp_maxid) 476 current_cpu = 0; 477 } while (!CPU_ISSET(current_cpu, &intr_cpus)); 478 mtx_unlock_spin(&icu_lock); 479 return (apic_id); 480} 481 482/* Attempt to bind the specified IRQ to the specified CPU. */ 483int 484intr_bind(u_int vector, u_char cpu) 485{ 486 struct intsrc *isrc; 487 488 isrc = intr_lookup_source(vector); 489 if (isrc == NULL) 490 return (EINVAL); 491 return (intr_event_bind(isrc->is_event, cpu)); 492} 493 494/* 495 * Add a CPU to our mask of valid CPUs that can be destinations of 496 * interrupts. 497 */ 498void 499intr_add_cpu(u_int cpu) 500{ 501 502 if (cpu >= MAXCPU) 503 panic("%s: Invalid CPU ID", __func__); 504 if (bootverbose) 505 printf("INTR: Adding local APIC %d as a target\n", 506 cpu_apic_ids[cpu]); 507 508 CPU_SET(cpu, &intr_cpus); 509} 510 511/* 512 * Distribute all the interrupt sources among the available CPUs once the 513 * AP's have been launched. 514 */ 515static void 516intr_shuffle_irqs(void *arg __unused) 517{ 518 struct intsrc *isrc; 519 int i; 520 521#ifdef XEN 522 /* 523 * Doesn't work yet 524 */ 525 return; 526#endif 527 528 /* Don't bother on UP. */ 529 if (mp_ncpus == 1) 530 return; 531 532 /* Round-robin assign a CPU to each enabled source. */ 533 sx_xlock(&intrsrc_lock); 534 assign_cpu = 1; 535 for (i = 0; i < NUM_IO_INTS; i++) { 536 isrc = interrupt_sources[i]; 537 if (isrc != NULL && isrc->is_handlers > 0) { 538 /* 539 * If this event is already bound to a CPU, 540 * then assign the source to that CPU instead 541 * of picking one via round-robin. Note that 542 * this is careful to only advance the 543 * round-robin if the CPU assignment succeeds. 544 */ 545 if (isrc->is_event->ie_cpu != NOCPU) 546 (void)isrc->is_pic->pic_assign_cpu(isrc, 547 cpu_apic_ids[isrc->is_event->ie_cpu]); 548 else if (isrc->is_pic->pic_assign_cpu(isrc, 549 cpu_apic_ids[current_cpu]) == 0) 550 (void)intr_next_cpu(); 551 552 } 553 } 554 sx_xunlock(&intrsrc_lock); 555} 556SYSINIT(intr_shuffle_irqs, SI_SUB_SMP, SI_ORDER_SECOND, intr_shuffle_irqs, 557 NULL); 558#else 559/* 560 * Always route interrupts to the current processor in the UP case. 561 */ 562u_int 563intr_next_cpu(void) 564{ 565 566 return (PCPU_GET(apic_id)); 567} 568#endif 569