i915_irq.c revision 282199
1/* i915_irq.c -- IRQ support for the I915 -*- linux-c -*- 2 */ 3/*- 4 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas. 5 * All Rights Reserved. 6 * 7 * Permission is hereby granted, free of charge, to any person obtaining a 8 * copy of this software and associated documentation files (the 9 * "Software"), to deal in the Software without restriction, including 10 * without limitation the rights to use, copy, modify, merge, publish, 11 * distribute, sub license, and/or sell copies of the Software, and to 12 * permit persons to whom the Software is furnished to do so, subject to 13 * the following conditions: 14 * 15 * The above copyright notice and this permission notice (including the 16 * next paragraph) shall be included in all copies or substantial portions 17 * of the Software. 18 * 19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 20 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. 22 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR 23 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 24 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 25 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 26 * 27 */ 28 29#include <sys/cdefs.h> 30__FBSDID("$FreeBSD: stable/10/sys/dev/drm2/i915/i915_irq.c 282199 2015-04-28 19:35:05Z dumbbell $"); 31 32#include <dev/drm2/drmP.h> 33#include <dev/drm2/drm.h> 34#include <dev/drm2/i915/i915_drm.h> 35#include <dev/drm2/i915/i915_drv.h> 36#include <dev/drm2/i915/intel_drv.h> 37#include <sys/sched.h> 38#include <sys/sf_buf.h> 39#include <sys/sleepqueue.h> 40 41static void i915_capture_error_state(struct drm_device *dev); 42static u32 ring_last_seqno(struct intel_ring_buffer *ring); 43 44/* For display hotplug interrupt */ 45static void 46ironlake_enable_display_irq(drm_i915_private_t *dev_priv, u32 mask) 47{ 48 if ((dev_priv->irq_mask & mask) != 0) { 49 dev_priv->irq_mask &= ~mask; 50 I915_WRITE(DEIMR, dev_priv->irq_mask); 51 POSTING_READ(DEIMR); 52 } 53} 54 55static inline void 56ironlake_disable_display_irq(drm_i915_private_t *dev_priv, u32 mask) 57{ 58 if ((dev_priv->irq_mask & mask) != mask) { 59 dev_priv->irq_mask |= mask; 60 I915_WRITE(DEIMR, dev_priv->irq_mask); 61 POSTING_READ(DEIMR); 62 } 63} 64 65void 66i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask) 67{ 68 if ((dev_priv->pipestat[pipe] & mask) != mask) { 69 u32 reg = PIPESTAT(pipe); 70 71 dev_priv->pipestat[pipe] |= mask; 72 /* Enable the interrupt, clear any pending status */ 73 I915_WRITE(reg, dev_priv->pipestat[pipe] | (mask >> 16)); 74 POSTING_READ(reg); 75 } 76} 77 78void 79i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask) 80{ 81 if ((dev_priv->pipestat[pipe] & mask) != 0) { 82 u32 reg = PIPESTAT(pipe); 83 84 dev_priv->pipestat[pipe] &= ~mask; 85 I915_WRITE(reg, dev_priv->pipestat[pipe]); 86 POSTING_READ(reg); 87 } 88} 89 90/** 91 * intel_enable_asle - enable ASLE interrupt for OpRegion 92 */ 93void intel_enable_asle(struct drm_device *dev) 94{ 95 drm_i915_private_t *dev_priv = dev->dev_private; 96 97 /* FIXME: opregion/asle for VLV */ 98 if (IS_VALLEYVIEW(dev)) 99 return; 100 101 mtx_lock(&dev_priv->irq_lock); 102 103 if (HAS_PCH_SPLIT(dev)) 104 ironlake_enable_display_irq(dev_priv, DE_GSE); 105 else { 106 i915_enable_pipestat(dev_priv, 1, 107 PIPE_LEGACY_BLC_EVENT_ENABLE); 108 if (INTEL_INFO(dev)->gen >= 4) 109 i915_enable_pipestat(dev_priv, 0, 110 PIPE_LEGACY_BLC_EVENT_ENABLE); 111 } 112 113 mtx_unlock(&dev_priv->irq_lock); 114} 115 116/** 117 * i915_pipe_enabled - check if a pipe is enabled 118 * @dev: DRM device 119 * @pipe: pipe to check 120 * 121 * Reading certain registers when the pipe is disabled can hang the chip. 122 * Use this routine to make sure the PLL is running and the pipe is active 123 * before reading such registers if unsure. 124 */ 125static int 126i915_pipe_enabled(struct drm_device *dev, int pipe) 127{ 128 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 129 return I915_READ(PIPECONF(pipe)) & PIPECONF_ENABLE; 130} 131 132/* Called from drm generic code, passed a 'crtc', which 133 * we use as a pipe index 134 */ 135static u32 136i915_get_vblank_counter(struct drm_device *dev, int pipe) 137{ 138 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 139 unsigned long high_frame; 140 unsigned long low_frame; 141 u32 high1, high2, low; 142 143 if (!i915_pipe_enabled(dev, pipe)) { 144 DRM_DEBUG("trying to get vblank count for disabled " 145 "pipe %c\n", pipe_name(pipe)); 146 return 0; 147 } 148 149 high_frame = PIPEFRAME(pipe); 150 low_frame = PIPEFRAMEPIXEL(pipe); 151 152 /* 153 * High & low register fields aren't synchronized, so make sure 154 * we get a low value that's stable across two reads of the high 155 * register. 156 */ 157 do { 158 high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK; 159 low = I915_READ(low_frame) & PIPE_FRAME_LOW_MASK; 160 high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK; 161 } while (high1 != high2); 162 163 high1 >>= PIPE_FRAME_HIGH_SHIFT; 164 low >>= PIPE_FRAME_LOW_SHIFT; 165 return (high1 << 8) | low; 166} 167 168static u32 169gm45_get_vblank_counter(struct drm_device *dev, int pipe) 170{ 171 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 172 int reg = PIPE_FRMCOUNT_GM45(pipe); 173 174 if (!i915_pipe_enabled(dev, pipe)) { 175 DRM_DEBUG("i915: trying to get vblank count for disabled " 176 "pipe %c\n", pipe_name(pipe)); 177 return 0; 178 } 179 180 return I915_READ(reg); 181} 182 183static int 184i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe, 185 int *vpos, int *hpos) 186{ 187 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 188 u32 vbl = 0, position = 0; 189 int vbl_start, vbl_end, htotal, vtotal; 190 bool in_vbl = true; 191 int ret = 0; 192 193 if (!i915_pipe_enabled(dev, pipe)) { 194 DRM_DEBUG("i915: trying to get scanoutpos for disabled " 195 "pipe %c\n", pipe_name(pipe)); 196 return 0; 197 } 198 199 /* Get vtotal. */ 200 vtotal = 1 + ((I915_READ(VTOTAL(pipe)) >> 16) & 0x1fff); 201 202 if (INTEL_INFO(dev)->gen >= 4) { 203 /* No obvious pixelcount register. Only query vertical 204 * scanout position from Display scan line register. 205 */ 206 position = I915_READ(PIPEDSL(pipe)); 207 208 /* Decode into vertical scanout position. Don't have 209 * horizontal scanout position. 210 */ 211 *vpos = position & 0x1fff; 212 *hpos = 0; 213 } else { 214 /* Have access to pixelcount since start of frame. 215 * We can split this into vertical and horizontal 216 * scanout position. 217 */ 218 position = (I915_READ(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT; 219 220 htotal = 1 + ((I915_READ(HTOTAL(pipe)) >> 16) & 0x1fff); 221 *vpos = position / htotal; 222 *hpos = position - (*vpos * htotal); 223 } 224 225 /* Query vblank area. */ 226 vbl = I915_READ(VBLANK(pipe)); 227 228 /* Test position against vblank region. */ 229 vbl_start = vbl & 0x1fff; 230 vbl_end = (vbl >> 16) & 0x1fff; 231 232 if ((*vpos < vbl_start) || (*vpos > vbl_end)) 233 in_vbl = false; 234 235 /* Inside "upper part" of vblank area? Apply corrective offset: */ 236 if (in_vbl && (*vpos >= vbl_start)) 237 *vpos = *vpos - vtotal; 238 239 /* Readouts valid? */ 240 if (vbl > 0) 241 ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE; 242 243 /* In vblank? */ 244 if (in_vbl) 245 ret |= DRM_SCANOUTPOS_INVBL; 246 247 return ret; 248} 249 250static int 251i915_get_vblank_timestamp(struct drm_device *dev, int pipe, int *max_error, 252 struct timeval *vblank_time, unsigned flags) 253{ 254 struct drm_i915_private *dev_priv = dev->dev_private; 255 struct drm_crtc *crtc; 256 257 if (pipe < 0 || pipe >= dev_priv->num_pipe) { 258 DRM_ERROR("Invalid crtc %d\n", pipe); 259 return -EINVAL; 260 } 261 262 /* Get drm_crtc to timestamp: */ 263 crtc = intel_get_crtc_for_pipe(dev, pipe); 264 if (crtc == NULL) { 265 DRM_ERROR("Invalid crtc %d\n", pipe); 266 return -EINVAL; 267 } 268 269 if (!crtc->enabled) { 270#if 0 271 DRM_DEBUG("crtc %d is disabled\n", pipe); 272#endif 273 return -EBUSY; 274 } 275 276 /* Helper routine in DRM core does all the work: */ 277 return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error, 278 vblank_time, flags, 279 crtc); 280} 281 282/* 283 * Handle hotplug events outside the interrupt handler proper. 284 */ 285static void 286i915_hotplug_work_func(void *context, int pending) 287{ 288 drm_i915_private_t *dev_priv = context; 289 struct drm_device *dev = dev_priv->dev; 290 struct drm_mode_config *mode_config; 291 struct intel_encoder *encoder; 292 293 DRM_DEBUG("running encoder hotplug functions\n"); 294 dev_priv = context; 295 dev = dev_priv->dev; 296 297 mode_config = &dev->mode_config; 298 299 sx_xlock(&mode_config->mutex); 300 DRM_DEBUG_KMS("running encoder hotplug functions\n"); 301 302 list_for_each_entry(encoder, &mode_config->encoder_list, base.head) 303 if (encoder->hot_plug) 304 encoder->hot_plug(encoder); 305 306 sx_xunlock(&mode_config->mutex); 307 308 /* Just fire off a uevent and let userspace tell us what to do */ 309#if 0 310 drm_helper_hpd_irq_event(dev); 311#endif 312} 313 314static void i915_handle_rps_change(struct drm_device *dev) 315{ 316 drm_i915_private_t *dev_priv = dev->dev_private; 317 u32 busy_up, busy_down, max_avg, min_avg; 318 u8 new_delay = dev_priv->cur_delay; 319 320 I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG); 321 busy_up = I915_READ(RCPREVBSYTUPAVG); 322 busy_down = I915_READ(RCPREVBSYTDNAVG); 323 max_avg = I915_READ(RCBMAXAVG); 324 min_avg = I915_READ(RCBMINAVG); 325 326 /* Handle RCS change request from hw */ 327 if (busy_up > max_avg) { 328 if (dev_priv->cur_delay != dev_priv->max_delay) 329 new_delay = dev_priv->cur_delay - 1; 330 if (new_delay < dev_priv->max_delay) 331 new_delay = dev_priv->max_delay; 332 } else if (busy_down < min_avg) { 333 if (dev_priv->cur_delay != dev_priv->min_delay) 334 new_delay = dev_priv->cur_delay + 1; 335 if (new_delay > dev_priv->min_delay) 336 new_delay = dev_priv->min_delay; 337 } 338 339 if (ironlake_set_drps(dev, new_delay)) 340 dev_priv->cur_delay = new_delay; 341 342 return; 343} 344 345static void notify_ring(struct drm_device *dev, 346 struct intel_ring_buffer *ring) 347{ 348 struct drm_i915_private *dev_priv = dev->dev_private; 349 350 if (ring->obj == NULL) 351 return; 352 353 CTR2(KTR_DRM, "request_complete %s %d", ring->name, 354 ring->get_seqno(ring)); 355 356 mtx_lock(&dev_priv->irq_lock); 357 wakeup(ring); 358 mtx_unlock(&dev_priv->irq_lock); 359 360 if (i915_enable_hangcheck) { 361 dev_priv->hangcheck_count = 0; 362 callout_schedule(&dev_priv->hangcheck_timer, 363 DRM_I915_HANGCHECK_PERIOD); 364 } 365} 366 367static void 368gen6_pm_rps_work_func(void *arg, int pending) 369{ 370 struct drm_device *dev; 371 drm_i915_private_t *dev_priv; 372 u8 new_delay; 373 u32 pm_iir, pm_imr; 374 375 dev_priv = (drm_i915_private_t *)arg; 376 dev = dev_priv->dev; 377 new_delay = dev_priv->cur_delay; 378 379 mtx_lock(&dev_priv->rps_lock); 380 pm_iir = dev_priv->pm_iir; 381 dev_priv->pm_iir = 0; 382 pm_imr = I915_READ(GEN6_PMIMR); 383 I915_WRITE(GEN6_PMIMR, 0); 384 mtx_unlock(&dev_priv->rps_lock); 385 386 if (!pm_iir) 387 return; 388 389 DRM_LOCK(dev); 390 if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) { 391 if (dev_priv->cur_delay != dev_priv->max_delay) 392 new_delay = dev_priv->cur_delay + 1; 393 if (new_delay > dev_priv->max_delay) 394 new_delay = dev_priv->max_delay; 395 } else if (pm_iir & (GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT)) { 396 gen6_gt_force_wake_get(dev_priv); 397 if (dev_priv->cur_delay != dev_priv->min_delay) 398 new_delay = dev_priv->cur_delay - 1; 399 if (new_delay < dev_priv->min_delay) { 400 new_delay = dev_priv->min_delay; 401 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, 402 I915_READ(GEN6_RP_INTERRUPT_LIMITS) | 403 ((new_delay << 16) & 0x3f0000)); 404 } else { 405 /* Make sure we continue to get down interrupts 406 * until we hit the minimum frequency */ 407 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, 408 I915_READ(GEN6_RP_INTERRUPT_LIMITS) & ~0x3f0000); 409 } 410 gen6_gt_force_wake_put(dev_priv); 411 } 412 413 gen6_set_rps(dev, new_delay); 414 dev_priv->cur_delay = new_delay; 415 416 /* 417 * rps_lock not held here because clearing is non-destructive. There is 418 * an *extremely* unlikely race with gen6_rps_enable() that is prevented 419 * by holding struct_mutex for the duration of the write. 420 */ 421 DRM_UNLOCK(dev); 422} 423 424static void snb_gt_irq_handler(struct drm_device *dev, 425 struct drm_i915_private *dev_priv, 426 u32 gt_iir) 427{ 428 429 if (gt_iir & (GEN6_RENDER_USER_INTERRUPT | 430 GEN6_RENDER_PIPE_CONTROL_NOTIFY_INTERRUPT)) 431 notify_ring(dev, &dev_priv->rings[RCS]); 432 if (gt_iir & GEN6_BSD_USER_INTERRUPT) 433 notify_ring(dev, &dev_priv->rings[VCS]); 434 if (gt_iir & GEN6_BLITTER_USER_INTERRUPT) 435 notify_ring(dev, &dev_priv->rings[BCS]); 436 437 if (gt_iir & (GT_GEN6_BLT_CS_ERROR_INTERRUPT | 438 GT_GEN6_BSD_CS_ERROR_INTERRUPT | 439 GT_RENDER_CS_ERROR_INTERRUPT)) { 440 DRM_ERROR("GT error interrupt 0x%08x\n", gt_iir); 441 i915_handle_error(dev, false); 442 } 443} 444 445static void gen6_queue_rps_work(struct drm_i915_private *dev_priv, 446 u32 pm_iir) 447{ 448 449 /* 450 * IIR bits should never already be set because IMR should 451 * prevent an interrupt from being shown in IIR. The warning 452 * displays a case where we've unsafely cleared 453 * dev_priv->pm_iir. Although missing an interrupt of the same 454 * type is not a problem, it displays a problem in the logic. 455 * 456 * The mask bit in IMR is cleared by rps_work. 457 */ 458 459 mtx_lock(&dev_priv->rps_lock); 460 if (dev_priv->pm_iir & pm_iir) 461 printf("Missed a PM interrupt\n"); 462 dev_priv->pm_iir |= pm_iir; 463 I915_WRITE(GEN6_PMIMR, dev_priv->pm_iir); 464 POSTING_READ(GEN6_PMIMR); 465 mtx_unlock(&dev_priv->rps_lock); 466 467 taskqueue_enqueue(dev_priv->tq, &dev_priv->rps_task); 468} 469 470static void valleyview_irq_handler(void *arg) 471{ 472 struct drm_device *dev = (struct drm_device *) arg; 473 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 474 u32 iir, gt_iir, pm_iir; 475 int pipe; 476 u32 pipe_stats[I915_MAX_PIPES]; 477 u32 vblank_status; 478 int vblank = 0; 479 bool blc_event; 480 481 atomic_inc(&dev_priv->irq_received); 482 483 vblank_status = PIPE_START_VBLANK_INTERRUPT_STATUS | 484 PIPE_VBLANK_INTERRUPT_STATUS; 485 486 while (true) { 487 iir = I915_READ(VLV_IIR); 488 gt_iir = I915_READ(GTIIR); 489 pm_iir = I915_READ(GEN6_PMIIR); 490 491 if (gt_iir == 0 && pm_iir == 0 && iir == 0) 492 goto out; 493 494 snb_gt_irq_handler(dev, dev_priv, gt_iir); 495 496 mtx_lock(&dev_priv->irq_lock); 497 for_each_pipe(pipe) { 498 int reg = PIPESTAT(pipe); 499 pipe_stats[pipe] = I915_READ(reg); 500 501 /* 502 * Clear the PIPE*STAT regs before the IIR 503 */ 504 if (pipe_stats[pipe] & 0x8000ffff) { 505 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS) 506 DRM_DEBUG_DRIVER("pipe %c underrun\n", 507 pipe_name(pipe)); 508 I915_WRITE(reg, pipe_stats[pipe]); 509 } 510 } 511 mtx_unlock(&dev_priv->irq_lock); 512 513 /* Consume port. Then clear IIR or we'll miss events */ 514 if (iir & I915_DISPLAY_PORT_INTERRUPT) { 515 u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT); 516 517 DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n", 518 hotplug_status); 519 if (hotplug_status & dev_priv->hotplug_supported_mask) 520 taskqueue_enqueue(dev_priv->tq, 521 &dev_priv->hotplug_task); 522 523 I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status); 524 I915_READ(PORT_HOTPLUG_STAT); 525 } 526 527 528 if (iir & I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT) { 529 drm_handle_vblank(dev, 0); 530 vblank++; 531 intel_finish_page_flip(dev, 0); 532 } 533 534 if (iir & I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT) { 535 drm_handle_vblank(dev, 1); 536 vblank++; 537 intel_finish_page_flip(dev, 0); 538 } 539 540 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS) 541 blc_event = true; 542 543 if (pm_iir & GEN6_PM_DEFERRED_EVENTS) 544 gen6_queue_rps_work(dev_priv, pm_iir); 545 546 I915_WRITE(GTIIR, gt_iir); 547 I915_WRITE(GEN6_PMIIR, pm_iir); 548 I915_WRITE(VLV_IIR, iir); 549 } 550 551out:; 552} 553 554static void pch_irq_handler(struct drm_device *dev, u32 pch_iir) 555{ 556 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 557 int pipe; 558 559 if (pch_iir & SDE_AUDIO_POWER_MASK) 560 DRM_DEBUG("i915: PCH audio power change on port %d\n", 561 (pch_iir & SDE_AUDIO_POWER_MASK) >> 562 SDE_AUDIO_POWER_SHIFT); 563 564 if (pch_iir & SDE_GMBUS) 565 DRM_DEBUG("i915: PCH GMBUS interrupt\n"); 566 567 if (pch_iir & SDE_AUDIO_HDCP_MASK) 568 DRM_DEBUG("i915: PCH HDCP audio interrupt\n"); 569 570 if (pch_iir & SDE_AUDIO_TRANS_MASK) 571 DRM_DEBUG("i915: PCH transcoder audio interrupt\n"); 572 573 if (pch_iir & SDE_POISON) 574 DRM_ERROR("i915: PCH poison interrupt\n"); 575 576 if (pch_iir & SDE_FDI_MASK) 577 for_each_pipe(pipe) 578 DRM_DEBUG(" pipe %c FDI IIR: 0x%08x\n", 579 pipe_name(pipe), 580 I915_READ(FDI_RX_IIR(pipe))); 581 582 if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE)) 583 DRM_DEBUG("i915: PCH transcoder CRC done interrupt\n"); 584 585 if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR)) 586 DRM_DEBUG("i915: PCH transcoder CRC error interrupt\n"); 587 588 if (pch_iir & SDE_TRANSB_FIFO_UNDER) 589 DRM_DEBUG("i915: PCH transcoder B underrun interrupt\n"); 590 if (pch_iir & SDE_TRANSA_FIFO_UNDER) 591 DRM_DEBUG("PCH transcoder A underrun interrupt\n"); 592} 593 594static void 595ivybridge_irq_handler(void *arg) 596{ 597 struct drm_device *dev = (struct drm_device *) arg; 598 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 599 u32 de_iir, gt_iir, de_ier, pm_iir; 600 int i; 601 602 atomic_inc(&dev_priv->irq_received); 603 604 /* disable master interrupt before clearing iir */ 605 de_ier = I915_READ(DEIER); 606 I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL); 607 POSTING_READ(DEIER); 608 609 gt_iir = I915_READ(GTIIR); 610 if (gt_iir) { 611 snb_gt_irq_handler(dev, dev_priv, gt_iir); 612 I915_WRITE(GTIIR, gt_iir); 613 } 614 615 de_iir = I915_READ(DEIIR); 616 if (de_iir) { 617 if (de_iir & DE_GSE_IVB) 618 intel_opregion_gse_intr(dev); 619 620 for (i = 0; i < 3; i++) { 621 if (de_iir & (DE_PLANEA_FLIP_DONE_IVB << (5 * i))) { 622 intel_prepare_page_flip(dev, i); 623 intel_finish_page_flip_plane(dev, i); 624 } 625 if (de_iir & (DE_PIPEA_VBLANK_IVB << (5 * i))) 626 drm_handle_vblank(dev, i); 627 } 628 629 /* check event from PCH */ 630 if (de_iir & DE_PCH_EVENT_IVB) { 631 u32 pch_iir = I915_READ(SDEIIR); 632 633 if (pch_iir & SDE_HOTPLUG_MASK_CPT) 634 taskqueue_enqueue(dev_priv->tq, 635 &dev_priv->hotplug_task); 636 pch_irq_handler(dev, pch_iir); 637 638 /* clear PCH hotplug event before clear CPU irq */ 639 I915_WRITE(SDEIIR, pch_iir); 640 } 641 642 I915_WRITE(DEIIR, de_iir); 643 } 644 645 pm_iir = I915_READ(GEN6_PMIIR); 646 if (pm_iir) { 647 if (pm_iir & GEN6_PM_DEFERRED_EVENTS) 648 gen6_queue_rps_work(dev_priv, pm_iir); 649 I915_WRITE(GEN6_PMIIR, pm_iir); 650 } 651 652 I915_WRITE(DEIER, de_ier); 653 POSTING_READ(DEIER); 654 655 CTR3(KTR_DRM, "ivybridge_irq de %x gt %x pm %x", de_iir, 656 gt_iir, pm_iir); 657} 658 659static void ilk_gt_irq_handler(struct drm_device *dev, 660 struct drm_i915_private *dev_priv, 661 u32 gt_iir) 662{ 663 if (gt_iir & (GT_USER_INTERRUPT | GT_PIPE_NOTIFY)) 664 notify_ring(dev, &dev_priv->rings[RCS]); 665 if (gt_iir & GT_BSD_USER_INTERRUPT) 666 notify_ring(dev, &dev_priv->rings[VCS]); 667} 668 669static void 670ironlake_irq_handler(void *arg) 671{ 672 struct drm_device *dev = arg; 673 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 674 u32 de_iir, gt_iir, de_ier, pch_iir, pm_iir; 675 u32 hotplug_mask; 676 677 atomic_inc(&dev_priv->irq_received); 678 679 /* disable master interrupt before clearing iir */ 680 de_ier = I915_READ(DEIER); 681 I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL); 682 POSTING_READ(DEIER); 683 684 de_iir = I915_READ(DEIIR); 685 gt_iir = I915_READ(GTIIR); 686 pch_iir = I915_READ(SDEIIR); 687 pm_iir = I915_READ(GEN6_PMIIR); 688 689 CTR4(KTR_DRM, "ironlake_irq de %x gt %x pch %x pm %x", de_iir, 690 gt_iir, pch_iir, pm_iir); 691 692 if (de_iir == 0 && gt_iir == 0 && pch_iir == 0 && 693 (!IS_GEN6(dev) || pm_iir == 0)) 694 goto done; 695 696 if (HAS_PCH_CPT(dev)) 697 hotplug_mask = SDE_HOTPLUG_MASK_CPT; 698 else 699 hotplug_mask = SDE_HOTPLUG_MASK; 700 701 if (IS_GEN5(dev)) 702 ilk_gt_irq_handler(dev, dev_priv, gt_iir); 703 else 704 snb_gt_irq_handler(dev, dev_priv, gt_iir); 705 706 if (de_iir & DE_GSE) { 707 intel_opregion_gse_intr(dev); 708 } 709 710 if (de_iir & DE_PLANEA_FLIP_DONE) { 711 intel_prepare_page_flip(dev, 0); 712 intel_finish_page_flip_plane(dev, 0); 713 } 714 715 if (de_iir & DE_PLANEB_FLIP_DONE) { 716 intel_prepare_page_flip(dev, 1); 717 intel_finish_page_flip_plane(dev, 1); 718 } 719 720 if (de_iir & DE_PIPEA_VBLANK) 721 drm_handle_vblank(dev, 0); 722 723 if (de_iir & DE_PIPEB_VBLANK) 724 drm_handle_vblank(dev, 1); 725 726 /* check event from PCH */ 727 if (de_iir & DE_PCH_EVENT) { 728 if (pch_iir & hotplug_mask) 729 taskqueue_enqueue(dev_priv->tq, 730 &dev_priv->hotplug_task); 731 pch_irq_handler(dev, pch_iir); 732 } 733 734 if (de_iir & DE_PCU_EVENT) { 735 I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS)); 736 i915_handle_rps_change(dev); 737 } 738 739 if (IS_GEN6(dev) && pm_iir & GEN6_PM_DEFERRED_EVENTS) 740 gen6_queue_rps_work(dev_priv, pm_iir); 741 742 /* should clear PCH hotplug event before clear CPU irq */ 743 I915_WRITE(SDEIIR, pch_iir); 744 I915_WRITE(GTIIR, gt_iir); 745 I915_WRITE(DEIIR, de_iir); 746 I915_WRITE(GEN6_PMIIR, pm_iir); 747 748done: 749 I915_WRITE(DEIER, de_ier); 750 POSTING_READ(DEIER); 751} 752 753/** 754 * i915_error_work_func - do process context error handling work 755 * @work: work struct 756 * 757 * Fire an error uevent so userspace can see that a hang or error 758 * was detected. 759 */ 760static void 761i915_error_work_func(void *context, int pending) 762{ 763 drm_i915_private_t *dev_priv = context; 764 struct drm_device *dev = dev_priv->dev; 765 766 /* kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, error_event); */ 767 768 if (atomic_load_acq_int(&dev_priv->mm.wedged)) { 769 DRM_DEBUG("i915: resetting chip\n"); 770 /* kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, reset_event); */ 771 if (!i915_reset(dev)) { 772 atomic_store_rel_int(&dev_priv->mm.wedged, 0); 773 /* kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, reset_done_event); */ 774 } 775 mtx_lock(&dev_priv->error_completion_lock); 776 dev_priv->error_completion++; 777 wakeup(&dev_priv->error_completion); 778 mtx_unlock(&dev_priv->error_completion_lock); 779 } 780} 781 782#define pr_err(...) printf(__VA_ARGS__) 783 784static void i915_report_and_clear_eir(struct drm_device *dev) 785{ 786 struct drm_i915_private *dev_priv = dev->dev_private; 787 u32 eir = I915_READ(EIR); 788 int pipe; 789 790 if (!eir) 791 return; 792 793 printf("i915: render error detected, EIR: 0x%08x\n", eir); 794 795 if (IS_G4X(dev)) { 796 if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) { 797 u32 ipeir = I915_READ(IPEIR_I965); 798 799 pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR_I965)); 800 pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR_I965)); 801 pr_err(" INSTDONE: 0x%08x\n", 802 I915_READ(INSTDONE_I965)); 803 pr_err(" INSTPS: 0x%08x\n", I915_READ(INSTPS)); 804 pr_err(" INSTDONE1: 0x%08x\n", I915_READ(INSTDONE1)); 805 pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD_I965)); 806 I915_WRITE(IPEIR_I965, ipeir); 807 POSTING_READ(IPEIR_I965); 808 } 809 if (eir & GM45_ERROR_PAGE_TABLE) { 810 u32 pgtbl_err = I915_READ(PGTBL_ER); 811 pr_err("page table error\n"); 812 pr_err(" PGTBL_ER: 0x%08x\n", pgtbl_err); 813 I915_WRITE(PGTBL_ER, pgtbl_err); 814 POSTING_READ(PGTBL_ER); 815 } 816 } 817 818 if (!IS_GEN2(dev)) { 819 if (eir & I915_ERROR_PAGE_TABLE) { 820 u32 pgtbl_err = I915_READ(PGTBL_ER); 821 pr_err("page table error\n"); 822 pr_err(" PGTBL_ER: 0x%08x\n", pgtbl_err); 823 I915_WRITE(PGTBL_ER, pgtbl_err); 824 POSTING_READ(PGTBL_ER); 825 } 826 } 827 828 if (eir & I915_ERROR_MEMORY_REFRESH) { 829 pr_err("memory refresh error:\n"); 830 for_each_pipe(pipe) 831 pr_err("pipe %c stat: 0x%08x\n", 832 pipe_name(pipe), I915_READ(PIPESTAT(pipe))); 833 /* pipestat has already been acked */ 834 } 835 if (eir & I915_ERROR_INSTRUCTION) { 836 pr_err("instruction error\n"); 837 pr_err(" INSTPM: 0x%08x\n", I915_READ(INSTPM)); 838 if (INTEL_INFO(dev)->gen < 4) { 839 u32 ipeir = I915_READ(IPEIR); 840 841 pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR)); 842 pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR)); 843 pr_err(" INSTDONE: 0x%08x\n", I915_READ(INSTDONE)); 844 pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD)); 845 I915_WRITE(IPEIR, ipeir); 846 POSTING_READ(IPEIR); 847 } else { 848 u32 ipeir = I915_READ(IPEIR_I965); 849 850 pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR_I965)); 851 pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR_I965)); 852 pr_err(" INSTDONE: 0x%08x\n", 853 I915_READ(INSTDONE_I965)); 854 pr_err(" INSTPS: 0x%08x\n", I915_READ(INSTPS)); 855 pr_err(" INSTDONE1: 0x%08x\n", I915_READ(INSTDONE1)); 856 pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD_I965)); 857 I915_WRITE(IPEIR_I965, ipeir); 858 POSTING_READ(IPEIR_I965); 859 } 860 } 861 862 I915_WRITE(EIR, eir); 863 POSTING_READ(EIR); 864 eir = I915_READ(EIR); 865 if (eir) { 866 /* 867 * some errors might have become stuck, 868 * mask them. 869 */ 870 DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir); 871 I915_WRITE(EMR, I915_READ(EMR) | eir); 872 I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT); 873 } 874} 875 876/** 877 * i915_handle_error - handle an error interrupt 878 * @dev: drm device 879 * 880 * Do some basic checking of regsiter state at error interrupt time and 881 * dump it to the syslog. Also call i915_capture_error_state() to make 882 * sure we get a record and make it available in debugfs. Fire a uevent 883 * so userspace knows something bad happened (should trigger collection 884 * of a ring dump etc.). 885 */ 886void i915_handle_error(struct drm_device *dev, bool wedged) 887{ 888 struct drm_i915_private *dev_priv = dev->dev_private; 889 struct intel_ring_buffer *ring; 890 int i; 891 892 i915_capture_error_state(dev); 893 i915_report_and_clear_eir(dev); 894 895 if (wedged) { 896 mtx_lock(&dev_priv->error_completion_lock); 897 dev_priv->error_completion = 0; 898 dev_priv->mm.wedged = 1; 899 /* unlock acts as rel barrier for store to wedged */ 900 mtx_unlock(&dev_priv->error_completion_lock); 901 902 /* 903 * Wakeup waiting processes so they don't hang 904 */ 905 for_each_ring(ring, dev_priv, i) { 906 mtx_lock(&dev_priv->irq_lock); 907 wakeup(ring); 908 mtx_unlock(&dev_priv->irq_lock); 909 } 910 } 911 912 taskqueue_enqueue(dev_priv->tq, &dev_priv->error_task); 913} 914 915static void i915_pageflip_stall_check(struct drm_device *dev, int pipe) 916{ 917 drm_i915_private_t *dev_priv = dev->dev_private; 918 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; 919 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 920 struct drm_i915_gem_object *obj; 921 struct intel_unpin_work *work; 922 bool stall_detected; 923 924 /* Ignore early vblank irqs */ 925 if (intel_crtc == NULL) 926 return; 927 928 mtx_lock(&dev->event_lock); 929 work = intel_crtc->unpin_work; 930 931 if (work == NULL || work->pending || !work->enable_stall_check) { 932 /* Either the pending flip IRQ arrived, or we're too early. Don't check */ 933 mtx_unlock(&dev->event_lock); 934 return; 935 } 936 937 /* Potential stall - if we see that the flip has happened, assume a missed interrupt */ 938 obj = work->pending_flip_obj; 939 if (INTEL_INFO(dev)->gen >= 4) { 940 int dspsurf = DSPSURF(intel_crtc->plane); 941 stall_detected = I915_HI_DISPBASE(I915_READ(dspsurf)) == 942 obj->gtt_offset; 943 } else { 944 int dspaddr = DSPADDR(intel_crtc->plane); 945 stall_detected = I915_READ(dspaddr) == (obj->gtt_offset + 946 crtc->y * crtc->fb->pitches[0] + 947 crtc->x * crtc->fb->bits_per_pixel/8); 948 } 949 950 mtx_unlock(&dev->event_lock); 951 952 if (stall_detected) { 953 DRM_DEBUG("Pageflip stall detected\n"); 954 intel_prepare_page_flip(dev, intel_crtc->plane); 955 } 956} 957 958/* Called from drm generic code, passed 'crtc' which 959 * we use as a pipe index 960 */ 961static int 962i915_enable_vblank(struct drm_device *dev, int pipe) 963{ 964 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 965 966 if (!i915_pipe_enabled(dev, pipe)) 967 return -EINVAL; 968 969 mtx_lock(&dev_priv->irq_lock); 970 if (INTEL_INFO(dev)->gen >= 4) 971 i915_enable_pipestat(dev_priv, pipe, 972 PIPE_START_VBLANK_INTERRUPT_ENABLE); 973 else 974 i915_enable_pipestat(dev_priv, pipe, 975 PIPE_VBLANK_INTERRUPT_ENABLE); 976 977 /* maintain vblank delivery even in deep C-states */ 978 if (dev_priv->info->gen == 3) 979 I915_WRITE(INSTPM, _MASKED_BIT_DISABLE(INSTPM_AGPBUSY_DIS)); 980 mtx_unlock(&dev_priv->irq_lock); 981 CTR1(KTR_DRM, "i915_enable_vblank %d", pipe); 982 983 return 0; 984} 985 986static int 987ironlake_enable_vblank(struct drm_device *dev, int pipe) 988{ 989 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 990 991 if (!i915_pipe_enabled(dev, pipe)) 992 return -EINVAL; 993 994 mtx_lock(&dev_priv->irq_lock); 995 ironlake_enable_display_irq(dev_priv, (pipe == 0) ? 996 DE_PIPEA_VBLANK : DE_PIPEB_VBLANK); 997 mtx_unlock(&dev_priv->irq_lock); 998 CTR1(KTR_DRM, "ironlake_enable_vblank %d", pipe); 999 1000 return 0; 1001} 1002 1003static int 1004ivybridge_enable_vblank(struct drm_device *dev, int pipe) 1005{ 1006 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 1007 1008 if (!i915_pipe_enabled(dev, pipe)) 1009 return -EINVAL; 1010 1011 mtx_lock(&dev_priv->irq_lock); 1012 ironlake_enable_display_irq(dev_priv, 1013 DE_PIPEA_VBLANK_IVB << (5 * pipe)); 1014 mtx_unlock(&dev_priv->irq_lock); 1015 CTR1(KTR_DRM, "ivybridge_enable_vblank %d", pipe); 1016 1017 return 0; 1018} 1019 1020static int valleyview_enable_vblank(struct drm_device *dev, int pipe) 1021{ 1022 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 1023 u32 dpfl, imr; 1024 1025 if (!i915_pipe_enabled(dev, pipe)) 1026 return -EINVAL; 1027 1028 mtx_lock(&dev_priv->irq_lock); 1029 dpfl = I915_READ(VLV_DPFLIPSTAT); 1030 imr = I915_READ(VLV_IMR); 1031 if (pipe == 0) { 1032 dpfl |= PIPEA_VBLANK_INT_EN; 1033 imr &= ~I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT; 1034 } else { 1035 dpfl |= PIPEA_VBLANK_INT_EN; 1036 imr &= ~I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT; 1037 } 1038 I915_WRITE(VLV_DPFLIPSTAT, dpfl); 1039 I915_WRITE(VLV_IMR, imr); 1040 mtx_unlock(&dev_priv->irq_lock); 1041 1042 return 0; 1043} 1044 1045/* Called from drm generic code, passed 'crtc' which 1046 * we use as a pipe index 1047 */ 1048static void 1049i915_disable_vblank(struct drm_device *dev, int pipe) 1050{ 1051 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 1052 1053 mtx_lock(&dev_priv->irq_lock); 1054 if (dev_priv->info->gen == 3) 1055 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_DIS)); 1056 1057 i915_disable_pipestat(dev_priv, pipe, 1058 PIPE_VBLANK_INTERRUPT_ENABLE | 1059 PIPE_START_VBLANK_INTERRUPT_ENABLE); 1060 mtx_unlock(&dev_priv->irq_lock); 1061 CTR1(KTR_DRM, "i915_disable_vblank %d", pipe); 1062} 1063 1064static void 1065ironlake_disable_vblank(struct drm_device *dev, int pipe) 1066{ 1067 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 1068 1069 mtx_lock(&dev_priv->irq_lock); 1070 ironlake_disable_display_irq(dev_priv, (pipe == 0) ? 1071 DE_PIPEA_VBLANK : DE_PIPEB_VBLANK); 1072 mtx_unlock(&dev_priv->irq_lock); 1073 CTR1(KTR_DRM, "ironlake_disable_vblank %d", pipe); 1074} 1075 1076static void 1077ivybridge_disable_vblank(struct drm_device *dev, int pipe) 1078{ 1079 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 1080 1081 mtx_lock(&dev_priv->irq_lock); 1082 ironlake_disable_display_irq(dev_priv, 1083 DE_PIPEA_VBLANK_IVB << (pipe * 5)); 1084 mtx_unlock(&dev_priv->irq_lock); 1085 CTR1(KTR_DRM, "ivybridge_disable_vblank %d", pipe); 1086} 1087 1088static void valleyview_disable_vblank(struct drm_device *dev, int pipe) 1089{ 1090 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 1091 u32 dpfl, imr; 1092 1093 mtx_lock(&dev_priv->irq_lock); 1094 dpfl = I915_READ(VLV_DPFLIPSTAT); 1095 imr = I915_READ(VLV_IMR); 1096 if (pipe == 0) { 1097 dpfl &= ~PIPEA_VBLANK_INT_EN; 1098 imr |= I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT; 1099 } else { 1100 dpfl &= ~PIPEB_VBLANK_INT_EN; 1101 imr |= I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT; 1102 } 1103 I915_WRITE(VLV_IMR, imr); 1104 I915_WRITE(VLV_DPFLIPSTAT, dpfl); 1105 mtx_unlock(&dev_priv->irq_lock); 1106} 1107 1108static u32 1109ring_last_seqno(struct intel_ring_buffer *ring) 1110{ 1111 1112 if (list_empty(&ring->request_list)) 1113 return (0); 1114 else 1115 return (list_entry(ring->request_list.prev, 1116 struct drm_i915_gem_request, list)->seqno); 1117} 1118 1119static bool i915_hangcheck_ring_idle(struct intel_ring_buffer *ring, bool *err) 1120{ 1121 if (list_empty(&ring->request_list) || 1122 i915_seqno_passed(ring->get_seqno(ring), ring_last_seqno(ring))) { 1123 /* Issue a wake-up to catch stuck h/w. */ 1124 sleepq_lock(ring); 1125 if (sleepq_sleepcnt(ring, 0) != 0) { 1126 sleepq_release(ring); 1127 DRM_ERROR("Hangcheck timer elapsed... %s idle\n", 1128 ring->name); 1129 wakeup(ring); 1130 *err = true; 1131 } else 1132 sleepq_release(ring); 1133 return true; 1134 } 1135 return false; 1136} 1137 1138static bool kick_ring(struct intel_ring_buffer *ring) 1139{ 1140 struct drm_device *dev = ring->dev; 1141 struct drm_i915_private *dev_priv = dev->dev_private; 1142 u32 tmp = I915_READ_CTL(ring); 1143 if (tmp & RING_WAIT) { 1144 DRM_ERROR("Kicking stuck wait on %s\n", 1145 ring->name); 1146 I915_WRITE_CTL(ring, tmp); 1147 return true; 1148 } 1149 return false; 1150} 1151 1152static bool i915_hangcheck_hung(struct drm_device *dev) 1153{ 1154 drm_i915_private_t *dev_priv = dev->dev_private; 1155 1156 if (dev_priv->hangcheck_count++ > 1) { 1157 bool hung = true; 1158 1159 DRM_ERROR("Hangcheck timer elapsed... GPU hung\n"); 1160 i915_handle_error(dev, true); 1161 1162 if (!IS_GEN2(dev)) { 1163 struct intel_ring_buffer *ring; 1164 int i; 1165 1166 /* Is the chip hanging on a WAIT_FOR_EVENT? 1167 * If so we can simply poke the RB_WAIT bit 1168 * and break the hang. This should work on 1169 * all but the second generation chipsets. 1170 */ 1171 for_each_ring(ring, dev_priv, i) 1172 hung &= !kick_ring(ring); 1173 } 1174 1175 return hung; 1176 } 1177 1178 return false; 1179} 1180 1181/** 1182 * This is called when the chip hasn't reported back with completed 1183 * batchbuffers in a long time. The first time this is called we simply record 1184 * ACTHD. If ACTHD hasn't changed by the time the hangcheck timer elapses 1185 * again, we assume the chip is wedged and try to fix it. 1186 */ 1187void 1188i915_hangcheck_elapsed(void *context) 1189{ 1190 struct drm_device *dev = (struct drm_device *)context; 1191 drm_i915_private_t *dev_priv = dev->dev_private; 1192 uint32_t acthd[I915_NUM_RINGS], instdone, instdone1; 1193 struct intel_ring_buffer *ring; 1194 bool err = false, idle; 1195 int i; 1196 1197 if (!i915_enable_hangcheck) 1198 return; 1199 1200 memset(acthd, 0, sizeof(acthd)); 1201 idle = true; 1202 for_each_ring(ring, dev_priv, i) { 1203 idle &= i915_hangcheck_ring_idle(ring, &err); 1204 acthd[i] = intel_ring_get_active_head(ring); 1205 } 1206 1207 /* If all work is done then ACTHD clearly hasn't advanced. */ 1208 if (idle) { 1209 if (err) { 1210 if (i915_hangcheck_hung(dev)) 1211 return; 1212 1213 goto repeat; 1214 } 1215 1216 dev_priv->hangcheck_count = 0; 1217 return; 1218 } 1219 1220 if (INTEL_INFO(dev)->gen < 4) { 1221 instdone = I915_READ(INSTDONE); 1222 instdone1 = 0; 1223 } else { 1224 instdone = I915_READ(INSTDONE_I965); 1225 instdone1 = I915_READ(INSTDONE1); 1226 } 1227 if (memcmp(dev_priv->last_acthd, acthd, sizeof(acthd)) == 0 && 1228 dev_priv->last_instdone == instdone && 1229 dev_priv->last_instdone1 == instdone1) { 1230 if (i915_hangcheck_hung(dev)) 1231 return; 1232 } else { 1233 dev_priv->hangcheck_count = 0; 1234 1235 memcpy(dev_priv->last_acthd, acthd, sizeof(acthd)); 1236 dev_priv->last_instdone = instdone; 1237 dev_priv->last_instdone1 = instdone1; 1238 } 1239 1240repeat: 1241 /* Reset timer case chip hangs without another request being added */ 1242 callout_schedule(&dev_priv->hangcheck_timer, DRM_I915_HANGCHECK_PERIOD); 1243} 1244 1245/* drm_dma.h hooks 1246*/ 1247static void 1248ironlake_irq_preinstall(struct drm_device *dev) 1249{ 1250 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 1251 1252 atomic_set(&dev_priv->irq_received, 0); 1253 1254 I915_WRITE(HWSTAM, 0xeffe); 1255 1256 /* XXX hotplug from PCH */ 1257 1258 I915_WRITE(DEIMR, 0xffffffff); 1259 I915_WRITE(DEIER, 0x0); 1260 POSTING_READ(DEIER); 1261 1262 /* and GT */ 1263 I915_WRITE(GTIMR, 0xffffffff); 1264 I915_WRITE(GTIER, 0x0); 1265 POSTING_READ(GTIER); 1266 1267 /* south display irq */ 1268 I915_WRITE(SDEIMR, 0xffffffff); 1269 I915_WRITE(SDEIER, 0x0); 1270 POSTING_READ(SDEIER); 1271} 1272 1273static void valleyview_irq_preinstall(struct drm_device *dev) 1274{ 1275 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 1276 int pipe; 1277 1278 atomic_set(&dev_priv->irq_received, 0); 1279 1280 /* VLV magic */ 1281 I915_WRITE(VLV_IMR, 0); 1282 I915_WRITE(RING_IMR(RENDER_RING_BASE), 0); 1283 I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0); 1284 I915_WRITE(RING_IMR(BLT_RING_BASE), 0); 1285 1286 /* and GT */ 1287 I915_WRITE(GTIIR, I915_READ(GTIIR)); 1288 I915_WRITE(GTIIR, I915_READ(GTIIR)); 1289 I915_WRITE(GTIMR, 0xffffffff); 1290 I915_WRITE(GTIER, 0x0); 1291 POSTING_READ(GTIER); 1292 1293 I915_WRITE(DPINVGTT, 0xff); 1294 1295 I915_WRITE(PORT_HOTPLUG_EN, 0); 1296 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); 1297 for_each_pipe(pipe) 1298 I915_WRITE(PIPESTAT(pipe), 0xffff); 1299 I915_WRITE(VLV_IIR, 0xffffffff); 1300 I915_WRITE(VLV_IMR, 0xffffffff); 1301 I915_WRITE(VLV_IER, 0x0); 1302 POSTING_READ(VLV_IER); 1303} 1304 1305/* 1306 * Enable digital hotplug on the PCH, and configure the DP short pulse 1307 * duration to 2ms (which is the minimum in the Display Port spec) 1308 * 1309 * This register is the same on all known PCH chips. 1310 */ 1311 1312static void ironlake_enable_pch_hotplug(struct drm_device *dev) 1313{ 1314 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 1315 u32 hotplug; 1316 1317 hotplug = I915_READ(PCH_PORT_HOTPLUG); 1318 hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK); 1319 hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms; 1320 hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms; 1321 hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms; 1322 I915_WRITE(PCH_PORT_HOTPLUG, hotplug); 1323} 1324 1325static int ironlake_irq_postinstall(struct drm_device *dev) 1326{ 1327 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 1328 /* enable kind of interrupts always enabled */ 1329 u32 display_mask = DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT | 1330 DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE; 1331 u32 render_irqs; 1332 u32 hotplug_mask; 1333 1334 dev_priv->irq_mask = ~display_mask; 1335 1336 /* should always can generate irq */ 1337 I915_WRITE(DEIIR, I915_READ(DEIIR)); 1338 I915_WRITE(DEIMR, dev_priv->irq_mask); 1339 I915_WRITE(DEIER, display_mask | DE_PIPEA_VBLANK | DE_PIPEB_VBLANK); 1340 POSTING_READ(DEIER); 1341 1342 dev_priv->gt_irq_mask = ~0; 1343 1344 I915_WRITE(GTIIR, I915_READ(GTIIR)); 1345 I915_WRITE(GTIMR, dev_priv->gt_irq_mask); 1346 1347 if (IS_GEN6(dev)) 1348 render_irqs = 1349 GT_USER_INTERRUPT | 1350 GEN6_BSD_USER_INTERRUPT | 1351 GEN6_BLITTER_USER_INTERRUPT; 1352 else 1353 render_irqs = 1354 GT_USER_INTERRUPT | 1355 GT_PIPE_NOTIFY | 1356 GT_BSD_USER_INTERRUPT; 1357 I915_WRITE(GTIER, render_irqs); 1358 POSTING_READ(GTIER); 1359 1360 if (HAS_PCH_CPT(dev)) { 1361 hotplug_mask = (SDE_CRT_HOTPLUG_CPT | 1362 SDE_PORTB_HOTPLUG_CPT | 1363 SDE_PORTC_HOTPLUG_CPT | 1364 SDE_PORTD_HOTPLUG_CPT); 1365 } else { 1366 hotplug_mask = (SDE_CRT_HOTPLUG | 1367 SDE_PORTB_HOTPLUG | 1368 SDE_PORTC_HOTPLUG | 1369 SDE_PORTD_HOTPLUG | 1370 SDE_AUX_MASK); 1371 } 1372 1373 dev_priv->pch_irq_mask = ~hotplug_mask; 1374 1375 I915_WRITE(SDEIIR, I915_READ(SDEIIR)); 1376 I915_WRITE(SDEIMR, dev_priv->pch_irq_mask); 1377 I915_WRITE(SDEIER, hotplug_mask); 1378 POSTING_READ(SDEIER); 1379 1380 ironlake_enable_pch_hotplug(dev); 1381 1382 if (IS_IRONLAKE_M(dev)) { 1383 /* Clear & enable PCU event interrupts */ 1384 I915_WRITE(DEIIR, DE_PCU_EVENT); 1385 I915_WRITE(DEIER, I915_READ(DEIER) | DE_PCU_EVENT); 1386 ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT); 1387 } 1388 1389 return 0; 1390} 1391 1392static int 1393ivybridge_irq_postinstall(struct drm_device *dev) 1394{ 1395 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 1396 /* enable kind of interrupts always enabled */ 1397 u32 display_mask = 1398 DE_MASTER_IRQ_CONTROL | DE_GSE_IVB | DE_PCH_EVENT_IVB | 1399 DE_PLANEC_FLIP_DONE_IVB | 1400 DE_PLANEB_FLIP_DONE_IVB | 1401 DE_PLANEA_FLIP_DONE_IVB; 1402 u32 render_irqs; 1403 u32 hotplug_mask; 1404 1405 dev_priv->irq_mask = ~display_mask; 1406 1407 /* should always can generate irq */ 1408 I915_WRITE(DEIIR, I915_READ(DEIIR)); 1409 I915_WRITE(DEIMR, dev_priv->irq_mask); 1410 I915_WRITE(DEIER, 1411 display_mask | 1412 DE_PIPEC_VBLANK_IVB | 1413 DE_PIPEB_VBLANK_IVB | 1414 DE_PIPEA_VBLANK_IVB); 1415 POSTING_READ(DEIER); 1416 1417 dev_priv->gt_irq_mask = ~0; 1418 1419 I915_WRITE(GTIIR, I915_READ(GTIIR)); 1420 I915_WRITE(GTIMR, dev_priv->gt_irq_mask); 1421 1422 render_irqs = GT_USER_INTERRUPT | GEN6_BSD_USER_INTERRUPT | 1423 GEN6_BLITTER_USER_INTERRUPT; 1424 I915_WRITE(GTIER, render_irqs); 1425 POSTING_READ(GTIER); 1426 1427 hotplug_mask = (SDE_CRT_HOTPLUG_CPT | 1428 SDE_PORTB_HOTPLUG_CPT | 1429 SDE_PORTC_HOTPLUG_CPT | 1430 SDE_PORTD_HOTPLUG_CPT); 1431 dev_priv->pch_irq_mask = ~hotplug_mask; 1432 1433 I915_WRITE(SDEIIR, I915_READ(SDEIIR)); 1434 I915_WRITE(SDEIMR, dev_priv->pch_irq_mask); 1435 I915_WRITE(SDEIER, hotplug_mask); 1436 POSTING_READ(SDEIER); 1437 1438 ironlake_enable_pch_hotplug(dev); 1439 1440 return 0; 1441} 1442 1443static int valleyview_irq_postinstall(struct drm_device *dev) 1444{ 1445 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 1446 u32 render_irqs; 1447 u32 enable_mask; 1448 u32 hotplug_en = I915_READ(PORT_HOTPLUG_EN); 1449 u16 msid; 1450 1451 enable_mask = I915_DISPLAY_PORT_INTERRUPT; 1452 enable_mask |= I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT | 1453 I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT; 1454 1455 dev_priv->irq_mask = ~enable_mask; 1456 1457 dev_priv->pipestat[0] = 0; 1458 dev_priv->pipestat[1] = 0; 1459 1460 /* Hack for broken MSIs on VLV */ 1461 pci_write_config(dev->dev, 0x94, 0xfee00000, 4); 1462 msid = pci_read_config(dev->dev, 0x98, 2); 1463 msid &= 0xff; /* mask out delivery bits */ 1464 msid |= (1<<14); 1465 pci_write_config(dev->dev, 0x98, msid, 2); 1466 1467 I915_WRITE(VLV_IMR, dev_priv->irq_mask); 1468 I915_WRITE(VLV_IER, enable_mask); 1469 I915_WRITE(VLV_IIR, 0xffffffff); 1470 I915_WRITE(PIPESTAT(0), 0xffff); 1471 I915_WRITE(PIPESTAT(1), 0xffff); 1472 POSTING_READ(VLV_IER); 1473 1474 I915_WRITE(VLV_IIR, 0xffffffff); 1475 I915_WRITE(VLV_IIR, 0xffffffff); 1476 1477 render_irqs = GT_GEN6_BLT_FLUSHDW_NOTIFY_INTERRUPT | 1478 GT_GEN6_BLT_CS_ERROR_INTERRUPT | 1479 GT_GEN6_BLT_USER_INTERRUPT | 1480 GT_GEN6_BSD_USER_INTERRUPT | 1481 GT_GEN6_BSD_CS_ERROR_INTERRUPT | 1482 GT_GEN7_L3_PARITY_ERROR_INTERRUPT | 1483 GT_PIPE_NOTIFY | 1484 GT_RENDER_CS_ERROR_INTERRUPT | 1485 GT_SYNC_STATUS | 1486 GT_USER_INTERRUPT; 1487 1488 dev_priv->gt_irq_mask = ~render_irqs; 1489 1490 I915_WRITE(GTIIR, I915_READ(GTIIR)); 1491 I915_WRITE(GTIIR, I915_READ(GTIIR)); 1492 I915_WRITE(GTIMR, 0); 1493 I915_WRITE(GTIER, render_irqs); 1494 POSTING_READ(GTIER); 1495 1496 /* ack & enable invalid PTE error interrupts */ 1497#if 0 /* FIXME: add support to irq handler for checking these bits */ 1498 I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK); 1499 I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK); 1500#endif 1501 1502 I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE); 1503#if 0 /* FIXME: check register definitions; some have moved */ 1504 /* Note HDMI and DP share bits */ 1505 if (dev_priv->hotplug_supported_mask & HDMIB_HOTPLUG_INT_STATUS) 1506 hotplug_en |= HDMIB_HOTPLUG_INT_EN; 1507 if (dev_priv->hotplug_supported_mask & HDMIC_HOTPLUG_INT_STATUS) 1508 hotplug_en |= HDMIC_HOTPLUG_INT_EN; 1509 if (dev_priv->hotplug_supported_mask & HDMID_HOTPLUG_INT_STATUS) 1510 hotplug_en |= HDMID_HOTPLUG_INT_EN; 1511 if (dev_priv->hotplug_supported_mask & SDVOC_HOTPLUG_INT_STATUS) 1512 hotplug_en |= SDVOC_HOTPLUG_INT_EN; 1513 if (dev_priv->hotplug_supported_mask & SDVOB_HOTPLUG_INT_STATUS) 1514 hotplug_en |= SDVOB_HOTPLUG_INT_EN; 1515 if (dev_priv->hotplug_supported_mask & CRT_HOTPLUG_INT_STATUS) { 1516 hotplug_en |= CRT_HOTPLUG_INT_EN; 1517 hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50; 1518 } 1519#endif 1520 1521 I915_WRITE(PORT_HOTPLUG_EN, hotplug_en); 1522 1523 return 0; 1524} 1525 1526static void valleyview_irq_uninstall(struct drm_device *dev) 1527{ 1528 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 1529 int pipe; 1530 1531 if (!dev_priv) 1532 return; 1533 1534 for_each_pipe(pipe) 1535 I915_WRITE(PIPESTAT(pipe), 0xffff); 1536 1537 I915_WRITE(HWSTAM, 0xffffffff); 1538 I915_WRITE(PORT_HOTPLUG_EN, 0); 1539 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); 1540 for_each_pipe(pipe) 1541 I915_WRITE(PIPESTAT(pipe), 0xffff); 1542 I915_WRITE(VLV_IIR, 0xffffffff); 1543 I915_WRITE(VLV_IMR, 0xffffffff); 1544 I915_WRITE(VLV_IER, 0x0); 1545 POSTING_READ(VLV_IER); 1546} 1547 1548static void 1549ironlake_irq_uninstall(struct drm_device *dev) 1550{ 1551 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 1552 1553 if (dev_priv == NULL) 1554 return; 1555 1556 I915_WRITE(HWSTAM, 0xffffffff); 1557 1558 I915_WRITE(DEIMR, 0xffffffff); 1559 I915_WRITE(DEIER, 0x0); 1560 I915_WRITE(DEIIR, I915_READ(DEIIR)); 1561 1562 I915_WRITE(GTIMR, 0xffffffff); 1563 I915_WRITE(GTIER, 0x0); 1564 I915_WRITE(GTIIR, I915_READ(GTIIR)); 1565 1566 I915_WRITE(SDEIMR, 0xffffffff); 1567 I915_WRITE(SDEIER, 0x0); 1568 I915_WRITE(SDEIIR, I915_READ(SDEIIR)); 1569} 1570 1571static void i8xx_irq_preinstall(struct drm_device * dev) 1572{ 1573 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 1574 int pipe; 1575 1576 atomic_set(&dev_priv->irq_received, 0); 1577 1578 for_each_pipe(pipe) 1579 I915_WRITE(PIPESTAT(pipe), 0); 1580 I915_WRITE16(IMR, 0xffff); 1581 I915_WRITE16(IER, 0x0); 1582 POSTING_READ16(IER); 1583} 1584 1585static int i8xx_irq_postinstall(struct drm_device *dev) 1586{ 1587 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 1588 1589 dev_priv->pipestat[0] = 0; 1590 dev_priv->pipestat[1] = 0; 1591 1592 I915_WRITE16(EMR, 1593 ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH)); 1594 1595 /* Unmask the interrupts that we always want on. */ 1596 dev_priv->irq_mask = 1597 ~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | 1598 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | 1599 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT | 1600 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT | 1601 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT); 1602 I915_WRITE16(IMR, dev_priv->irq_mask); 1603 1604 I915_WRITE16(IER, 1605 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | 1606 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | 1607 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT | 1608 I915_USER_INTERRUPT); 1609 POSTING_READ16(IER); 1610 1611 return 0; 1612} 1613 1614static void i8xx_irq_handler(void *arg) 1615{ 1616 struct drm_device *dev = (struct drm_device *) arg; 1617 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 1618 u16 iir, new_iir; 1619 u32 pipe_stats[2]; 1620 int irq_received; 1621 int pipe; 1622 u16 flip_mask = 1623 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT | 1624 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT; 1625 1626 atomic_inc(&dev_priv->irq_received); 1627 1628 iir = I915_READ16(IIR); 1629 if (iir == 0) 1630 return; 1631 1632 while (iir & ~flip_mask) { 1633 /* Can't rely on pipestat interrupt bit in iir as it might 1634 * have been cleared after the pipestat interrupt was received. 1635 * It doesn't set the bit in iir again, but it still produces 1636 * interrupts (for non-MSI). 1637 */ 1638 mtx_lock(&dev_priv->irq_lock); 1639 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT) 1640 i915_handle_error(dev, false); 1641 1642 for_each_pipe(pipe) { 1643 int reg = PIPESTAT(pipe); 1644 pipe_stats[pipe] = I915_READ(reg); 1645 1646 /* 1647 * Clear the PIPE*STAT regs before the IIR 1648 */ 1649 if (pipe_stats[pipe] & 0x8000ffff) { 1650 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS) 1651 DRM_DEBUG_DRIVER("pipe %c underrun\n", 1652 pipe_name(pipe)); 1653 I915_WRITE(reg, pipe_stats[pipe]); 1654 irq_received = 1; 1655 } 1656 } 1657 mtx_unlock(&dev_priv->irq_lock); 1658 1659 I915_WRITE16(IIR, iir & ~flip_mask); 1660 new_iir = I915_READ16(IIR); /* Flush posted writes */ 1661 1662 i915_update_dri1_breadcrumb(dev); 1663 1664 if (iir & I915_USER_INTERRUPT) 1665 notify_ring(dev, &dev_priv->rings[RCS]); 1666 1667 if (pipe_stats[0] & PIPE_VBLANK_INTERRUPT_STATUS && 1668 drm_handle_vblank(dev, 0)) { 1669 if (iir & I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT) { 1670 intel_prepare_page_flip(dev, 0); 1671 intel_finish_page_flip(dev, 0); 1672 flip_mask &= ~I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT; 1673 } 1674 } 1675 1676 if (pipe_stats[1] & PIPE_VBLANK_INTERRUPT_STATUS && 1677 drm_handle_vblank(dev, 1)) { 1678 if (iir & I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT) { 1679 intel_prepare_page_flip(dev, 1); 1680 intel_finish_page_flip(dev, 1); 1681 flip_mask &= ~I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT; 1682 } 1683 } 1684 1685 iir = new_iir; 1686 } 1687} 1688 1689static void i8xx_irq_uninstall(struct drm_device * dev) 1690{ 1691 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 1692 int pipe; 1693 1694 for_each_pipe(pipe) { 1695 /* Clear enable bits; then clear status bits */ 1696 I915_WRITE(PIPESTAT(pipe), 0); 1697 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe))); 1698 } 1699 I915_WRITE16(IMR, 0xffff); 1700 I915_WRITE16(IER, 0x0); 1701 I915_WRITE16(IIR, I915_READ16(IIR)); 1702} 1703 1704static void i915_irq_preinstall(struct drm_device * dev) 1705{ 1706 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 1707 int pipe; 1708 1709 atomic_set(&dev_priv->irq_received, 0); 1710 1711 if (I915_HAS_HOTPLUG(dev)) { 1712 I915_WRITE(PORT_HOTPLUG_EN, 0); 1713 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); 1714 } 1715 1716 I915_WRITE16(HWSTAM, 0xeffe); 1717 for_each_pipe(pipe) 1718 I915_WRITE(PIPESTAT(pipe), 0); 1719 I915_WRITE(IMR, 0xffffffff); 1720 I915_WRITE(IER, 0x0); 1721 POSTING_READ(IER); 1722} 1723 1724static int i915_irq_postinstall(struct drm_device *dev) 1725{ 1726 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 1727 u32 enable_mask; 1728 1729 dev_priv->pipestat[0] = 0; 1730 dev_priv->pipestat[1] = 0; 1731 1732 I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH)); 1733 1734 /* Unmask the interrupts that we always want on. */ 1735 dev_priv->irq_mask = 1736 ~(I915_ASLE_INTERRUPT | 1737 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | 1738 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | 1739 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT | 1740 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT | 1741 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT); 1742 1743 enable_mask = 1744 I915_ASLE_INTERRUPT | 1745 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | 1746 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | 1747 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT | 1748 I915_USER_INTERRUPT; 1749 1750 if (I915_HAS_HOTPLUG(dev)) { 1751 /* Enable in IER... */ 1752 enable_mask |= I915_DISPLAY_PORT_INTERRUPT; 1753 /* and unmask in IMR */ 1754 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT; 1755 } 1756 1757 I915_WRITE(IMR, dev_priv->irq_mask); 1758 I915_WRITE(IER, enable_mask); 1759 POSTING_READ(IER); 1760 1761 if (I915_HAS_HOTPLUG(dev)) { 1762 u32 hotplug_en = I915_READ(PORT_HOTPLUG_EN); 1763 1764 if (dev_priv->hotplug_supported_mask & HDMIB_HOTPLUG_INT_STATUS) 1765 hotplug_en |= HDMIB_HOTPLUG_INT_EN; 1766 if (dev_priv->hotplug_supported_mask & HDMIC_HOTPLUG_INT_STATUS) 1767 hotplug_en |= HDMIC_HOTPLUG_INT_EN; 1768 if (dev_priv->hotplug_supported_mask & HDMID_HOTPLUG_INT_STATUS) 1769 hotplug_en |= HDMID_HOTPLUG_INT_EN; 1770 if (dev_priv->hotplug_supported_mask & SDVOC_HOTPLUG_INT_STATUS) 1771 hotplug_en |= SDVOC_HOTPLUG_INT_EN; 1772 if (dev_priv->hotplug_supported_mask & SDVOB_HOTPLUG_INT_STATUS) 1773 hotplug_en |= SDVOB_HOTPLUG_INT_EN; 1774 if (dev_priv->hotplug_supported_mask & CRT_HOTPLUG_INT_STATUS) { 1775 hotplug_en |= CRT_HOTPLUG_INT_EN; 1776 hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50; 1777 } 1778 1779 /* Ignore TV since it's buggy */ 1780 1781 I915_WRITE(PORT_HOTPLUG_EN, hotplug_en); 1782 } 1783 1784 intel_opregion_enable_asle(dev); 1785 1786 return 0; 1787} 1788 1789static void i915_irq_handler(void *arg) 1790{ 1791 struct drm_device *dev = (struct drm_device *) arg; 1792 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 1793 u32 iir, new_iir, pipe_stats[I915_MAX_PIPES]; 1794 u32 flip_mask = 1795 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT | 1796 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT; 1797 u32 flip[2] = { 1798 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT, 1799 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT 1800 }; 1801 int pipe; 1802 1803 atomic_inc(&dev_priv->irq_received); 1804 1805 iir = I915_READ(IIR); 1806 do { 1807 bool irq_received = (iir & ~flip_mask) != 0; 1808 bool blc_event = false; 1809 1810 /* Can't rely on pipestat interrupt bit in iir as it might 1811 * have been cleared after the pipestat interrupt was received. 1812 * It doesn't set the bit in iir again, but it still produces 1813 * interrupts (for non-MSI). 1814 */ 1815 mtx_lock(&dev_priv->irq_lock); 1816 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT) 1817 i915_handle_error(dev, false); 1818 1819 for_each_pipe(pipe) { 1820 int reg = PIPESTAT(pipe); 1821 pipe_stats[pipe] = I915_READ(reg); 1822 1823 /* Clear the PIPE*STAT regs before the IIR */ 1824 if (pipe_stats[pipe] & 0x8000ffff) { 1825 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS) 1826 DRM_DEBUG_DRIVER("pipe %c underrun\n", 1827 pipe_name(pipe)); 1828 I915_WRITE(reg, pipe_stats[pipe]); 1829 irq_received = true; 1830 } 1831 } 1832 mtx_unlock(&dev_priv->irq_lock); 1833 1834 if (!irq_received) 1835 break; 1836 1837 /* Consume port. Then clear IIR or we'll miss events */ 1838 if ((I915_HAS_HOTPLUG(dev)) && 1839 (iir & I915_DISPLAY_PORT_INTERRUPT)) { 1840 u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT); 1841 1842 DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n", 1843 hotplug_status); 1844 if (hotplug_status & dev_priv->hotplug_supported_mask) 1845 taskqueue_enqueue(dev_priv->tq, 1846 &dev_priv->hotplug_task); 1847 1848 I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status); 1849 POSTING_READ(PORT_HOTPLUG_STAT); 1850 } 1851 1852 I915_WRITE(IIR, iir & ~flip_mask); 1853 new_iir = I915_READ(IIR); /* Flush posted writes */ 1854 1855 if (iir & I915_USER_INTERRUPT) 1856 notify_ring(dev, &dev_priv->rings[RCS]); 1857 1858 for_each_pipe(pipe) { 1859 int plane = pipe; 1860 if (IS_MOBILE(dev)) 1861 plane = !plane; 1862 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS && 1863 drm_handle_vblank(dev, pipe)) { 1864 if (iir & flip[plane]) { 1865 intel_prepare_page_flip(dev, plane); 1866 intel_finish_page_flip(dev, pipe); 1867 flip_mask &= ~flip[plane]; 1868 } 1869 } 1870 1871 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS) 1872 blc_event = true; 1873 } 1874 1875 if (blc_event || (iir & I915_ASLE_INTERRUPT)) 1876 intel_opregion_asle_intr(dev); 1877 1878 1879 /* With MSI, interrupts are only generated when iir 1880 * transitions from zero to nonzero. If another bit got 1881 * set while we were handling the existing iir bits, then 1882 * we would never get another interrupt. 1883 * 1884 * This is fine on non-MSI as well, as if we hit this path 1885 * we avoid exiting the interrupt handler only to generate 1886 * another one. 1887 * 1888 * Note that for MSI this could cause a stray interrupt report 1889 * if an interrupt landed in the time between writing IIR and 1890 * the posting read. This should be rare enough to never 1891 * trigger the 99% of 100,000 interrupts test for disabling 1892 * stray interrupts. 1893 */ 1894 iir = new_iir; 1895 } while (iir & ~flip_mask); 1896 1897 i915_update_dri1_breadcrumb(dev); 1898} 1899 1900static void i915_irq_uninstall(struct drm_device * dev) 1901{ 1902 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 1903 int pipe; 1904 1905 if (!dev_priv) 1906 return; 1907 1908 if (I915_HAS_HOTPLUG(dev)) { 1909 I915_WRITE(PORT_HOTPLUG_EN, 0); 1910 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); 1911 } 1912 1913 I915_WRITE16(HWSTAM, 0xffff); 1914 for_each_pipe(pipe) { 1915 /* Clear enable bits; then clear status bits */ 1916 I915_WRITE(PIPESTAT(pipe), 0); 1917 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe))); 1918 } 1919 I915_WRITE(IMR, 0xffffffff); 1920 I915_WRITE(IER, 0x0); 1921 1922 I915_WRITE(IIR, I915_READ(IIR)); 1923} 1924 1925static void i965_irq_preinstall(struct drm_device * dev) 1926{ 1927 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 1928 int pipe; 1929 1930 atomic_set(&dev_priv->irq_received, 0); 1931 1932 if (I915_HAS_HOTPLUG(dev)) { 1933 I915_WRITE(PORT_HOTPLUG_EN, 0); 1934 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); 1935 } 1936 1937 I915_WRITE(HWSTAM, 0xeffe); 1938 for_each_pipe(pipe) 1939 I915_WRITE(PIPESTAT(pipe), 0); 1940 I915_WRITE(IMR, 0xffffffff); 1941 I915_WRITE(IER, 0x0); 1942 POSTING_READ(IER); 1943} 1944 1945static int i965_irq_postinstall(struct drm_device *dev) 1946{ 1947 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 1948 u32 enable_mask; 1949 u32 error_mask; 1950 1951 /* Unmask the interrupts that we always want on. */ 1952 dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT | 1953 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | 1954 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | 1955 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT | 1956 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT | 1957 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT); 1958 1959 enable_mask = ~dev_priv->irq_mask; 1960 enable_mask |= I915_USER_INTERRUPT; 1961 1962 if (IS_G4X(dev)) 1963 enable_mask |= I915_BSD_USER_INTERRUPT; 1964 1965 dev_priv->pipestat[0] = 0; 1966 dev_priv->pipestat[1] = 0; 1967 1968 if (I915_HAS_HOTPLUG(dev)) { 1969 /* Enable in IER... */ 1970 enable_mask |= I915_DISPLAY_PORT_INTERRUPT; 1971 /* and unmask in IMR */ 1972 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT; 1973 } 1974 1975 /* 1976 * Enable some error detection, note the instruction error mask 1977 * bit is reserved, so we leave it masked. 1978 */ 1979 if (IS_G4X(dev)) { 1980 error_mask = ~(GM45_ERROR_PAGE_TABLE | 1981 GM45_ERROR_MEM_PRIV | 1982 GM45_ERROR_CP_PRIV | 1983 I915_ERROR_MEMORY_REFRESH); 1984 } else { 1985 error_mask = ~(I915_ERROR_PAGE_TABLE | 1986 I915_ERROR_MEMORY_REFRESH); 1987 } 1988 I915_WRITE(EMR, error_mask); 1989 1990 I915_WRITE(IMR, dev_priv->irq_mask); 1991 I915_WRITE(IER, enable_mask); 1992 POSTING_READ(IER); 1993 1994 if (I915_HAS_HOTPLUG(dev)) { 1995 u32 hotplug_en = I915_READ(PORT_HOTPLUG_EN); 1996 1997 /* Note HDMI and DP share bits */ 1998 if (dev_priv->hotplug_supported_mask & HDMIB_HOTPLUG_INT_STATUS) 1999 hotplug_en |= HDMIB_HOTPLUG_INT_EN; 2000 if (dev_priv->hotplug_supported_mask & HDMIC_HOTPLUG_INT_STATUS) 2001 hotplug_en |= HDMIC_HOTPLUG_INT_EN; 2002 if (dev_priv->hotplug_supported_mask & HDMID_HOTPLUG_INT_STATUS) 2003 hotplug_en |= HDMID_HOTPLUG_INT_EN; 2004 if (dev_priv->hotplug_supported_mask & SDVOC_HOTPLUG_INT_STATUS) 2005 hotplug_en |= SDVOC_HOTPLUG_INT_EN; 2006 if (dev_priv->hotplug_supported_mask & SDVOB_HOTPLUG_INT_STATUS) 2007 hotplug_en |= SDVOB_HOTPLUG_INT_EN; 2008 if (dev_priv->hotplug_supported_mask & CRT_HOTPLUG_INT_STATUS) { 2009 hotplug_en |= CRT_HOTPLUG_INT_EN; 2010 2011 /* Programming the CRT detection parameters tends 2012 to generate a spurious hotplug event about three 2013 seconds later. So just do it once. 2014 */ 2015 if (IS_G4X(dev)) 2016 hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64; 2017 hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50; 2018 } 2019 2020 /* Ignore TV since it's buggy */ 2021 2022 I915_WRITE(PORT_HOTPLUG_EN, hotplug_en); 2023 } 2024 2025 intel_opregion_enable_asle(dev); 2026 2027 return 0; 2028} 2029 2030static void i965_irq_handler(void *arg) 2031{ 2032 struct drm_device *dev = (struct drm_device *) arg; 2033 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 2034 u32 iir, new_iir; 2035 u32 pipe_stats[I915_MAX_PIPES]; 2036 int irq_received; 2037 int pipe; 2038 2039 atomic_inc(&dev_priv->irq_received); 2040 2041 iir = I915_READ(IIR); 2042 2043 for (;;) { 2044 bool blc_event = false; 2045 2046 irq_received = iir != 0; 2047 2048 /* Can't rely on pipestat interrupt bit in iir as it might 2049 * have been cleared after the pipestat interrupt was received. 2050 * It doesn't set the bit in iir again, but it still produces 2051 * interrupts (for non-MSI). 2052 */ 2053 mtx_lock(&dev_priv->irq_lock); 2054 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT) 2055 i915_handle_error(dev, false); 2056 2057 for_each_pipe(pipe) { 2058 int reg = PIPESTAT(pipe); 2059 pipe_stats[pipe] = I915_READ(reg); 2060 2061 /* 2062 * Clear the PIPE*STAT regs before the IIR 2063 */ 2064 if (pipe_stats[pipe] & 0x8000ffff) { 2065 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS) 2066 DRM_DEBUG_DRIVER("pipe %c underrun\n", 2067 pipe_name(pipe)); 2068 I915_WRITE(reg, pipe_stats[pipe]); 2069 irq_received = 1; 2070 } 2071 } 2072 mtx_unlock(&dev_priv->irq_lock); 2073 2074 if (!irq_received) 2075 break; 2076 2077 /* Consume port. Then clear IIR or we'll miss events */ 2078 if ((I915_HAS_HOTPLUG(dev)) && 2079 (iir & I915_DISPLAY_PORT_INTERRUPT)) { 2080 u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT); 2081 2082 DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n", 2083 hotplug_status); 2084 if (hotplug_status & dev_priv->hotplug_supported_mask) 2085 taskqueue_enqueue(dev_priv->tq, 2086 &dev_priv->hotplug_task); 2087 2088 I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status); 2089 I915_READ(PORT_HOTPLUG_STAT); 2090 } 2091 2092 I915_WRITE(IIR, iir); 2093 new_iir = I915_READ(IIR); /* Flush posted writes */ 2094 2095 if (iir & I915_USER_INTERRUPT) 2096 notify_ring(dev, &dev_priv->rings[RCS]); 2097 if (iir & I915_BSD_USER_INTERRUPT) 2098 notify_ring(dev, &dev_priv->rings[VCS]); 2099 2100 if (iir & I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT) 2101 intel_prepare_page_flip(dev, 0); 2102 2103 if (iir & I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT) 2104 intel_prepare_page_flip(dev, 1); 2105 2106 for_each_pipe(pipe) { 2107 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS && 2108 drm_handle_vblank(dev, pipe)) { 2109 i915_pageflip_stall_check(dev, pipe); 2110 intel_finish_page_flip(dev, pipe); 2111 } 2112 2113 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS) 2114 blc_event = true; 2115 } 2116 2117 2118 if (blc_event || (iir & I915_ASLE_INTERRUPT)) 2119 intel_opregion_asle_intr(dev); 2120 2121 /* With MSI, interrupts are only generated when iir 2122 * transitions from zero to nonzero. If another bit got 2123 * set while we were handling the existing iir bits, then 2124 * we would never get another interrupt. 2125 * 2126 * This is fine on non-MSI as well, as if we hit this path 2127 * we avoid exiting the interrupt handler only to generate 2128 * another one. 2129 * 2130 * Note that for MSI this could cause a stray interrupt report 2131 * if an interrupt landed in the time between writing IIR and 2132 * the posting read. This should be rare enough to never 2133 * trigger the 99% of 100,000 interrupts test for disabling 2134 * stray interrupts. 2135 */ 2136 iir = new_iir; 2137 } 2138 2139 i915_update_dri1_breadcrumb(dev); 2140} 2141 2142static void i965_irq_uninstall(struct drm_device * dev) 2143{ 2144 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 2145 int pipe; 2146 2147 if (I915_HAS_HOTPLUG(dev)) { 2148 I915_WRITE(PORT_HOTPLUG_EN, 0); 2149 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); 2150 } 2151 2152 I915_WRITE(HWSTAM, 0xffffffff); 2153 for_each_pipe(pipe) 2154 I915_WRITE(PIPESTAT(pipe), 0); 2155 I915_WRITE(IMR, 0xffffffff); 2156 I915_WRITE(IER, 0x0); 2157 2158 for_each_pipe(pipe) 2159 I915_WRITE(PIPESTAT(pipe), 2160 I915_READ(PIPESTAT(pipe)) & 0x8000ffff); 2161 I915_WRITE(IIR, I915_READ(IIR)); 2162} 2163 2164void intel_irq_init(struct drm_device *dev) 2165{ 2166 struct drm_i915_private *dev_priv = dev->dev_private; 2167 2168 TASK_INIT(&dev_priv->hotplug_task, 0, i915_hotplug_work_func, 2169 dev->dev_private); 2170 TASK_INIT(&dev_priv->error_task, 0, i915_error_work_func, 2171 dev->dev_private); 2172 TASK_INIT(&dev_priv->rps_task, 0, gen6_pm_rps_work_func, 2173 dev->dev_private); 2174 2175 dev->driver->get_vblank_counter = i915_get_vblank_counter; 2176 dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */ 2177 if (IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) { 2178 dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */ 2179 dev->driver->get_vblank_counter = gm45_get_vblank_counter; 2180 } 2181 if (drm_core_check_feature(dev, DRIVER_MODESET)) 2182 dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp; 2183 else 2184 dev->driver->get_vblank_timestamp = NULL; 2185 dev->driver->get_scanout_position = i915_get_crtc_scanoutpos; 2186 2187 if (IS_VALLEYVIEW(dev)) { 2188 dev->driver->irq_handler = valleyview_irq_handler; 2189 dev->driver->irq_preinstall = valleyview_irq_preinstall; 2190 dev->driver->irq_postinstall = valleyview_irq_postinstall; 2191 dev->driver->irq_uninstall = valleyview_irq_uninstall; 2192 dev->driver->enable_vblank = valleyview_enable_vblank; 2193 dev->driver->disable_vblank = valleyview_disable_vblank; 2194 } else if (IS_IVYBRIDGE(dev)) { 2195 /* Share pre & uninstall handlers with ILK/SNB */ 2196 dev->driver->irq_handler = ivybridge_irq_handler; 2197 dev->driver->irq_preinstall = ironlake_irq_preinstall; 2198 dev->driver->irq_postinstall = ivybridge_irq_postinstall; 2199 dev->driver->irq_uninstall = ironlake_irq_uninstall; 2200 dev->driver->enable_vblank = ivybridge_enable_vblank; 2201 dev->driver->disable_vblank = ivybridge_disable_vblank; 2202 } else if (IS_HASWELL(dev)) { 2203 /* Share interrupts handling with IVB */ 2204 dev->driver->irq_handler = ivybridge_irq_handler; 2205 dev->driver->irq_preinstall = ironlake_irq_preinstall; 2206 dev->driver->irq_postinstall = ivybridge_irq_postinstall; 2207 dev->driver->irq_uninstall = ironlake_irq_uninstall; 2208 dev->driver->enable_vblank = ivybridge_enable_vblank; 2209 dev->driver->disable_vblank = ivybridge_disable_vblank; 2210 } else if (HAS_PCH_SPLIT(dev)) { 2211 dev->driver->irq_handler = ironlake_irq_handler; 2212 dev->driver->irq_preinstall = ironlake_irq_preinstall; 2213 dev->driver->irq_postinstall = ironlake_irq_postinstall; 2214 dev->driver->irq_uninstall = ironlake_irq_uninstall; 2215 dev->driver->enable_vblank = ironlake_enable_vblank; 2216 dev->driver->disable_vblank = ironlake_disable_vblank; 2217 } else { 2218 if (INTEL_INFO(dev)->gen == 2) { 2219 dev->driver->irq_preinstall = i8xx_irq_preinstall; 2220 dev->driver->irq_postinstall = i8xx_irq_postinstall; 2221 dev->driver->irq_handler = i8xx_irq_handler; 2222 dev->driver->irq_uninstall = i8xx_irq_uninstall; 2223 } else if (INTEL_INFO(dev)->gen == 3) { 2224 /* IIR "flip pending" means done if this bit is set */ 2225 I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE)); 2226 2227 dev->driver->irq_preinstall = i915_irq_preinstall; 2228 dev->driver->irq_postinstall = i915_irq_postinstall; 2229 dev->driver->irq_uninstall = i915_irq_uninstall; 2230 dev->driver->irq_handler = i915_irq_handler; 2231 } else { 2232 dev->driver->irq_preinstall = i965_irq_preinstall; 2233 dev->driver->irq_postinstall = i965_irq_postinstall; 2234 dev->driver->irq_uninstall = i965_irq_uninstall; 2235 dev->driver->irq_handler = i965_irq_handler; 2236 } 2237 dev->driver->enable_vblank = i915_enable_vblank; 2238 dev->driver->disable_vblank = i915_disable_vblank; 2239 } 2240} 2241 2242static struct drm_i915_error_object * 2243i915_error_object_create(struct drm_i915_private *dev_priv, 2244 struct drm_i915_gem_object *src) 2245{ 2246 struct drm_i915_error_object *dst; 2247 struct sf_buf *sf; 2248 void *d, *s; 2249 int page, page_count; 2250 u32 reloc_offset; 2251 2252 if (src == NULL || src->pages == NULL) 2253 return NULL; 2254 2255 page_count = src->base.size / PAGE_SIZE; 2256 2257 dst = malloc(sizeof(*dst) + page_count * sizeof(u32 *), DRM_I915_GEM, 2258 M_NOWAIT); 2259 if (dst == NULL) 2260 return (NULL); 2261 2262 reloc_offset = src->gtt_offset; 2263 for (page = 0; page < page_count; page++) { 2264 d = malloc(PAGE_SIZE, DRM_I915_GEM, M_NOWAIT); 2265 if (d == NULL) 2266 goto unwind; 2267 2268 if (reloc_offset < dev_priv->mm.gtt_mappable_end && 2269 src->has_global_gtt_mapping) { 2270 /* Simply ignore tiling or any overlapping fence. 2271 * It's part of the error state, and this hopefully 2272 * captures what the GPU read. 2273 */ 2274 s = pmap_mapdev_attr(src->base.dev->agp->base + 2275 reloc_offset, PAGE_SIZE, PAT_WRITE_COMBINING); 2276 memcpy(d, s, PAGE_SIZE); 2277 pmap_unmapdev((vm_offset_t)s, PAGE_SIZE); 2278 } else { 2279 drm_clflush_pages(&src->pages[page], 1); 2280 2281 sched_pin(); 2282 sf = sf_buf_alloc(src->pages[page], SFB_CPUPRIVATE | 2283 SFB_NOWAIT); 2284 if (sf != NULL) { 2285 s = (void *)(uintptr_t)sf_buf_kva(sf); 2286 memcpy(d, s, PAGE_SIZE); 2287 sf_buf_free(sf); 2288 } else { 2289 bzero(d, PAGE_SIZE); 2290 strcpy(d, "XXXKIB"); 2291 } 2292 sched_unpin(); 2293 2294 drm_clflush_pages(&src->pages[page], 1); 2295 } 2296 2297 dst->pages[page] = d; 2298 2299 reloc_offset += PAGE_SIZE; 2300 } 2301 dst->page_count = page_count; 2302 dst->gtt_offset = src->gtt_offset; 2303 2304 return (dst); 2305 2306unwind: 2307 while (page--) 2308 free(dst->pages[page], DRM_I915_GEM); 2309 free(dst, DRM_I915_GEM); 2310 return (NULL); 2311} 2312 2313static void 2314i915_error_object_free(struct drm_i915_error_object *obj) 2315{ 2316 int page; 2317 2318 if (obj == NULL) 2319 return; 2320 2321 for (page = 0; page < obj->page_count; page++) 2322 free(obj->pages[page], DRM_I915_GEM); 2323 2324 free(obj, DRM_I915_GEM); 2325} 2326 2327void 2328i915_error_state_free(struct drm_i915_error_state *error) 2329{ 2330 int i; 2331 2332 for (i = 0; i < ARRAY_SIZE(error->ring); i++) { 2333 i915_error_object_free(error->ring[i].batchbuffer); 2334 i915_error_object_free(error->ring[i].ringbuffer); 2335 free(error->ring[i].requests, DRM_I915_GEM); 2336 } 2337 2338 free(error->active_bo, DRM_I915_GEM); 2339 free(error->overlay, DRM_I915_GEM); 2340 free(error, DRM_I915_GEM); 2341} 2342 2343static void capture_bo(struct drm_i915_error_buffer *err, 2344 struct drm_i915_gem_object *obj) 2345{ 2346 err->size = obj->base.size; 2347 err->name = obj->base.name; 2348 err->seqno = obj->last_rendering_seqno; 2349 err->gtt_offset = obj->gtt_offset; 2350 err->read_domains = obj->base.read_domains; 2351 err->write_domain = obj->base.write_domain; 2352 err->fence_reg = obj->fence_reg; 2353 err->pinned = 0; 2354 if (obj->pin_count > 0) 2355 err->pinned = 1; 2356 if (obj->user_pin_count > 0) 2357 err->pinned = -1; 2358 err->tiling = obj->tiling_mode; 2359 err->dirty = obj->dirty; 2360 err->purgeable = obj->madv != I915_MADV_WILLNEED; 2361 err->ring = obj->ring ? obj->ring->id : -1; 2362 err->cache_level = obj->cache_level; 2363} 2364 2365static u32 capture_active_bo(struct drm_i915_error_buffer *err, 2366 int count, struct list_head *head) 2367{ 2368 struct drm_i915_gem_object *obj; 2369 int i = 0; 2370 2371 list_for_each_entry(obj, head, mm_list) { 2372 capture_bo(err++, obj); 2373 if (++i == count) 2374 break; 2375 } 2376 2377 return i; 2378} 2379 2380static u32 capture_pinned_bo(struct drm_i915_error_buffer *err, 2381 int count, struct list_head *head) 2382{ 2383 struct drm_i915_gem_object *obj; 2384 int i = 0; 2385 2386 list_for_each_entry(obj, head, gtt_list) { 2387 if (obj->pin_count == 0) 2388 continue; 2389 2390 capture_bo(err++, obj); 2391 if (++i == count) 2392 break; 2393 } 2394 2395 return i; 2396} 2397 2398static void 2399i915_gem_record_fences(struct drm_device *dev, 2400 struct drm_i915_error_state *error) 2401{ 2402 struct drm_i915_private *dev_priv = dev->dev_private; 2403 int i; 2404 2405 /* Fences */ 2406 switch (INTEL_INFO(dev)->gen) { 2407 case 7: 2408 case 6: 2409 for (i = 0; i < 16; i++) 2410 error->fence[i] = I915_READ64(FENCE_REG_SANDYBRIDGE_0 + (i * 8)); 2411 break; 2412 case 5: 2413 case 4: 2414 for (i = 0; i < 16; i++) 2415 error->fence[i] = I915_READ64(FENCE_REG_965_0 + 2416 (i * 8)); 2417 break; 2418 case 3: 2419 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) 2420 for (i = 0; i < 8; i++) 2421 error->fence[i+8] = I915_READ(FENCE_REG_945_8 + 2422 (i * 4)); 2423 case 2: 2424 for (i = 0; i < 8; i++) 2425 error->fence[i] = I915_READ(FENCE_REG_830_0 + (i * 4)); 2426 break; 2427 2428 } 2429} 2430 2431static struct drm_i915_error_object * 2432i915_error_first_batchbuffer(struct drm_i915_private *dev_priv, 2433 struct intel_ring_buffer *ring) 2434{ 2435 struct drm_i915_gem_object *obj; 2436 u32 seqno; 2437 2438 if (!ring->get_seqno) 2439 return (NULL); 2440 2441 seqno = ring->get_seqno(ring); 2442 list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) { 2443 if (obj->ring != ring) 2444 continue; 2445 2446 if (i915_seqno_passed(seqno, obj->last_rendering_seqno)) 2447 continue; 2448 2449 if ((obj->base.read_domains & I915_GEM_DOMAIN_COMMAND) == 0) 2450 continue; 2451 2452 /* We need to copy these to an anonymous buffer as the simplest 2453 * method to avoid being overwritten by userspace. 2454 */ 2455 return (i915_error_object_create(dev_priv, obj)); 2456 } 2457 2458 return NULL; 2459} 2460 2461static void 2462i915_record_ring_state(struct drm_device *dev, 2463 struct drm_i915_error_state *error, 2464 struct intel_ring_buffer *ring) 2465{ 2466 struct drm_i915_private *dev_priv = dev->dev_private; 2467 2468 if (INTEL_INFO(dev)->gen >= 6) { 2469 error->fault_reg[ring->id] = I915_READ(RING_FAULT_REG(ring)); 2470 error->semaphore_mboxes[ring->id][0] 2471 = I915_READ(RING_SYNC_0(ring->mmio_base)); 2472 error->semaphore_mboxes[ring->id][1] 2473 = I915_READ(RING_SYNC_1(ring->mmio_base)); 2474 } 2475 2476 if (INTEL_INFO(dev)->gen >= 4) { 2477 error->faddr[ring->id] = I915_READ(RING_DMA_FADD(ring->mmio_base)); 2478 error->ipeir[ring->id] = I915_READ(RING_IPEIR(ring->mmio_base)); 2479 error->ipehr[ring->id] = I915_READ(RING_IPEHR(ring->mmio_base)); 2480 error->instdone[ring->id] = I915_READ(RING_INSTDONE(ring->mmio_base)); 2481 error->instps[ring->id] = I915_READ(RING_INSTPS(ring->mmio_base)); 2482 if (ring->id == RCS) { 2483 error->instdone1 = I915_READ(INSTDONE1); 2484 error->bbaddr = I915_READ64(BB_ADDR); 2485 } 2486 } else { 2487 error->faddr[ring->id] = I915_READ(DMA_FADD_I8XX); 2488 error->ipeir[ring->id] = I915_READ(IPEIR); 2489 error->ipehr[ring->id] = I915_READ(IPEHR); 2490 error->instdone[ring->id] = I915_READ(INSTDONE); 2491 } 2492 2493 sleepq_lock(ring); 2494 error->waiting[ring->id] = sleepq_sleepcnt(ring, 0) != 0; 2495 sleepq_release(ring); 2496 error->instpm[ring->id] = I915_READ(RING_INSTPM(ring->mmio_base)); 2497 error->seqno[ring->id] = ring->get_seqno(ring); 2498 error->acthd[ring->id] = intel_ring_get_active_head(ring); 2499 error->head[ring->id] = I915_READ_HEAD(ring); 2500 error->tail[ring->id] = I915_READ_TAIL(ring); 2501 2502 error->cpu_ring_head[ring->id] = ring->head; 2503 error->cpu_ring_tail[ring->id] = ring->tail; 2504} 2505 2506static void 2507i915_gem_record_rings(struct drm_device *dev, 2508 struct drm_i915_error_state *error) 2509{ 2510 struct drm_i915_private *dev_priv = dev->dev_private; 2511 struct intel_ring_buffer *ring; 2512 struct drm_i915_gem_request *request; 2513 int i, count; 2514 2515 for_each_ring(ring, dev_priv, i) { 2516 i915_record_ring_state(dev, error, ring); 2517 2518 error->ring[i].batchbuffer = 2519 i915_error_first_batchbuffer(dev_priv, ring); 2520 2521 error->ring[i].ringbuffer = 2522 i915_error_object_create(dev_priv, ring->obj); 2523 2524 count = 0; 2525 list_for_each_entry(request, &ring->request_list, list) 2526 count++; 2527 2528 error->ring[i].num_requests = count; 2529 error->ring[i].requests = malloc(count * 2530 sizeof(struct drm_i915_error_request), DRM_I915_GEM, 2531 M_WAITOK); 2532 if (error->ring[i].requests == NULL) { 2533 error->ring[i].num_requests = 0; 2534 continue; 2535 } 2536 2537 count = 0; 2538 list_for_each_entry(request, &ring->request_list, list) { 2539 struct drm_i915_error_request *erq; 2540 2541 erq = &error->ring[i].requests[count++]; 2542 erq->seqno = request->seqno; 2543 erq->jiffies = request->emitted_jiffies; 2544 erq->tail = request->tail; 2545 } 2546 } 2547} 2548 2549static void 2550i915_capture_error_state(struct drm_device *dev) 2551{ 2552 struct drm_i915_private *dev_priv = dev->dev_private; 2553 struct drm_i915_gem_object *obj; 2554 struct drm_i915_error_state *error; 2555 int i, pipe; 2556 2557 mtx_lock(&dev_priv->error_lock); 2558 error = dev_priv->first_error; 2559 mtx_unlock(&dev_priv->error_lock); 2560 if (error != NULL) 2561 return; 2562 2563 /* Account for pipe specific data like PIPE*STAT */ 2564 error = malloc(sizeof(*error), DRM_I915_GEM, M_NOWAIT | M_ZERO); 2565 if (error == NULL) { 2566 DRM_DEBUG("out of memory, not capturing error state\n"); 2567 return; 2568 } 2569 2570 DRM_INFO("capturing error event; look for more information in " 2571 "sysctl hw.dri.%d.info.i915_error_state\n", dev->sysctl_node_idx); 2572 2573 refcount_init(&error->ref, 1); 2574 error->eir = I915_READ(EIR); 2575 error->pgtbl_er = I915_READ(PGTBL_ER); 2576 2577 if (HAS_PCH_SPLIT(dev)) 2578 error->ier = I915_READ(DEIER) | I915_READ(GTIER); 2579 else if (IS_VALLEYVIEW(dev)) 2580 error->ier = I915_READ(GTIER) | I915_READ(VLV_IER); 2581 else if (IS_GEN2(dev)) 2582 error->ier = I915_READ16(IER); 2583 else 2584 error->ier = I915_READ(IER); 2585 2586 for_each_pipe(pipe) 2587 error->pipestat[pipe] = I915_READ(PIPESTAT(pipe)); 2588 2589 if (INTEL_INFO(dev)->gen >= 6) { 2590 error->error = I915_READ(ERROR_GEN6); 2591 error->done_reg = I915_READ(DONE_REG); 2592 } 2593 2594 i915_gem_record_fences(dev, error); 2595 i915_gem_record_rings(dev, error); 2596 2597 /* Record buffers on the active and pinned lists. */ 2598 error->active_bo = NULL; 2599 error->pinned_bo = NULL; 2600 2601 i = 0; 2602 list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) 2603 i++; 2604 error->active_bo_count = i; 2605 list_for_each_entry(obj, &dev_priv->mm.gtt_list, mm_list) 2606 if (obj->pin_count) 2607 i++; 2608 error->pinned_bo_count = i - error->active_bo_count; 2609 2610 error->active_bo = NULL; 2611 error->pinned_bo = NULL; 2612 if (i) { 2613 error->active_bo = malloc(sizeof(*error->active_bo) * i, 2614 DRM_I915_GEM, M_NOWAIT); 2615 if (error->active_bo) 2616 error->pinned_bo = error->active_bo + 2617 error->active_bo_count; 2618 } 2619 2620 if (error->active_bo) 2621 error->active_bo_count = 2622 capture_active_bo(error->active_bo, 2623 error->active_bo_count, 2624 &dev_priv->mm.active_list); 2625 2626 if (error->pinned_bo) 2627 error->pinned_bo_count = 2628 capture_pinned_bo(error->pinned_bo, 2629 error->pinned_bo_count, 2630 &dev_priv->mm.gtt_list); 2631 2632 microtime(&error->time); 2633 2634 error->overlay = intel_overlay_capture_error_state(dev); 2635 error->display = intel_display_capture_error_state(dev); 2636 2637 mtx_lock(&dev_priv->error_lock); 2638 if (dev_priv->first_error == NULL) { 2639 dev_priv->first_error = error; 2640 error = NULL; 2641 } 2642 mtx_unlock(&dev_priv->error_lock); 2643 2644 if (error != NULL) 2645 i915_error_state_free(error); 2646} 2647 2648void 2649i915_destroy_error_state(struct drm_device *dev) 2650{ 2651 struct drm_i915_private *dev_priv = dev->dev_private; 2652 struct drm_i915_error_state *error; 2653 2654 mtx_lock(&dev_priv->error_lock); 2655 error = dev_priv->first_error; 2656 dev_priv->first_error = NULL; 2657 mtx_unlock(&dev_priv->error_lock); 2658 2659 if (error != NULL && refcount_release(&error->ref)) 2660 i915_error_state_free(error); 2661} 2662