1/*- 2 * Copyright © 2008 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eric Anholt <eric@anholt.net> 25 * 26 * Copyright (c) 2011 The FreeBSD Foundation 27 * All rights reserved. 28 * 29 * This software was developed by Konstantin Belousov under sponsorship from 30 * the FreeBSD Foundation. 31 * 32 * Redistribution and use in source and binary forms, with or without 33 * modification, are permitted provided that the following conditions 34 * are met: 35 * 1. Redistributions of source code must retain the above copyright 36 * notice, this list of conditions and the following disclaimer. 37 * 2. Redistributions in binary form must reproduce the above copyright 38 * notice, this list of conditions and the following disclaimer in the 39 * documentation and/or other materials provided with the distribution. 40 * 41 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 42 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 43 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 44 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 45 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 46 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 47 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 48 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 49 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 50 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 51 * SUCH DAMAGE. 52 */ 53 54#include <sys/cdefs.h> 55__FBSDID("$FreeBSD$"); 56 57#include <dev/drm2/drmP.h> 58#include <dev/drm2/drm.h> 59#include <dev/drm2/i915/i915_drm.h> 60#include <dev/drm2/i915/i915_drv.h> 61#include <dev/drm2/i915/intel_drv.h> 62#include <dev/drm2/i915/intel_ringbuffer.h> 63#include <sys/resourcevar.h> 64#include <sys/sched.h> 65#include <sys/sf_buf.h> 66 67#include <vm/vm.h> 68#include <vm/vm_pageout.h> 69 70static void i915_gem_object_flush_cpu_write_domain( 71 struct drm_i915_gem_object *obj); 72static uint32_t i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, 73 int tiling_mode); 74static uint32_t i915_gem_get_gtt_alignment(struct drm_device *dev, 75 uint32_t size, int tiling_mode); 76static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, 77 unsigned alignment, bool map_and_fenceable); 78static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, 79 int flags); 80static void i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj); 81static int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, 82 bool write); 83static void i915_gem_object_set_to_full_cpu_read_domain( 84 struct drm_i915_gem_object *obj); 85static int i915_gem_object_set_cpu_read_domain_range( 86 struct drm_i915_gem_object *obj, uint64_t offset, uint64_t size); 87static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj); 88static void i915_gem_object_truncate(struct drm_i915_gem_object *obj); 89static int i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj); 90static bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj); 91static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj); 92static vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex); 93static void i915_gem_process_flushing_list(struct intel_ring_buffer *ring, 94 uint32_t flush_domains); 95static void i915_gem_clear_fence_reg(struct drm_device *dev, 96 struct drm_i915_fence_reg *reg); 97static void i915_gem_reset_fences(struct drm_device *dev); 98static void i915_gem_retire_task_handler(void *arg, int pending); 99static int i915_gem_phys_pwrite(struct drm_device *dev, 100 struct drm_i915_gem_object *obj, uint64_t data_ptr, uint64_t offset, 101 uint64_t size, struct drm_file *file_priv); 102static void i915_gem_lowmem(void *arg); 103 104MALLOC_DEFINE(DRM_I915_GEM, "i915gem", "Allocations from i915 gem"); 105long i915_gem_wired_pages_cnt; 106 107static void 108i915_gem_info_add_obj(struct drm_i915_private *dev_priv, size_t size) 109{ 110 111 dev_priv->mm.object_count++; 112 dev_priv->mm.object_memory += size; 113} 114 115static void 116i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, size_t size) 117{ 118 119 dev_priv->mm.object_count--; 120 dev_priv->mm.object_memory -= size; 121} 122 123static int 124i915_gem_wait_for_error(struct drm_device *dev) 125{ 126 struct drm_i915_private *dev_priv; 127 int ret; 128 129 dev_priv = dev->dev_private; 130 if (!atomic_load_acq_int(&dev_priv->mm.wedged)) 131 return (0); 132 133 mtx_lock(&dev_priv->error_completion_lock); 134 while (dev_priv->error_completion == 0) { 135 ret = -msleep(&dev_priv->error_completion, 136 &dev_priv->error_completion_lock, PCATCH, "915wco", 0); 137 if (ret != 0) { 138 mtx_unlock(&dev_priv->error_completion_lock); 139 return (ret); 140 } 141 } 142 mtx_unlock(&dev_priv->error_completion_lock); 143 144 if (atomic_load_acq_int(&dev_priv->mm.wedged)) { 145 mtx_lock(&dev_priv->error_completion_lock); 146 dev_priv->error_completion++; 147 mtx_unlock(&dev_priv->error_completion_lock); 148 } 149 return (0); 150} 151 152int 153i915_mutex_lock_interruptible(struct drm_device *dev) 154{ 155 struct drm_i915_private *dev_priv; 156 int ret; 157 158 dev_priv = dev->dev_private; 159 ret = i915_gem_wait_for_error(dev); 160 if (ret != 0) 161 return (ret); 162 163 /* 164 * interruptible shall it be. might indeed be if dev_lock is 165 * changed to sx 166 */ 167 ret = sx_xlock_sig(&dev->dev_struct_lock); 168 if (ret != 0) 169 return (-ret); 170 171 return (0); 172} 173 174 175static void 176i915_gem_free_object_tail(struct drm_i915_gem_object *obj) 177{ 178 struct drm_device *dev; 179 drm_i915_private_t *dev_priv; 180 int ret; 181 182 dev = obj->base.dev; 183 dev_priv = dev->dev_private; 184 185 ret = i915_gem_object_unbind(obj); 186 if (ret == -ERESTART) { 187 list_move(&obj->mm_list, &dev_priv->mm.deferred_free_list); 188 return; 189 } 190 191 CTR1(KTR_DRM, "object_destroy_tail %p", obj); 192 drm_gem_free_mmap_offset(&obj->base); 193 drm_gem_object_release(&obj->base); 194 i915_gem_info_remove_obj(dev_priv, obj->base.size); 195 196 free(obj->page_cpu_valid, DRM_I915_GEM); 197 free(obj->bit_17, DRM_I915_GEM); 198 free(obj, DRM_I915_GEM); 199} 200 201void 202i915_gem_free_object(struct drm_gem_object *gem_obj) 203{ 204 struct drm_i915_gem_object *obj; 205 struct drm_device *dev; 206 207 obj = to_intel_bo(gem_obj); 208 dev = obj->base.dev; 209 210 while (obj->pin_count > 0) 211 i915_gem_object_unpin(obj); 212 213 if (obj->phys_obj != NULL) 214 i915_gem_detach_phys_object(dev, obj); 215 216 i915_gem_free_object_tail(obj); 217} 218 219static void 220init_ring_lists(struct intel_ring_buffer *ring) 221{ 222 223 INIT_LIST_HEAD(&ring->active_list); 224 INIT_LIST_HEAD(&ring->request_list); 225 INIT_LIST_HEAD(&ring->gpu_write_list); 226} 227 228void 229i915_gem_load(struct drm_device *dev) 230{ 231 drm_i915_private_t *dev_priv; 232 int i; 233 234 dev_priv = dev->dev_private; 235 236 INIT_LIST_HEAD(&dev_priv->mm.active_list); 237 INIT_LIST_HEAD(&dev_priv->mm.flushing_list); 238 INIT_LIST_HEAD(&dev_priv->mm.inactive_list); 239 INIT_LIST_HEAD(&dev_priv->mm.pinned_list); 240 INIT_LIST_HEAD(&dev_priv->mm.fence_list); 241 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list); 242 INIT_LIST_HEAD(&dev_priv->mm.gtt_list); 243 for (i = 0; i < I915_NUM_RINGS; i++) 244 init_ring_lists(&dev_priv->rings[i]); 245 for (i = 0; i < I915_MAX_NUM_FENCES; i++) 246 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); 247 TIMEOUT_TASK_INIT(dev_priv->tq, &dev_priv->mm.retire_task, 0, 248 i915_gem_retire_task_handler, dev_priv); 249 dev_priv->error_completion = 0; 250 251 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ 252 if (IS_GEN3(dev)) { 253 u32 tmp = I915_READ(MI_ARB_STATE); 254 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) { 255 /* 256 * arb state is a masked write, so set bit + 257 * bit in mask. 258 */ 259 tmp = MI_ARB_C3_LP_WRITE_ENABLE | 260 (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT); 261 I915_WRITE(MI_ARB_STATE, tmp); 262 } 263 } 264 265 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL; 266 267 /* Old X drivers will take 0-2 for front, back, depth buffers */ 268 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 269 dev_priv->fence_reg_start = 3; 270 271 if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || 272 IS_G33(dev)) 273 dev_priv->num_fence_regs = 16; 274 else 275 dev_priv->num_fence_regs = 8; 276 277 /* Initialize fence registers to zero */ 278 for (i = 0; i < dev_priv->num_fence_regs; i++) { 279 i915_gem_clear_fence_reg(dev, &dev_priv->fence_regs[i]); 280 } 281 i915_gem_detect_bit_6_swizzle(dev); 282 dev_priv->mm.interruptible = true; 283 284 dev_priv->mm.i915_lowmem = EVENTHANDLER_REGISTER(vm_lowmem, 285 i915_gem_lowmem, dev, EVENTHANDLER_PRI_ANY); 286} 287 288int 289i915_gem_do_init(struct drm_device *dev, unsigned long start, 290 unsigned long mappable_end, unsigned long end) 291{ 292 drm_i915_private_t *dev_priv; 293 unsigned long mappable; 294 int error; 295 296 dev_priv = dev->dev_private; 297 mappable = min(end, mappable_end) - start; 298 299 drm_mm_init(&dev_priv->mm.gtt_space, start, end - start); 300 301 dev_priv->mm.gtt_start = start; 302 dev_priv->mm.gtt_mappable_end = mappable_end; 303 dev_priv->mm.gtt_end = end; 304 dev_priv->mm.gtt_total = end - start; 305 dev_priv->mm.mappable_gtt_total = mappable; 306 307 /* Take over this portion of the GTT */ 308 intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE); 309 device_printf(dev->device, 310 "taking over the fictitious range 0x%lx-0x%lx\n", 311 dev->agp->base + start, dev->agp->base + start + mappable); 312 error = -vm_phys_fictitious_reg_range(dev->agp->base + start, 313 dev->agp->base + start + mappable, VM_MEMATTR_WRITE_COMBINING); 314 return (error); 315} 316 317int 318i915_gem_init_ioctl(struct drm_device *dev, void *data, 319 struct drm_file *file) 320{ 321 struct drm_i915_gem_init *args; 322 drm_i915_private_t *dev_priv; 323 324 dev_priv = dev->dev_private; 325 args = data; 326 327 if (args->gtt_start >= args->gtt_end || 328 (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1)) 329 return (-EINVAL); 330 331 if (mtx_initialized(&dev_priv->mm.gtt_space.unused_lock)) 332 return (-EBUSY); 333 /* 334 * XXXKIB. The second-time initialization should be guarded 335 * against. 336 */ 337 return (i915_gem_do_init(dev, args->gtt_start, args->gtt_end, 338 args->gtt_end)); 339} 340 341int 342i915_gem_idle(struct drm_device *dev) 343{ 344 drm_i915_private_t *dev_priv; 345 int ret; 346 347 dev_priv = dev->dev_private; 348 if (dev_priv->mm.suspended) 349 return (0); 350 351 ret = i915_gpu_idle(dev, true); 352 if (ret != 0) 353 return (ret); 354 355 /* Under UMS, be paranoid and evict. */ 356 if (!drm_core_check_feature(dev, DRIVER_MODESET)) { 357 ret = i915_gem_evict_inactive(dev, false); 358 if (ret != 0) 359 return ret; 360 } 361 362 i915_gem_reset_fences(dev); 363 364 /* Hack! Don't let anybody do execbuf while we don't control the chip. 365 * We need to replace this with a semaphore, or something. 366 * And not confound mm.suspended! 367 */ 368 dev_priv->mm.suspended = 1; 369 callout_stop(&dev_priv->hangcheck_timer); 370 371 i915_kernel_lost_context(dev); 372 i915_gem_cleanup_ringbuffer(dev); 373 374 /* Cancel the retire work handler, which should be idle now. */ 375 taskqueue_cancel_timeout(dev_priv->tq, &dev_priv->mm.retire_task, NULL); 376 return (ret); 377} 378 379void 380i915_gem_init_swizzling(struct drm_device *dev) 381{ 382 drm_i915_private_t *dev_priv; 383 384 dev_priv = dev->dev_private; 385 386 if (INTEL_INFO(dev)->gen < 5 || 387 dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) 388 return; 389 390 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | 391 DISP_TILE_SURFACE_SWIZZLING); 392 393 if (IS_GEN5(dev)) 394 return; 395 396 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL); 397 if (IS_GEN6(dev)) 398 I915_WRITE(ARB_MODE, ARB_MODE_ENABLE(ARB_MODE_SWIZZLE_SNB)); 399 else 400 I915_WRITE(ARB_MODE, ARB_MODE_ENABLE(ARB_MODE_SWIZZLE_IVB)); 401} 402 403void 404i915_gem_init_ppgtt(struct drm_device *dev) 405{ 406 drm_i915_private_t *dev_priv; 407 struct i915_hw_ppgtt *ppgtt; 408 uint32_t pd_offset, pd_entry; 409 vm_paddr_t pt_addr; 410 struct intel_ring_buffer *ring; 411 u_int first_pd_entry_in_global_pt, i; 412 413 dev_priv = dev->dev_private; 414 ppgtt = dev_priv->mm.aliasing_ppgtt; 415 if (ppgtt == NULL) 416 return; 417 418 first_pd_entry_in_global_pt = 512 * 1024 - I915_PPGTT_PD_ENTRIES; 419 for (i = 0; i < ppgtt->num_pd_entries; i++) { 420 pt_addr = VM_PAGE_TO_PHYS(ppgtt->pt_pages[i]); 421 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr); 422 pd_entry |= GEN6_PDE_VALID; 423 intel_gtt_write(first_pd_entry_in_global_pt + i, pd_entry); 424 } 425 intel_gtt_read_pte(first_pd_entry_in_global_pt); 426 427 pd_offset = ppgtt->pd_offset; 428 pd_offset /= 64; /* in cachelines, */ 429 pd_offset <<= 16; 430 431 if (INTEL_INFO(dev)->gen == 6) { 432 uint32_t ecochk = I915_READ(GAM_ECOCHK); 433 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | 434 ECOCHK_PPGTT_CACHE64B); 435 I915_WRITE(GFX_MODE, GFX_MODE_ENABLE(GFX_PPGTT_ENABLE)); 436 } else if (INTEL_INFO(dev)->gen >= 7) { 437 I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B); 438 /* GFX_MODE is per-ring on gen7+ */ 439 } 440 441 for (i = 0; i < I915_NUM_RINGS; i++) { 442 ring = &dev_priv->rings[i]; 443 444 if (INTEL_INFO(dev)->gen >= 7) 445 I915_WRITE(RING_MODE_GEN7(ring), 446 GFX_MODE_ENABLE(GFX_PPGTT_ENABLE)); 447 448 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); 449 I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset); 450 } 451} 452 453int 454i915_gem_init_hw(struct drm_device *dev) 455{ 456 drm_i915_private_t *dev_priv; 457 int ret; 458 459 dev_priv = dev->dev_private; 460 461 i915_gem_init_swizzling(dev); 462 463 ret = intel_init_render_ring_buffer(dev); 464 if (ret != 0) 465 return (ret); 466 467 if (HAS_BSD(dev)) { 468 ret = intel_init_bsd_ring_buffer(dev); 469 if (ret != 0) 470 goto cleanup_render_ring; 471 } 472 473 if (HAS_BLT(dev)) { 474 ret = intel_init_blt_ring_buffer(dev); 475 if (ret != 0) 476 goto cleanup_bsd_ring; 477 } 478 479 dev_priv->next_seqno = 1; 480 i915_gem_context_init(dev); 481 i915_gem_init_ppgtt(dev); 482 return (0); 483 484cleanup_bsd_ring: 485 intel_cleanup_ring_buffer(&dev_priv->rings[VCS]); 486cleanup_render_ring: 487 intel_cleanup_ring_buffer(&dev_priv->rings[RCS]); 488 return (ret); 489} 490 491int 492i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, 493 struct drm_file *file) 494{ 495 struct drm_i915_private *dev_priv; 496 struct drm_i915_gem_get_aperture *args; 497 struct drm_i915_gem_object *obj; 498 size_t pinned; 499 500 dev_priv = dev->dev_private; 501 args = data; 502 503 if (!(dev->driver->driver_features & DRIVER_GEM)) 504 return (-ENODEV); 505 506 pinned = 0; 507 DRM_LOCK(dev); 508 list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list) 509 pinned += obj->gtt_space->size; 510 DRM_UNLOCK(dev); 511 512 args->aper_size = dev_priv->mm.gtt_total; 513 args->aper_available_size = args->aper_size - pinned; 514 515 return (0); 516} 517 518int 519i915_gem_object_pin(struct drm_i915_gem_object *obj, uint32_t alignment, 520 bool map_and_fenceable) 521{ 522 struct drm_device *dev; 523 struct drm_i915_private *dev_priv; 524 int ret; 525 526 dev = obj->base.dev; 527 dev_priv = dev->dev_private; 528 529 KASSERT(obj->pin_count != DRM_I915_GEM_OBJECT_MAX_PIN_COUNT, 530 ("Max pin count")); 531 532 if (obj->gtt_space != NULL) { 533 if ((alignment && obj->gtt_offset & (alignment - 1)) || 534 (map_and_fenceable && !obj->map_and_fenceable)) { 535 DRM_DEBUG("bo is already pinned with incorrect alignment:" 536 " offset=%x, req.alignment=%x, req.map_and_fenceable=%d," 537 " obj->map_and_fenceable=%d\n", 538 obj->gtt_offset, alignment, 539 map_and_fenceable, 540 obj->map_and_fenceable); 541 ret = i915_gem_object_unbind(obj); 542 if (ret != 0) 543 return (ret); 544 } 545 } 546 547 if (obj->gtt_space == NULL) { 548 ret = i915_gem_object_bind_to_gtt(obj, alignment, 549 map_and_fenceable); 550 if (ret) 551 return (ret); 552 } 553 554 if (obj->pin_count++ == 0 && !obj->active) 555 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list); 556 obj->pin_mappable |= map_and_fenceable; 557 558#if 1 559 KIB_NOTYET(); 560#else 561 WARN_ON(i915_verify_lists(dev)); 562#endif 563 return (0); 564} 565 566void 567i915_gem_object_unpin(struct drm_i915_gem_object *obj) 568{ 569 struct drm_device *dev; 570 drm_i915_private_t *dev_priv; 571 572 dev = obj->base.dev; 573 dev_priv = dev->dev_private; 574 575#if 1 576 KIB_NOTYET(); 577#else 578 WARN_ON(i915_verify_lists(dev)); 579#endif 580 581 KASSERT(obj->pin_count != 0, ("zero pin count")); 582 KASSERT(obj->gtt_space != NULL, ("No gtt mapping")); 583 584 if (--obj->pin_count == 0) { 585 if (!obj->active) 586 list_move_tail(&obj->mm_list, 587 &dev_priv->mm.inactive_list); 588 obj->pin_mappable = false; 589 } 590#if 1 591 KIB_NOTYET(); 592#else 593 WARN_ON(i915_verify_lists(dev)); 594#endif 595} 596 597int 598i915_gem_pin_ioctl(struct drm_device *dev, void *data, 599 struct drm_file *file) 600{ 601 struct drm_i915_gem_pin *args; 602 struct drm_i915_gem_object *obj; 603 struct drm_gem_object *gobj; 604 int ret; 605 606 args = data; 607 608 ret = i915_mutex_lock_interruptible(dev); 609 if (ret != 0) 610 return ret; 611 612 gobj = drm_gem_object_lookup(dev, file, args->handle); 613 if (gobj == NULL) { 614 ret = -ENOENT; 615 goto unlock; 616 } 617 obj = to_intel_bo(gobj); 618 619 if (obj->madv != I915_MADV_WILLNEED) { 620 DRM_ERROR("Attempting to pin a purgeable buffer\n"); 621 ret = -EINVAL; 622 goto out; 623 } 624 625 if (obj->pin_filp != NULL && obj->pin_filp != file) { 626 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n", 627 args->handle); 628 ret = -EINVAL; 629 goto out; 630 } 631 632 obj->user_pin_count++; 633 obj->pin_filp = file; 634 if (obj->user_pin_count == 1) { 635 ret = i915_gem_object_pin(obj, args->alignment, true); 636 if (ret != 0) 637 goto out; 638 } 639 640 /* XXX - flush the CPU caches for pinned objects 641 * as the X server doesn't manage domains yet 642 */ 643 i915_gem_object_flush_cpu_write_domain(obj); 644 args->offset = obj->gtt_offset; 645out: 646 drm_gem_object_unreference(&obj->base); 647unlock: 648 DRM_UNLOCK(dev); 649 return (ret); 650} 651 652int 653i915_gem_unpin_ioctl(struct drm_device *dev, void *data, 654 struct drm_file *file) 655{ 656 struct drm_i915_gem_pin *args; 657 struct drm_i915_gem_object *obj; 658 int ret; 659 660 args = data; 661 ret = i915_mutex_lock_interruptible(dev); 662 if (ret != 0) 663 return (ret); 664 665 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 666 if (&obj->base == NULL) { 667 ret = -ENOENT; 668 goto unlock; 669 } 670 671 if (obj->pin_filp != file) { 672 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", 673 args->handle); 674 ret = -EINVAL; 675 goto out; 676 } 677 obj->user_pin_count--; 678 if (obj->user_pin_count == 0) { 679 obj->pin_filp = NULL; 680 i915_gem_object_unpin(obj); 681 } 682 683out: 684 drm_gem_object_unreference(&obj->base); 685unlock: 686 DRM_UNLOCK(dev); 687 return (ret); 688} 689 690int 691i915_gem_busy_ioctl(struct drm_device *dev, void *data, 692 struct drm_file *file) 693{ 694 struct drm_i915_gem_busy *args; 695 struct drm_i915_gem_object *obj; 696 struct drm_i915_gem_request *request; 697 int ret; 698 699 args = data; 700 701 ret = i915_mutex_lock_interruptible(dev); 702 if (ret != 0) 703 return ret; 704 705 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 706 if (&obj->base == NULL) { 707 ret = -ENOENT; 708 goto unlock; 709 } 710 711 args->busy = obj->active; 712 if (args->busy) { 713 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 714 ret = i915_gem_flush_ring(obj->ring, 715 0, obj->base.write_domain); 716 } else if (obj->ring->outstanding_lazy_request == 717 obj->last_rendering_seqno) { 718 request = malloc(sizeof(*request), DRM_I915_GEM, 719 M_WAITOK | M_ZERO); 720 ret = i915_add_request(obj->ring, NULL, request); 721 if (ret != 0) 722 free(request, DRM_I915_GEM); 723 } 724 725 i915_gem_retire_requests_ring(obj->ring); 726 args->busy = obj->active; 727 } 728 729 drm_gem_object_unreference(&obj->base); 730unlock: 731 DRM_UNLOCK(dev); 732 return (ret); 733} 734 735static int 736i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) 737{ 738 struct drm_i915_private *dev_priv; 739 struct drm_i915_file_private *file_priv; 740 unsigned long recent_enough; 741 struct drm_i915_gem_request *request; 742 struct intel_ring_buffer *ring; 743 u32 seqno; 744 int ret; 745 746 dev_priv = dev->dev_private; 747 if (atomic_load_acq_int(&dev_priv->mm.wedged)) 748 return (-EIO); 749 750 file_priv = file->driver_priv; 751 recent_enough = ticks - (20 * hz / 1000); 752 ring = NULL; 753 seqno = 0; 754 755 mtx_lock(&file_priv->mm.lck); 756 list_for_each_entry(request, &file_priv->mm.request_list, client_list) { 757 if (time_after_eq(request->emitted_jiffies, recent_enough)) 758 break; 759 ring = request->ring; 760 seqno = request->seqno; 761 } 762 mtx_unlock(&file_priv->mm.lck); 763 if (seqno == 0) 764 return (0); 765 766 ret = 0; 767 mtx_lock(&ring->irq_lock); 768 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) { 769 if (ring->irq_get(ring)) { 770 while (ret == 0 && 771 !(i915_seqno_passed(ring->get_seqno(ring), seqno) || 772 atomic_load_acq_int(&dev_priv->mm.wedged))) 773 ret = -msleep(ring, &ring->irq_lock, PCATCH, 774 "915thr", 0); 775 ring->irq_put(ring); 776 if (ret == 0 && atomic_load_acq_int(&dev_priv->mm.wedged)) 777 ret = -EIO; 778 } else if (_intel_wait_for(dev, 779 i915_seqno_passed(ring->get_seqno(ring), seqno) || 780 atomic_load_acq_int(&dev_priv->mm.wedged), 3000, 0, "915rtr")) { 781 ret = -EBUSY; 782 } 783 } 784 mtx_unlock(&ring->irq_lock); 785 786 if (ret == 0) 787 taskqueue_enqueue_timeout(dev_priv->tq, 788 &dev_priv->mm.retire_task, 0); 789 790 return (ret); 791} 792 793int 794i915_gem_throttle_ioctl(struct drm_device *dev, void *data, 795 struct drm_file *file_priv) 796{ 797 798 return (i915_gem_ring_throttle(dev, file_priv)); 799} 800 801int 802i915_gem_madvise_ioctl(struct drm_device *dev, void *data, 803 struct drm_file *file_priv) 804{ 805 struct drm_i915_gem_madvise *args; 806 struct drm_i915_gem_object *obj; 807 int ret; 808 809 args = data; 810 switch (args->madv) { 811 case I915_MADV_DONTNEED: 812 case I915_MADV_WILLNEED: 813 break; 814 default: 815 return (-EINVAL); 816 } 817 818 ret = i915_mutex_lock_interruptible(dev); 819 if (ret != 0) 820 return (ret); 821 822 obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle)); 823 if (&obj->base == NULL) { 824 ret = -ENOENT; 825 goto unlock; 826 } 827 828 if (obj->pin_count != 0) { 829 ret = -EINVAL; 830 goto out; 831 } 832 833 if (obj->madv != I915_MADV_PURGED_INTERNAL) 834 obj->madv = args->madv; 835 if (i915_gem_object_is_purgeable(obj) && obj->gtt_space == NULL) 836 i915_gem_object_truncate(obj); 837 args->retained = obj->madv != I915_MADV_PURGED_INTERNAL; 838 839out: 840 drm_gem_object_unreference(&obj->base); 841unlock: 842 DRM_UNLOCK(dev); 843 return (ret); 844} 845 846void 847i915_gem_cleanup_ringbuffer(struct drm_device *dev) 848{ 849 drm_i915_private_t *dev_priv; 850 int i; 851 852 dev_priv = dev->dev_private; 853 for (i = 0; i < I915_NUM_RINGS; i++) 854 intel_cleanup_ring_buffer(&dev_priv->rings[i]); 855} 856 857int 858i915_gem_entervt_ioctl(struct drm_device *dev, void *data, 859 struct drm_file *file_priv) 860{ 861 drm_i915_private_t *dev_priv; 862 int ret, i; 863 864 if (drm_core_check_feature(dev, DRIVER_MODESET)) 865 return (0); 866 dev_priv = dev->dev_private; 867 if (atomic_load_acq_int(&dev_priv->mm.wedged) != 0) { 868 DRM_ERROR("Reenabling wedged hardware, good luck\n"); 869 atomic_store_rel_int(&dev_priv->mm.wedged, 0); 870 } 871 872 dev_priv->mm.suspended = 0; 873 874 ret = i915_gem_init_hw(dev); 875 if (ret != 0) { 876 return (ret); 877 } 878 879 KASSERT(list_empty(&dev_priv->mm.active_list), ("active list")); 880 KASSERT(list_empty(&dev_priv->mm.flushing_list), ("flushing list")); 881 KASSERT(list_empty(&dev_priv->mm.inactive_list), ("inactive list")); 882 for (i = 0; i < I915_NUM_RINGS; i++) { 883 KASSERT(list_empty(&dev_priv->rings[i].active_list), 884 ("ring %d active list", i)); 885 KASSERT(list_empty(&dev_priv->rings[i].request_list), 886 ("ring %d request list", i)); 887 } 888 889 DRM_UNLOCK(dev); 890 ret = drm_irq_install(dev); 891 DRM_LOCK(dev); 892 if (ret) 893 goto cleanup_ringbuffer; 894 895 return (0); 896 897cleanup_ringbuffer: 898 i915_gem_cleanup_ringbuffer(dev); 899 dev_priv->mm.suspended = 1; 900 901 return (ret); 902} 903 904int 905i915_gem_leavevt_ioctl(struct drm_device *dev, void *data, 906 struct drm_file *file_priv) 907{ 908 909 if (drm_core_check_feature(dev, DRIVER_MODESET)) 910 return 0; 911 912 drm_irq_uninstall(dev); 913 return (i915_gem_idle(dev)); 914} 915 916int 917i915_gem_create(struct drm_file *file, struct drm_device *dev, uint64_t size, 918 uint32_t *handle_p) 919{ 920 struct drm_i915_gem_object *obj; 921 uint32_t handle; 922 int ret; 923 924 size = roundup(size, PAGE_SIZE); 925 if (size == 0) 926 return (-EINVAL); 927 928 obj = i915_gem_alloc_object(dev, size); 929 if (obj == NULL) 930 return (-ENOMEM); 931 932 handle = 0; 933 ret = drm_gem_handle_create(file, &obj->base, &handle); 934 if (ret != 0) { 935 drm_gem_object_release(&obj->base); 936 i915_gem_info_remove_obj(dev->dev_private, obj->base.size); 937 free(obj, DRM_I915_GEM); 938 return (-ret); 939 } 940 941 /* drop reference from allocate - handle holds it now */ 942 drm_gem_object_unreference(&obj->base); 943 CTR2(KTR_DRM, "object_create %p %x", obj, size); 944 *handle_p = handle; 945 return (0); 946} 947 948int 949i915_gem_dumb_create(struct drm_file *file, struct drm_device *dev, 950 struct drm_mode_create_dumb *args) 951{ 952 953 /* have to work out size/pitch and return them */ 954 args->pitch = roundup2(args->width * ((args->bpp + 7) / 8), 64); 955 args->size = args->pitch * args->height; 956 return (i915_gem_create(file, dev, args->size, &args->handle)); 957} 958 959int 960i915_gem_dumb_destroy(struct drm_file *file, struct drm_device *dev, 961 uint32_t handle) 962{ 963 964 return (drm_gem_handle_delete(file, handle)); 965} 966 967int 968i915_gem_create_ioctl(struct drm_device *dev, void *data, 969 struct drm_file *file) 970{ 971 struct drm_i915_gem_create *args = data; 972 973 return (i915_gem_create(file, dev, args->size, &args->handle)); 974} 975 976static int 977i915_gem_swap_io(struct drm_device *dev, struct drm_i915_gem_object *obj, 978 uint64_t data_ptr, uint64_t size, uint64_t offset, enum uio_rw rw, 979 struct drm_file *file) 980{ 981 vm_object_t vm_obj; 982 vm_page_t m; 983 struct sf_buf *sf; 984 vm_offset_t mkva; 985 vm_pindex_t obj_pi; 986 int cnt, do_bit17_swizzling, length, obj_po, ret, swizzled_po; 987 988 if (obj->gtt_offset != 0 && rw == UIO_READ) 989 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 990 else 991 do_bit17_swizzling = 0; 992 993 obj->dirty = 1; 994 vm_obj = obj->base.vm_obj; 995 ret = 0; 996 997 VM_OBJECT_WLOCK(vm_obj); 998 vm_object_pip_add(vm_obj, 1); 999 while (size > 0) { 1000 obj_pi = OFF_TO_IDX(offset); 1001 obj_po = offset & PAGE_MASK; 1002 1003 m = i915_gem_wire_page(vm_obj, obj_pi); 1004 VM_OBJECT_WUNLOCK(vm_obj); 1005 1006 sched_pin(); 1007 sf = sf_buf_alloc(m, SFB_CPUPRIVATE); 1008 mkva = sf_buf_kva(sf); 1009 length = min(size, PAGE_SIZE - obj_po); 1010 while (length > 0) { 1011 if (do_bit17_swizzling && 1012 (VM_PAGE_TO_PHYS(m) & (1 << 17)) != 0) { 1013 cnt = roundup2(obj_po + 1, 64); 1014 cnt = min(cnt - obj_po, length); 1015 swizzled_po = obj_po ^ 64; 1016 } else { 1017 cnt = length; 1018 swizzled_po = obj_po; 1019 } 1020 if (rw == UIO_READ) 1021 ret = -copyout_nofault( 1022 (char *)mkva + swizzled_po, 1023 (void *)(uintptr_t)data_ptr, cnt); 1024 else 1025 ret = -copyin_nofault( 1026 (void *)(uintptr_t)data_ptr, 1027 (char *)mkva + swizzled_po, cnt); 1028 if (ret != 0) 1029 break; 1030 data_ptr += cnt; 1031 size -= cnt; 1032 length -= cnt; 1033 offset += cnt; 1034 obj_po += cnt; 1035 } 1036 sf_buf_free(sf); 1037 sched_unpin(); 1038 VM_OBJECT_WLOCK(vm_obj); 1039 if (rw == UIO_WRITE) 1040 vm_page_dirty(m); 1041 vm_page_reference(m); 1042 vm_page_lock(m); 1043 vm_page_unwire(m, 1); 1044 vm_page_unlock(m); 1045 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 1046 1047 if (ret != 0) 1048 break; 1049 } 1050 vm_object_pip_wakeup(vm_obj); 1051 VM_OBJECT_WUNLOCK(vm_obj); 1052 1053 return (ret); 1054} 1055 1056static int 1057i915_gem_gtt_write(struct drm_device *dev, struct drm_i915_gem_object *obj, 1058 uint64_t data_ptr, uint64_t size, uint64_t offset, struct drm_file *file) 1059{ 1060 vm_offset_t mkva; 1061 vm_pindex_t obj_pi; 1062 int obj_po, ret; 1063 1064 obj_pi = OFF_TO_IDX(offset); 1065 obj_po = offset & PAGE_MASK; 1066 1067 mkva = (vm_offset_t)pmap_mapdev_attr(dev->agp->base + obj->gtt_offset + 1068 IDX_TO_OFF(obj_pi), size, PAT_WRITE_COMBINING); 1069 ret = -copyin_nofault((void *)(uintptr_t)data_ptr, (char *)mkva + 1070 obj_po, size); 1071 pmap_unmapdev(mkva, size); 1072 return (ret); 1073} 1074 1075static int 1076i915_gem_obj_io(struct drm_device *dev, uint32_t handle, uint64_t data_ptr, 1077 uint64_t size, uint64_t offset, enum uio_rw rw, struct drm_file *file) 1078{ 1079 struct drm_i915_gem_object *obj; 1080 vm_page_t *ma; 1081 vm_offset_t start, end; 1082 int npages, ret; 1083 1084 if (size == 0) 1085 return (0); 1086 start = trunc_page(data_ptr); 1087 end = round_page(data_ptr + size); 1088 npages = howmany(end - start, PAGE_SIZE); 1089 ma = malloc(npages * sizeof(vm_page_t), DRM_I915_GEM, M_WAITOK | 1090 M_ZERO); 1091 npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, 1092 (vm_offset_t)data_ptr, size, 1093 (rw == UIO_READ ? VM_PROT_WRITE : 0 ) | VM_PROT_READ, ma, npages); 1094 if (npages == -1) { 1095 ret = -EFAULT; 1096 goto free_ma; 1097 } 1098 1099 ret = i915_mutex_lock_interruptible(dev); 1100 if (ret != 0) 1101 goto unlocked; 1102 1103 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); 1104 if (&obj->base == NULL) { 1105 ret = -ENOENT; 1106 goto unlock; 1107 } 1108 if (offset > obj->base.size || size > obj->base.size - offset) { 1109 ret = -EINVAL; 1110 goto out; 1111 } 1112 1113 if (rw == UIO_READ) { 1114 CTR3(KTR_DRM, "object_pread %p %jx %jx", obj, offset, size); 1115 ret = i915_gem_object_set_cpu_read_domain_range(obj, 1116 offset, size); 1117 if (ret != 0) 1118 goto out; 1119 ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset, 1120 UIO_READ, file); 1121 } else { 1122 if (obj->phys_obj) { 1123 CTR3(KTR_DRM, "object_phys_write %p %jx %jx", obj, 1124 offset, size); 1125 ret = i915_gem_phys_pwrite(dev, obj, data_ptr, offset, 1126 size, file); 1127 } else if (obj->gtt_space && 1128 obj->base.write_domain != I915_GEM_DOMAIN_CPU) { 1129 CTR3(KTR_DRM, "object_gtt_write %p %jx %jx", obj, 1130 offset, size); 1131 ret = i915_gem_object_pin(obj, 0, true); 1132 if (ret != 0) 1133 goto out; 1134 ret = i915_gem_object_set_to_gtt_domain(obj, true); 1135 if (ret != 0) 1136 goto out_unpin; 1137 ret = i915_gem_object_put_fence(obj); 1138 if (ret != 0) 1139 goto out_unpin; 1140 ret = i915_gem_gtt_write(dev, obj, data_ptr, size, 1141 offset, file); 1142out_unpin: 1143 i915_gem_object_unpin(obj); 1144 } else { 1145 CTR3(KTR_DRM, "object_pwrite %p %jx %jx", obj, 1146 offset, size); 1147 ret = i915_gem_object_set_to_cpu_domain(obj, true); 1148 if (ret != 0) 1149 goto out; 1150 ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset, 1151 UIO_WRITE, file); 1152 } 1153 } 1154out: 1155 drm_gem_object_unreference(&obj->base); 1156unlock: 1157 DRM_UNLOCK(dev); 1158unlocked: 1159 vm_page_unhold_pages(ma, npages); 1160free_ma: 1161 free(ma, DRM_I915_GEM); 1162 return (ret); 1163} 1164 1165int 1166i915_gem_pread_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 1167{ 1168 struct drm_i915_gem_pread *args; 1169 1170 args = data; 1171 return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size, 1172 args->offset, UIO_READ, file)); 1173} 1174 1175int 1176i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 1177{ 1178 struct drm_i915_gem_pwrite *args; 1179 1180 args = data; 1181 return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size, 1182 args->offset, UIO_WRITE, file)); 1183} 1184 1185int 1186i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, 1187 struct drm_file *file) 1188{ 1189 struct drm_i915_gem_set_domain *args; 1190 struct drm_i915_gem_object *obj; 1191 uint32_t read_domains; 1192 uint32_t write_domain; 1193 int ret; 1194 1195 if ((dev->driver->driver_features & DRIVER_GEM) == 0) 1196 return (-ENODEV); 1197 1198 args = data; 1199 read_domains = args->read_domains; 1200 write_domain = args->write_domain; 1201 1202 if ((write_domain & I915_GEM_GPU_DOMAINS) != 0 || 1203 (read_domains & I915_GEM_GPU_DOMAINS) != 0 || 1204 (write_domain != 0 && read_domains != write_domain)) 1205 return (-EINVAL); 1206 1207 ret = i915_mutex_lock_interruptible(dev); 1208 if (ret != 0) 1209 return (ret); 1210 1211 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1212 if (&obj->base == NULL) { 1213 ret = -ENOENT; 1214 goto unlock; 1215 } 1216 1217 if ((read_domains & I915_GEM_DOMAIN_GTT) != 0) { 1218 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); 1219 if (ret == -EINVAL) 1220 ret = 0; 1221 } else 1222 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); 1223 1224 drm_gem_object_unreference(&obj->base); 1225unlock: 1226 DRM_UNLOCK(dev); 1227 return (ret); 1228} 1229 1230int 1231i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, 1232 struct drm_file *file) 1233{ 1234 struct drm_i915_gem_sw_finish *args; 1235 struct drm_i915_gem_object *obj; 1236 int ret; 1237 1238 args = data; 1239 ret = 0; 1240 if ((dev->driver->driver_features & DRIVER_GEM) == 0) 1241 return (ENODEV); 1242 ret = i915_mutex_lock_interruptible(dev); 1243 if (ret != 0) 1244 return (ret); 1245 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1246 if (&obj->base == NULL) { 1247 ret = -ENOENT; 1248 goto unlock; 1249 } 1250 if (obj->pin_count != 0) 1251 i915_gem_object_flush_cpu_write_domain(obj); 1252 drm_gem_object_unreference(&obj->base); 1253unlock: 1254 DRM_UNLOCK(dev); 1255 return (ret); 1256} 1257 1258int 1259i915_gem_mmap_ioctl(struct drm_device *dev, void *data, 1260 struct drm_file *file) 1261{ 1262 struct drm_i915_gem_mmap *args; 1263 struct drm_gem_object *obj; 1264 struct proc *p; 1265 vm_map_t map; 1266 vm_offset_t addr; 1267 vm_size_t size; 1268 int error, rv; 1269 1270 args = data; 1271 1272 if ((dev->driver->driver_features & DRIVER_GEM) == 0) 1273 return (-ENODEV); 1274 1275 obj = drm_gem_object_lookup(dev, file, args->handle); 1276 if (obj == NULL) 1277 return (-ENOENT); 1278 error = 0; 1279 if (args->size == 0) 1280 goto out; 1281 p = curproc; 1282 map = &p->p_vmspace->vm_map; 1283 size = round_page(args->size); 1284 PROC_LOCK(p); 1285 if (map->size + size > lim_cur(p, RLIMIT_VMEM)) { 1286 PROC_UNLOCK(p); 1287 error = ENOMEM; 1288 goto out; 1289 } 1290 PROC_UNLOCK(p); 1291 1292 addr = 0; 1293 vm_object_reference(obj->vm_obj); 1294 DRM_UNLOCK(dev); 1295 rv = vm_map_find(map, obj->vm_obj, args->offset, &addr, args->size, 0, 1296 VMFS_OPTIMAL_SPACE, VM_PROT_READ | VM_PROT_WRITE, 1297 VM_PROT_READ | VM_PROT_WRITE, MAP_INHERIT_SHARE); 1298 if (rv != KERN_SUCCESS) { 1299 vm_object_deallocate(obj->vm_obj); 1300 error = -vm_mmap_to_errno(rv); 1301 } else { 1302 args->addr_ptr = (uint64_t)addr; 1303 } 1304 DRM_LOCK(dev); 1305out: 1306 drm_gem_object_unreference(obj); 1307 return (error); 1308} 1309 1310static int 1311i915_gem_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 1312 vm_ooffset_t foff, struct ucred *cred, u_short *color) 1313{ 1314 1315 *color = 0; /* XXXKIB */ 1316 return (0); 1317} 1318 1319int i915_intr_pf; 1320 1321static int 1322i915_gem_pager_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot, 1323 vm_page_t *mres) 1324{ 1325 struct drm_gem_object *gem_obj; 1326 struct drm_i915_gem_object *obj; 1327 struct drm_device *dev; 1328 drm_i915_private_t *dev_priv; 1329 vm_page_t m, oldm; 1330 int cause, ret; 1331 bool write; 1332 1333 gem_obj = vm_obj->handle; 1334 obj = to_intel_bo(gem_obj); 1335 dev = obj->base.dev; 1336 dev_priv = dev->dev_private; 1337#if 0 1338 write = (prot & VM_PROT_WRITE) != 0; 1339#else 1340 write = true; 1341#endif 1342 vm_object_pip_add(vm_obj, 1); 1343 1344 /* 1345 * Remove the placeholder page inserted by vm_fault() from the 1346 * object before dropping the object lock. If 1347 * i915_gem_release_mmap() is active in parallel on this gem 1348 * object, then it owns the drm device sx and might find the 1349 * placeholder already. Then, since the page is busy, 1350 * i915_gem_release_mmap() sleeps waiting for the busy state 1351 * of the page cleared. We will be not able to acquire drm 1352 * device lock until i915_gem_release_mmap() is able to make a 1353 * progress. 1354 */ 1355 if (*mres != NULL) { 1356 oldm = *mres; 1357 vm_page_lock(oldm); 1358 vm_page_remove(oldm); 1359 vm_page_unlock(oldm); 1360 *mres = NULL; 1361 } else 1362 oldm = NULL; 1363 VM_OBJECT_WUNLOCK(vm_obj); 1364retry: 1365 cause = ret = 0; 1366 m = NULL; 1367 1368 if (i915_intr_pf) { 1369 ret = i915_mutex_lock_interruptible(dev); 1370 if (ret != 0) { 1371 cause = 10; 1372 goto out; 1373 } 1374 } else 1375 DRM_LOCK(dev); 1376 1377 /* 1378 * Since the object lock was dropped, other thread might have 1379 * faulted on the same GTT address and instantiated the 1380 * mapping for the page. Recheck. 1381 */ 1382 VM_OBJECT_WLOCK(vm_obj); 1383 m = vm_page_lookup(vm_obj, OFF_TO_IDX(offset)); 1384 if (m != NULL) { 1385 if (vm_page_busied(m)) { 1386 DRM_UNLOCK(dev); 1387 vm_page_lock(m); 1388 VM_OBJECT_WUNLOCK(vm_obj); 1389 vm_page_busy_sleep(m, "915pee"); 1390 goto retry; 1391 } 1392 goto have_page; 1393 } else 1394 VM_OBJECT_WUNLOCK(vm_obj); 1395 1396 /* Now bind it into the GTT if needed */ 1397 if (!obj->map_and_fenceable) { 1398 ret = i915_gem_object_unbind(obj); 1399 if (ret != 0) { 1400 cause = 20; 1401 goto unlock; 1402 } 1403 } 1404 if (!obj->gtt_space) { 1405 ret = i915_gem_object_bind_to_gtt(obj, 0, true); 1406 if (ret != 0) { 1407 cause = 30; 1408 goto unlock; 1409 } 1410 1411 ret = i915_gem_object_set_to_gtt_domain(obj, write); 1412 if (ret != 0) { 1413 cause = 40; 1414 goto unlock; 1415 } 1416 } 1417 1418 if (obj->tiling_mode == I915_TILING_NONE) 1419 ret = i915_gem_object_put_fence(obj); 1420 else 1421 ret = i915_gem_object_get_fence(obj, NULL); 1422 if (ret != 0) { 1423 cause = 50; 1424 goto unlock; 1425 } 1426 1427 if (i915_gem_object_is_inactive(obj)) 1428 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 1429 1430 obj->fault_mappable = true; 1431 VM_OBJECT_WLOCK(vm_obj); 1432 m = vm_phys_fictitious_to_vm_page(dev->agp->base + obj->gtt_offset + 1433 offset); 1434 if (m == NULL) { 1435 VM_OBJECT_WUNLOCK(vm_obj); 1436 cause = 60; 1437 ret = -EFAULT; 1438 goto unlock; 1439 } 1440 KASSERT((m->flags & PG_FICTITIOUS) != 0, 1441 ("not fictitious %p", m)); 1442 KASSERT(m->wire_count == 1, ("wire_count not 1 %p", m)); 1443 1444 if (vm_page_busied(m)) { 1445 DRM_UNLOCK(dev); 1446 vm_page_lock(m); 1447 VM_OBJECT_WUNLOCK(vm_obj); 1448 vm_page_busy_sleep(m, "915pbs"); 1449 goto retry; 1450 } 1451 if (vm_page_insert(m, vm_obj, OFF_TO_IDX(offset))) { 1452 DRM_UNLOCK(dev); 1453 VM_OBJECT_WUNLOCK(vm_obj); 1454 VM_WAIT; 1455 goto retry; 1456 } 1457 m->valid = VM_PAGE_BITS_ALL; 1458have_page: 1459 *mres = m; 1460 vm_page_xbusy(m); 1461 1462 CTR4(KTR_DRM, "fault %p %jx %x phys %x", gem_obj, offset, prot, 1463 m->phys_addr); 1464 DRM_UNLOCK(dev); 1465 if (oldm != NULL) { 1466 vm_page_lock(oldm); 1467 vm_page_free(oldm); 1468 vm_page_unlock(oldm); 1469 } 1470 vm_object_pip_wakeup(vm_obj); 1471 return (VM_PAGER_OK); 1472 1473unlock: 1474 DRM_UNLOCK(dev); 1475out: 1476 KASSERT(ret != 0, ("i915_gem_pager_fault: wrong return")); 1477 CTR5(KTR_DRM, "fault_fail %p %jx %x err %d %d", gem_obj, offset, prot, 1478 -ret, cause); 1479 if (ret == -EAGAIN || ret == -EIO || ret == -EINTR) { 1480 kern_yield(PRI_USER); 1481 goto retry; 1482 } 1483 VM_OBJECT_WLOCK(vm_obj); 1484 vm_object_pip_wakeup(vm_obj); 1485 return (VM_PAGER_ERROR); 1486} 1487 1488static void 1489i915_gem_pager_dtor(void *handle) 1490{ 1491 struct drm_gem_object *obj; 1492 struct drm_device *dev; 1493 1494 obj = handle; 1495 dev = obj->dev; 1496 1497 DRM_LOCK(dev); 1498 drm_gem_free_mmap_offset(obj); 1499 i915_gem_release_mmap(to_intel_bo(obj)); 1500 drm_gem_object_unreference(obj); 1501 DRM_UNLOCK(dev); 1502} 1503 1504struct cdev_pager_ops i915_gem_pager_ops = { 1505 .cdev_pg_fault = i915_gem_pager_fault, 1506 .cdev_pg_ctor = i915_gem_pager_ctor, 1507 .cdev_pg_dtor = i915_gem_pager_dtor 1508}; 1509 1510int 1511i915_gem_mmap_gtt(struct drm_file *file, struct drm_device *dev, 1512 uint32_t handle, uint64_t *offset) 1513{ 1514 struct drm_i915_private *dev_priv; 1515 struct drm_i915_gem_object *obj; 1516 int ret; 1517 1518 if (!(dev->driver->driver_features & DRIVER_GEM)) 1519 return (-ENODEV); 1520 1521 dev_priv = dev->dev_private; 1522 1523 ret = i915_mutex_lock_interruptible(dev); 1524 if (ret != 0) 1525 return (ret); 1526 1527 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); 1528 if (&obj->base == NULL) { 1529 ret = -ENOENT; 1530 goto unlock; 1531 } 1532 1533 if (obj->base.size > dev_priv->mm.gtt_mappable_end) { 1534 ret = -E2BIG; 1535 goto out; 1536 } 1537 1538 if (obj->madv != I915_MADV_WILLNEED) { 1539 DRM_ERROR("Attempting to mmap a purgeable buffer\n"); 1540 ret = -EINVAL; 1541 goto out; 1542 } 1543 1544 ret = drm_gem_create_mmap_offset(&obj->base); 1545 if (ret != 0) 1546 goto out; 1547 1548 *offset = DRM_GEM_MAPPING_OFF(obj->base.map_list.key) | 1549 DRM_GEM_MAPPING_KEY; 1550out: 1551 drm_gem_object_unreference(&obj->base); 1552unlock: 1553 DRM_UNLOCK(dev); 1554 return (ret); 1555} 1556 1557int 1558i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, 1559 struct drm_file *file) 1560{ 1561 struct drm_i915_private *dev_priv; 1562 struct drm_i915_gem_mmap_gtt *args; 1563 1564 dev_priv = dev->dev_private; 1565 args = data; 1566 1567 return (i915_gem_mmap_gtt(file, dev, args->handle, &args->offset)); 1568} 1569 1570struct drm_i915_gem_object * 1571i915_gem_alloc_object(struct drm_device *dev, size_t size) 1572{ 1573 struct drm_i915_private *dev_priv; 1574 struct drm_i915_gem_object *obj; 1575 1576 dev_priv = dev->dev_private; 1577 1578 obj = malloc(sizeof(*obj), DRM_I915_GEM, M_WAITOK | M_ZERO); 1579 1580 if (drm_gem_object_init(dev, &obj->base, size) != 0) { 1581 free(obj, DRM_I915_GEM); 1582 return (NULL); 1583 } 1584 1585 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 1586 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 1587 1588 if (HAS_LLC(dev)) 1589 obj->cache_level = I915_CACHE_LLC; 1590 else 1591 obj->cache_level = I915_CACHE_NONE; 1592 obj->base.driver_private = NULL; 1593 obj->fence_reg = I915_FENCE_REG_NONE; 1594 INIT_LIST_HEAD(&obj->mm_list); 1595 INIT_LIST_HEAD(&obj->gtt_list); 1596 INIT_LIST_HEAD(&obj->ring_list); 1597 INIT_LIST_HEAD(&obj->exec_list); 1598 INIT_LIST_HEAD(&obj->gpu_write_list); 1599 obj->madv = I915_MADV_WILLNEED; 1600 /* Avoid an unnecessary call to unbind on the first bind. */ 1601 obj->map_and_fenceable = true; 1602 1603 i915_gem_info_add_obj(dev_priv, size); 1604 1605 return (obj); 1606} 1607 1608void 1609i915_gem_clflush_object(struct drm_i915_gem_object *obj) 1610{ 1611 1612 /* If we don't have a page list set up, then we're not pinned 1613 * to GPU, and we can ignore the cache flush because it'll happen 1614 * again at bind time. 1615 */ 1616 if (obj->pages == NULL) 1617 return; 1618 1619 /* If the GPU is snooping the contents of the CPU cache, 1620 * we do not need to manually clear the CPU cache lines. However, 1621 * the caches are only snooped when the render cache is 1622 * flushed/invalidated. As we always have to emit invalidations 1623 * and flushes when moving into and out of the RENDER domain, correct 1624 * snooping behaviour occurs naturally as the result of our domain 1625 * tracking. 1626 */ 1627 if (obj->cache_level != I915_CACHE_NONE) 1628 return; 1629 1630 CTR1(KTR_DRM, "object_clflush %p", obj); 1631 drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE); 1632} 1633 1634static void 1635i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) 1636{ 1637 uint32_t old_write_domain; 1638 1639 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) 1640 return; 1641 1642 i915_gem_clflush_object(obj); 1643 intel_gtt_chipset_flush(); 1644 old_write_domain = obj->base.write_domain; 1645 obj->base.write_domain = 0; 1646 1647 CTR3(KTR_DRM, "object_change_domain flush_cpu_write %p %x %x", obj, 1648 obj->base.read_domains, old_write_domain); 1649} 1650 1651static int 1652i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj) 1653{ 1654 1655 if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0) 1656 return (0); 1657 return (i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain)); 1658} 1659 1660static void 1661i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj) 1662{ 1663 uint32_t old_write_domain; 1664 1665 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT) 1666 return; 1667 1668 wmb(); 1669 1670 old_write_domain = obj->base.write_domain; 1671 obj->base.write_domain = 0; 1672 1673 CTR3(KTR_DRM, "object_change_domain flush gtt_write %p %x %x", obj, 1674 obj->base.read_domains, old_write_domain); 1675} 1676 1677int 1678i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) 1679{ 1680 uint32_t old_write_domain, old_read_domains; 1681 int ret; 1682 1683 if (obj->gtt_space == NULL) 1684 return (-EINVAL); 1685 1686 if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) 1687 return 0; 1688 1689 ret = i915_gem_object_flush_gpu_write_domain(obj); 1690 if (ret != 0) 1691 return (ret); 1692 1693 if (obj->pending_gpu_write || write) { 1694 ret = i915_gem_object_wait_rendering(obj); 1695 if (ret != 0) 1696 return (ret); 1697 } 1698 1699 i915_gem_object_flush_cpu_write_domain(obj); 1700 1701 old_write_domain = obj->base.write_domain; 1702 old_read_domains = obj->base.read_domains; 1703 1704 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0, 1705 ("In GTT write domain")); 1706 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 1707 if (write) { 1708 obj->base.read_domains = I915_GEM_DOMAIN_GTT; 1709 obj->base.write_domain = I915_GEM_DOMAIN_GTT; 1710 obj->dirty = 1; 1711 } 1712 1713 CTR3(KTR_DRM, "object_change_domain set_to_gtt %p %x %x", obj, 1714 old_read_domains, old_write_domain); 1715 return (0); 1716} 1717 1718int 1719i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, 1720 enum i915_cache_level cache_level) 1721{ 1722 struct drm_device *dev; 1723 drm_i915_private_t *dev_priv; 1724 int ret; 1725 1726 if (obj->cache_level == cache_level) 1727 return 0; 1728 1729 if (obj->pin_count) { 1730 DRM_DEBUG("can not change the cache level of pinned objects\n"); 1731 return (-EBUSY); 1732 } 1733 1734 dev = obj->base.dev; 1735 dev_priv = dev->dev_private; 1736 if (obj->gtt_space) { 1737 ret = i915_gem_object_finish_gpu(obj); 1738 if (ret != 0) 1739 return (ret); 1740 1741 i915_gem_object_finish_gtt(obj); 1742 1743 /* Before SandyBridge, you could not use tiling or fence 1744 * registers with snooped memory, so relinquish any fences 1745 * currently pointing to our region in the aperture. 1746 */ 1747 if (INTEL_INFO(obj->base.dev)->gen < 6) { 1748 ret = i915_gem_object_put_fence(obj); 1749 if (ret != 0) 1750 return (ret); 1751 } 1752 1753 i915_gem_gtt_rebind_object(obj, cache_level); 1754 if (obj->has_aliasing_ppgtt_mapping) 1755 i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt, 1756 obj, cache_level); 1757 } 1758 1759 if (cache_level == I915_CACHE_NONE) { 1760 u32 old_read_domains, old_write_domain; 1761 1762 /* If we're coming from LLC cached, then we haven't 1763 * actually been tracking whether the data is in the 1764 * CPU cache or not, since we only allow one bit set 1765 * in obj->write_domain and have been skipping the clflushes. 1766 * Just set it to the CPU cache for now. 1767 */ 1768 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 1769 ("obj %p in CPU write domain", obj)); 1770 KASSERT((obj->base.read_domains & ~I915_GEM_DOMAIN_CPU) == 0, 1771 ("obj %p in CPU read domain", obj)); 1772 1773 old_read_domains = obj->base.read_domains; 1774 old_write_domain = obj->base.write_domain; 1775 1776 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 1777 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 1778 1779 CTR3(KTR_DRM, "object_change_domain set_cache_level %p %x %x", 1780 obj, old_read_domains, old_write_domain); 1781 } 1782 1783 obj->cache_level = cache_level; 1784 return (0); 1785} 1786 1787int 1788i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, 1789 u32 alignment, struct intel_ring_buffer *pipelined) 1790{ 1791 u32 old_read_domains, old_write_domain; 1792 int ret; 1793 1794 ret = i915_gem_object_flush_gpu_write_domain(obj); 1795 if (ret != 0) 1796 return (ret); 1797 1798 if (pipelined != obj->ring) { 1799 ret = i915_gem_object_wait_rendering(obj); 1800 if (ret == -ERESTART || ret == -EINTR) 1801 return (ret); 1802 } 1803 1804 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE); 1805 if (ret != 0) 1806 return (ret); 1807 1808 ret = i915_gem_object_pin(obj, alignment, true); 1809 if (ret != 0) 1810 return (ret); 1811 1812 i915_gem_object_flush_cpu_write_domain(obj); 1813 1814 old_write_domain = obj->base.write_domain; 1815 old_read_domains = obj->base.read_domains; 1816 1817 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0, 1818 ("obj %p in GTT write domain", obj)); 1819 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 1820 1821 CTR3(KTR_DRM, "object_change_domain pin_to_display_plan %p %x %x", 1822 obj, old_read_domains, obj->base.write_domain); 1823 return (0); 1824} 1825 1826int 1827i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj) 1828{ 1829 int ret; 1830 1831 if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0) 1832 return (0); 1833 1834 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 1835 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain); 1836 if (ret != 0) 1837 return (ret); 1838 } 1839 1840 ret = i915_gem_object_wait_rendering(obj); 1841 if (ret != 0) 1842 return (ret); 1843 1844 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 1845 1846 return (0); 1847} 1848 1849static int 1850i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) 1851{ 1852 uint32_t old_write_domain, old_read_domains; 1853 int ret; 1854 1855 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) 1856 return 0; 1857 1858 ret = i915_gem_object_flush_gpu_write_domain(obj); 1859 if (ret != 0) 1860 return (ret); 1861 1862 ret = i915_gem_object_wait_rendering(obj); 1863 if (ret != 0) 1864 return (ret); 1865 1866 i915_gem_object_flush_gtt_write_domain(obj); 1867 i915_gem_object_set_to_full_cpu_read_domain(obj); 1868 1869 old_write_domain = obj->base.write_domain; 1870 old_read_domains = obj->base.read_domains; 1871 1872 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) { 1873 i915_gem_clflush_object(obj); 1874 obj->base.read_domains |= I915_GEM_DOMAIN_CPU; 1875 } 1876 1877 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 1878 ("In cpu write domain")); 1879 1880 if (write) { 1881 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 1882 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 1883 } 1884 1885 CTR3(KTR_DRM, "object_change_domain set_to_cpu %p %x %x", obj, 1886 old_read_domains, old_write_domain); 1887 return (0); 1888} 1889 1890static void 1891i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj) 1892{ 1893 int i; 1894 1895 if (obj->page_cpu_valid == NULL) 1896 return; 1897 1898 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0) { 1899 for (i = 0; i <= (obj->base.size - 1) / PAGE_SIZE; i++) { 1900 if (obj->page_cpu_valid[i] != 0) 1901 continue; 1902 drm_clflush_pages(obj->pages + i, 1); 1903 } 1904 } 1905 1906 free(obj->page_cpu_valid, DRM_I915_GEM); 1907 obj->page_cpu_valid = NULL; 1908} 1909 1910static int 1911i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj, 1912 uint64_t offset, uint64_t size) 1913{ 1914 uint32_t old_read_domains; 1915 int i, ret; 1916 1917 if (offset == 0 && size == obj->base.size) 1918 return (i915_gem_object_set_to_cpu_domain(obj, 0)); 1919 1920 ret = i915_gem_object_flush_gpu_write_domain(obj); 1921 if (ret != 0) 1922 return (ret); 1923 ret = i915_gem_object_wait_rendering(obj); 1924 if (ret != 0) 1925 return (ret); 1926 1927 i915_gem_object_flush_gtt_write_domain(obj); 1928 1929 if (obj->page_cpu_valid == NULL && 1930 (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0) 1931 return (0); 1932 1933 if (obj->page_cpu_valid == NULL) { 1934 obj->page_cpu_valid = malloc(obj->base.size / PAGE_SIZE, 1935 DRM_I915_GEM, M_WAITOK | M_ZERO); 1936 } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) 1937 memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE); 1938 1939 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE; 1940 i++) { 1941 if (obj->page_cpu_valid[i]) 1942 continue; 1943 drm_clflush_pages(obj->pages + i, 1); 1944 obj->page_cpu_valid[i] = 1; 1945 } 1946 1947 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 1948 ("In gpu write domain")); 1949 1950 old_read_domains = obj->base.read_domains; 1951 obj->base.read_domains |= I915_GEM_DOMAIN_CPU; 1952 1953 CTR3(KTR_DRM, "object_change_domain set_cpu_read %p %x %x", obj, 1954 old_read_domains, obj->base.write_domain); 1955 return (0); 1956} 1957 1958static uint32_t 1959i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode) 1960{ 1961 uint32_t gtt_size; 1962 1963 if (INTEL_INFO(dev)->gen >= 4 || 1964 tiling_mode == I915_TILING_NONE) 1965 return (size); 1966 1967 /* Previous chips need a power-of-two fence region when tiling */ 1968 if (INTEL_INFO(dev)->gen == 3) 1969 gtt_size = 1024*1024; 1970 else 1971 gtt_size = 512*1024; 1972 1973 while (gtt_size < size) 1974 gtt_size <<= 1; 1975 1976 return (gtt_size); 1977} 1978 1979/** 1980 * i915_gem_get_gtt_alignment - return required GTT alignment for an object 1981 * @obj: object to check 1982 * 1983 * Return the required GTT alignment for an object, taking into account 1984 * potential fence register mapping. 1985 */ 1986static uint32_t 1987i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size, 1988 int tiling_mode) 1989{ 1990 1991 /* 1992 * Minimum alignment is 4k (GTT page size), but might be greater 1993 * if a fence register is needed for the object. 1994 */ 1995 if (INTEL_INFO(dev)->gen >= 4 || 1996 tiling_mode == I915_TILING_NONE) 1997 return (4096); 1998 1999 /* 2000 * Previous chips need to be aligned to the size of the smallest 2001 * fence register that can contain the object. 2002 */ 2003 return (i915_gem_get_gtt_size(dev, size, tiling_mode)); 2004} 2005 2006uint32_t 2007i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev, uint32_t size, 2008 int tiling_mode) 2009{ 2010 2011 if (tiling_mode == I915_TILING_NONE) 2012 return (4096); 2013 2014 /* 2015 * Minimum alignment is 4k (GTT page size) for sane hw. 2016 */ 2017 if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev)) 2018 return (4096); 2019 2020 /* 2021 * Previous hardware however needs to be aligned to a power-of-two 2022 * tile height. The simplest method for determining this is to reuse 2023 * the power-of-tile object size. 2024 */ 2025 return (i915_gem_get_gtt_size(dev, size, tiling_mode)); 2026} 2027 2028static int 2029i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, 2030 unsigned alignment, bool map_and_fenceable) 2031{ 2032 struct drm_device *dev; 2033 struct drm_i915_private *dev_priv; 2034 struct drm_mm_node *free_space; 2035 uint32_t size, fence_size, fence_alignment, unfenced_alignment; 2036 bool mappable, fenceable; 2037 int ret; 2038 2039 dev = obj->base.dev; 2040 dev_priv = dev->dev_private; 2041 2042 if (obj->madv != I915_MADV_WILLNEED) { 2043 DRM_ERROR("Attempting to bind a purgeable object\n"); 2044 return (-EINVAL); 2045 } 2046 2047 fence_size = i915_gem_get_gtt_size(dev, obj->base.size, 2048 obj->tiling_mode); 2049 fence_alignment = i915_gem_get_gtt_alignment(dev, obj->base.size, 2050 obj->tiling_mode); 2051 unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(dev, 2052 obj->base.size, obj->tiling_mode); 2053 if (alignment == 0) 2054 alignment = map_and_fenceable ? fence_alignment : 2055 unfenced_alignment; 2056 if (map_and_fenceable && (alignment & (fence_alignment - 1)) != 0) { 2057 DRM_ERROR("Invalid object alignment requested %u\n", alignment); 2058 return (-EINVAL); 2059 } 2060 2061 size = map_and_fenceable ? fence_size : obj->base.size; 2062 2063 /* If the object is bigger than the entire aperture, reject it early 2064 * before evicting everything in a vain attempt to find space. 2065 */ 2066 if (obj->base.size > (map_and_fenceable ? 2067 dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) { 2068 DRM_ERROR( 2069"Attempting to bind an object larger than the aperture\n"); 2070 return (-E2BIG); 2071 } 2072 2073 search_free: 2074 if (map_and_fenceable) 2075 free_space = drm_mm_search_free_in_range( 2076 &dev_priv->mm.gtt_space, size, alignment, 0, 2077 dev_priv->mm.gtt_mappable_end, 0); 2078 else 2079 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space, 2080 size, alignment, 0); 2081 if (free_space != NULL) { 2082 if (map_and_fenceable) 2083 obj->gtt_space = drm_mm_get_block_range_generic( 2084 free_space, size, alignment, 0, 2085 dev_priv->mm.gtt_mappable_end, 1); 2086 else 2087 obj->gtt_space = drm_mm_get_block_generic(free_space, 2088 size, alignment, 1); 2089 } 2090 if (obj->gtt_space == NULL) { 2091 ret = i915_gem_evict_something(dev, size, alignment, 2092 map_and_fenceable); 2093 if (ret != 0) 2094 return (ret); 2095 goto search_free; 2096 } 2097 ret = i915_gem_object_get_pages_gtt(obj, 0); 2098 if (ret != 0) { 2099 drm_mm_put_block(obj->gtt_space); 2100 obj->gtt_space = NULL; 2101 /* 2102 * i915_gem_object_get_pages_gtt() cannot return 2103 * ENOMEM, since we use vm_page_grab(). 2104 */ 2105 return (ret); 2106 } 2107 2108 ret = i915_gem_gtt_bind_object(obj); 2109 if (ret != 0) { 2110 i915_gem_object_put_pages_gtt(obj); 2111 drm_mm_put_block(obj->gtt_space); 2112 obj->gtt_space = NULL; 2113 if (i915_gem_evict_everything(dev, false)) 2114 return (ret); 2115 goto search_free; 2116 } 2117 2118 list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list); 2119 list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 2120 2121 KASSERT((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0, 2122 ("Object in gpu read domain")); 2123 KASSERT((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0, 2124 ("Object in gpu write domain")); 2125 2126 obj->gtt_offset = obj->gtt_space->start; 2127 2128 fenceable = 2129 obj->gtt_space->size == fence_size && 2130 (obj->gtt_space->start & (fence_alignment - 1)) == 0; 2131 2132 mappable = 2133 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end; 2134 obj->map_and_fenceable = mappable && fenceable; 2135 2136 CTR4(KTR_DRM, "object_bind %p %x %x %d", obj, obj->gtt_offset, 2137 obj->base.size, map_and_fenceable); 2138 return (0); 2139} 2140 2141static void 2142i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj) 2143{ 2144 u32 old_write_domain, old_read_domains; 2145 2146 /* Act a barrier for all accesses through the GTT */ 2147 mb(); 2148 2149 /* Force a pagefault for domain tracking on next user access */ 2150 i915_gem_release_mmap(obj); 2151 2152 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) 2153 return; 2154 2155 old_read_domains = obj->base.read_domains; 2156 old_write_domain = obj->base.write_domain; 2157 2158 obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT; 2159 obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT; 2160 2161 CTR3(KTR_DRM, "object_change_domain finish gtt %p %x %x", 2162 obj, old_read_domains, old_write_domain); 2163} 2164 2165int 2166i915_gem_object_unbind(struct drm_i915_gem_object *obj) 2167{ 2168 drm_i915_private_t *dev_priv; 2169 int ret; 2170 2171 dev_priv = obj->base.dev->dev_private; 2172 ret = 0; 2173 if (obj->gtt_space == NULL) 2174 return (0); 2175 if (obj->pin_count != 0) { 2176 DRM_ERROR("Attempting to unbind pinned buffer\n"); 2177 return (-EINVAL); 2178 } 2179 2180 ret = i915_gem_object_finish_gpu(obj); 2181 if (ret == -ERESTART || ret == -EINTR) 2182 return (ret); 2183 2184 i915_gem_object_finish_gtt(obj); 2185 2186 if (ret == 0) 2187 ret = i915_gem_object_set_to_cpu_domain(obj, 1); 2188 if (ret == -ERESTART || ret == -EINTR) 2189 return (ret); 2190 if (ret != 0) { 2191 i915_gem_clflush_object(obj); 2192 obj->base.read_domains = obj->base.write_domain = 2193 I915_GEM_DOMAIN_CPU; 2194 } 2195 2196 ret = i915_gem_object_put_fence(obj); 2197 if (ret == -ERESTART) 2198 return (ret); 2199 2200 i915_gem_gtt_unbind_object(obj); 2201 if (obj->has_aliasing_ppgtt_mapping) { 2202 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj); 2203 obj->has_aliasing_ppgtt_mapping = 0; 2204 } 2205 i915_gem_object_put_pages_gtt(obj); 2206 2207 list_del_init(&obj->gtt_list); 2208 list_del_init(&obj->mm_list); 2209 obj->map_and_fenceable = true; 2210 2211 drm_mm_put_block(obj->gtt_space); 2212 obj->gtt_space = NULL; 2213 obj->gtt_offset = 0; 2214 2215 if (i915_gem_object_is_purgeable(obj)) 2216 i915_gem_object_truncate(obj); 2217 CTR1(KTR_DRM, "object_unbind %p", obj); 2218 2219 return (ret); 2220} 2221 2222static int 2223i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, 2224 int flags) 2225{ 2226 struct drm_device *dev; 2227 vm_object_t vm_obj; 2228 vm_page_t m; 2229 int page_count, i, j; 2230 2231 dev = obj->base.dev; 2232 KASSERT(obj->pages == NULL, ("Obj already has pages")); 2233 page_count = obj->base.size / PAGE_SIZE; 2234 obj->pages = malloc(page_count * sizeof(vm_page_t), DRM_I915_GEM, 2235 M_WAITOK); 2236 vm_obj = obj->base.vm_obj; 2237 VM_OBJECT_WLOCK(vm_obj); 2238 for (i = 0; i < page_count; i++) { 2239 if ((obj->pages[i] = i915_gem_wire_page(vm_obj, i)) == NULL) 2240 goto failed; 2241 } 2242 VM_OBJECT_WUNLOCK(vm_obj); 2243 if (i915_gem_object_needs_bit17_swizzle(obj)) 2244 i915_gem_object_do_bit_17_swizzle(obj); 2245 return (0); 2246 2247failed: 2248 for (j = 0; j < i; j++) { 2249 m = obj->pages[j]; 2250 vm_page_lock(m); 2251 vm_page_unwire(m, 0); 2252 vm_page_unlock(m); 2253 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 2254 } 2255 VM_OBJECT_WUNLOCK(vm_obj); 2256 free(obj->pages, DRM_I915_GEM); 2257 obj->pages = NULL; 2258 return (-EIO); 2259} 2260 2261#define GEM_PARANOID_CHECK_GTT 0 2262#if GEM_PARANOID_CHECK_GTT 2263static void 2264i915_gem_assert_pages_not_mapped(struct drm_device *dev, vm_page_t *ma, 2265 int page_count) 2266{ 2267 struct drm_i915_private *dev_priv; 2268 vm_paddr_t pa; 2269 unsigned long start, end; 2270 u_int i; 2271 int j; 2272 2273 dev_priv = dev->dev_private; 2274 start = OFF_TO_IDX(dev_priv->mm.gtt_start); 2275 end = OFF_TO_IDX(dev_priv->mm.gtt_end); 2276 for (i = start; i < end; i++) { 2277 pa = intel_gtt_read_pte_paddr(i); 2278 for (j = 0; j < page_count; j++) { 2279 if (pa == VM_PAGE_TO_PHYS(ma[j])) { 2280 panic("Page %p in GTT pte index %d pte %x", 2281 ma[i], i, intel_gtt_read_pte(i)); 2282 } 2283 } 2284 } 2285} 2286#endif 2287 2288static void 2289i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) 2290{ 2291 vm_page_t m; 2292 int page_count, i; 2293 2294 KASSERT(obj->madv != I915_MADV_PURGED_INTERNAL, ("Purged object")); 2295 2296 if (obj->tiling_mode != I915_TILING_NONE) 2297 i915_gem_object_save_bit_17_swizzle(obj); 2298 if (obj->madv == I915_MADV_DONTNEED) 2299 obj->dirty = 0; 2300 page_count = obj->base.size / PAGE_SIZE; 2301 VM_OBJECT_WLOCK(obj->base.vm_obj); 2302#if GEM_PARANOID_CHECK_GTT 2303 i915_gem_assert_pages_not_mapped(obj->base.dev, obj->pages, page_count); 2304#endif 2305 for (i = 0; i < page_count; i++) { 2306 m = obj->pages[i]; 2307 if (obj->dirty) 2308 vm_page_dirty(m); 2309 if (obj->madv == I915_MADV_WILLNEED) 2310 vm_page_reference(m); 2311 vm_page_lock(m); 2312 vm_page_unwire(obj->pages[i], 1); 2313 vm_page_unlock(m); 2314 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 2315 } 2316 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 2317 obj->dirty = 0; 2318 free(obj->pages, DRM_I915_GEM); 2319 obj->pages = NULL; 2320} 2321 2322void 2323i915_gem_release_mmap(struct drm_i915_gem_object *obj) 2324{ 2325 vm_object_t devobj; 2326 vm_page_t m; 2327 int i, page_count; 2328 2329 if (!obj->fault_mappable) 2330 return; 2331 2332 CTR3(KTR_DRM, "release_mmap %p %x %x", obj, obj->gtt_offset, 2333 OFF_TO_IDX(obj->base.size)); 2334 devobj = cdev_pager_lookup(obj); 2335 if (devobj != NULL) { 2336 page_count = OFF_TO_IDX(obj->base.size); 2337 2338 VM_OBJECT_WLOCK(devobj); 2339retry: 2340 for (i = 0; i < page_count; i++) { 2341 m = vm_page_lookup(devobj, i); 2342 if (m == NULL) 2343 continue; 2344 if (vm_page_sleep_if_busy(m, "915unm")) 2345 goto retry; 2346 cdev_pager_free_page(devobj, m); 2347 } 2348 VM_OBJECT_WUNLOCK(devobj); 2349 vm_object_deallocate(devobj); 2350 } 2351 2352 obj->fault_mappable = false; 2353} 2354 2355int 2356i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj) 2357{ 2358 int ret; 2359 2360 KASSERT((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0, 2361 ("In GPU write domain")); 2362 2363 CTR5(KTR_DRM, "object_wait_rendering %p %s %x %d %d", obj, 2364 obj->ring != NULL ? obj->ring->name : "none", obj->gtt_offset, 2365 obj->active, obj->last_rendering_seqno); 2366 if (obj->active) { 2367 ret = i915_wait_request(obj->ring, obj->last_rendering_seqno, 2368 true); 2369 if (ret != 0) 2370 return (ret); 2371 } 2372 return (0); 2373} 2374 2375void 2376i915_gem_object_move_to_active(struct drm_i915_gem_object *obj, 2377 struct intel_ring_buffer *ring, uint32_t seqno) 2378{ 2379 struct drm_device *dev = obj->base.dev; 2380 struct drm_i915_private *dev_priv = dev->dev_private; 2381 struct drm_i915_fence_reg *reg; 2382 2383 obj->ring = ring; 2384 KASSERT(ring != NULL, ("NULL ring")); 2385 2386 /* Add a reference if we're newly entering the active list. */ 2387 if (!obj->active) { 2388 drm_gem_object_reference(&obj->base); 2389 obj->active = 1; 2390 } 2391 2392 /* Move from whatever list we were on to the tail of execution. */ 2393 list_move_tail(&obj->mm_list, &dev_priv->mm.active_list); 2394 list_move_tail(&obj->ring_list, &ring->active_list); 2395 2396 obj->last_rendering_seqno = seqno; 2397 if (obj->fenced_gpu_access) { 2398 obj->last_fenced_seqno = seqno; 2399 obj->last_fenced_ring = ring; 2400 2401 /* Bump MRU to take account of the delayed flush */ 2402 if (obj->fence_reg != I915_FENCE_REG_NONE) { 2403 reg = &dev_priv->fence_regs[obj->fence_reg]; 2404 list_move_tail(®->lru_list, 2405 &dev_priv->mm.fence_list); 2406 } 2407 } 2408} 2409 2410static void 2411i915_gem_object_move_off_active(struct drm_i915_gem_object *obj) 2412{ 2413 list_del_init(&obj->ring_list); 2414 obj->last_rendering_seqno = 0; 2415 obj->last_fenced_seqno = 0; 2416} 2417 2418static void 2419i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj) 2420{ 2421 struct drm_device *dev = obj->base.dev; 2422 drm_i915_private_t *dev_priv = dev->dev_private; 2423 2424 KASSERT(obj->active, ("Object not active")); 2425 list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list); 2426 2427 i915_gem_object_move_off_active(obj); 2428} 2429 2430static void 2431i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj) 2432{ 2433 struct drm_device *dev = obj->base.dev; 2434 struct drm_i915_private *dev_priv = dev->dev_private; 2435 2436 if (obj->pin_count != 0) 2437 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list); 2438 else 2439 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 2440 2441 KASSERT(list_empty(&obj->gpu_write_list), ("On gpu_write_list")); 2442 KASSERT(obj->active, ("Object not active")); 2443 obj->ring = NULL; 2444 obj->last_fenced_ring = NULL; 2445 2446 i915_gem_object_move_off_active(obj); 2447 obj->fenced_gpu_access = false; 2448 2449 obj->active = 0; 2450 obj->pending_gpu_write = false; 2451 drm_gem_object_unreference(&obj->base); 2452 2453#if 1 2454 KIB_NOTYET(); 2455#else 2456 WARN_ON(i915_verify_lists(dev)); 2457#endif 2458} 2459 2460static void 2461i915_gem_object_truncate(struct drm_i915_gem_object *obj) 2462{ 2463 vm_object_t vm_obj; 2464 2465 vm_obj = obj->base.vm_obj; 2466 VM_OBJECT_WLOCK(vm_obj); 2467 vm_object_page_remove(vm_obj, 0, 0, false); 2468 VM_OBJECT_WUNLOCK(vm_obj); 2469 obj->madv = I915_MADV_PURGED_INTERNAL; 2470} 2471 2472static inline int 2473i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj) 2474{ 2475 2476 return (obj->madv == I915_MADV_DONTNEED); 2477} 2478 2479static void 2480i915_gem_process_flushing_list(struct intel_ring_buffer *ring, 2481 uint32_t flush_domains) 2482{ 2483 struct drm_i915_gem_object *obj, *next; 2484 uint32_t old_write_domain; 2485 2486 list_for_each_entry_safe(obj, next, &ring->gpu_write_list, 2487 gpu_write_list) { 2488 if (obj->base.write_domain & flush_domains) { 2489 old_write_domain = obj->base.write_domain; 2490 obj->base.write_domain = 0; 2491 list_del_init(&obj->gpu_write_list); 2492 i915_gem_object_move_to_active(obj, ring, 2493 i915_gem_next_request_seqno(ring)); 2494 2495 CTR3(KTR_DRM, "object_change_domain process_flush %p %x %x", 2496 obj, obj->base.read_domains, old_write_domain); 2497 } 2498 } 2499} 2500 2501static int 2502i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj) 2503{ 2504 drm_i915_private_t *dev_priv; 2505 2506 dev_priv = obj->base.dev->dev_private; 2507 return (dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && 2508 obj->tiling_mode != I915_TILING_NONE); 2509} 2510 2511static vm_page_t 2512i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex) 2513{ 2514 vm_page_t m; 2515 int rv; 2516 2517 VM_OBJECT_ASSERT_WLOCKED(object); 2518 m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL); 2519 if (m->valid != VM_PAGE_BITS_ALL) { 2520 if (vm_pager_has_page(object, pindex, NULL, NULL)) { 2521 rv = vm_pager_get_pages(object, &m, 1, 0); 2522 m = vm_page_lookup(object, pindex); 2523 if (m == NULL) 2524 return (NULL); 2525 if (rv != VM_PAGER_OK) { 2526 vm_page_lock(m); 2527 vm_page_free(m); 2528 vm_page_unlock(m); 2529 return (NULL); 2530 } 2531 } else { 2532 pmap_zero_page(m); 2533 m->valid = VM_PAGE_BITS_ALL; 2534 m->dirty = 0; 2535 } 2536 } 2537 vm_page_lock(m); 2538 vm_page_wire(m); 2539 vm_page_unlock(m); 2540 vm_page_xunbusy(m); 2541 atomic_add_long(&i915_gem_wired_pages_cnt, 1); 2542 return (m); 2543} 2544 2545int 2546i915_gem_flush_ring(struct intel_ring_buffer *ring, uint32_t invalidate_domains, 2547 uint32_t flush_domains) 2548{ 2549 int ret; 2550 2551 if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0) 2552 return 0; 2553 2554 CTR3(KTR_DRM, "ring_flush %s %x %x", ring->name, invalidate_domains, 2555 flush_domains); 2556 ret = ring->flush(ring, invalidate_domains, flush_domains); 2557 if (ret) 2558 return ret; 2559 2560 if (flush_domains & I915_GEM_GPU_DOMAINS) 2561 i915_gem_process_flushing_list(ring, flush_domains); 2562 return 0; 2563} 2564 2565static int 2566i915_ring_idle(struct intel_ring_buffer *ring, bool do_retire) 2567{ 2568 int ret; 2569 2570 if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list)) 2571 return 0; 2572 2573 if (!list_empty(&ring->gpu_write_list)) { 2574 ret = i915_gem_flush_ring(ring, I915_GEM_GPU_DOMAINS, 2575 I915_GEM_GPU_DOMAINS); 2576 if (ret != 0) 2577 return ret; 2578 } 2579 2580 return (i915_wait_request(ring, i915_gem_next_request_seqno(ring), 2581 do_retire)); 2582} 2583 2584int 2585i915_gpu_idle(struct drm_device *dev, bool do_retire) 2586{ 2587 drm_i915_private_t *dev_priv = dev->dev_private; 2588 struct intel_ring_buffer *ring; 2589 int ret, i; 2590 2591 /* Flush everything onto the inactive list. */ 2592 for_each_ring(ring, dev_priv, i) { 2593 ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID); 2594 if (ret) 2595 return ret; 2596 2597 ret = i915_ring_idle(ring, do_retire); 2598 if (ret) 2599 return ret; 2600 } 2601 2602 return 0; 2603} 2604 2605int 2606i915_wait_request(struct intel_ring_buffer *ring, uint32_t seqno, bool do_retire) 2607{ 2608 drm_i915_private_t *dev_priv; 2609 struct drm_i915_gem_request *request; 2610 uint32_t ier; 2611 int flags, ret; 2612 bool recovery_complete; 2613 2614 KASSERT(seqno != 0, ("Zero seqno")); 2615 2616 dev_priv = ring->dev->dev_private; 2617 ret = 0; 2618 2619 if (atomic_load_acq_int(&dev_priv->mm.wedged) != 0) { 2620 /* Give the error handler a chance to run. */ 2621 mtx_lock(&dev_priv->error_completion_lock); 2622 recovery_complete = (&dev_priv->error_completion) > 0; 2623 mtx_unlock(&dev_priv->error_completion_lock); 2624 return (recovery_complete ? -EIO : -EAGAIN); 2625 } 2626 2627 if (seqno == ring->outstanding_lazy_request) { 2628 request = malloc(sizeof(*request), DRM_I915_GEM, 2629 M_WAITOK | M_ZERO); 2630 if (request == NULL) 2631 return (-ENOMEM); 2632 2633 ret = i915_add_request(ring, NULL, request); 2634 if (ret != 0) { 2635 free(request, DRM_I915_GEM); 2636 return (ret); 2637 } 2638 2639 seqno = request->seqno; 2640 } 2641 2642 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) { 2643 if (HAS_PCH_SPLIT(ring->dev)) 2644 ier = I915_READ(DEIER) | I915_READ(GTIER); 2645 else 2646 ier = I915_READ(IER); 2647 if (!ier) { 2648 DRM_ERROR("something (likely vbetool) disabled " 2649 "interrupts, re-enabling\n"); 2650 ring->dev->driver->irq_preinstall(ring->dev); 2651 ring->dev->driver->irq_postinstall(ring->dev); 2652 } 2653 2654 CTR2(KTR_DRM, "request_wait_begin %s %d", ring->name, seqno); 2655 2656 ring->waiting_seqno = seqno; 2657 mtx_lock(&ring->irq_lock); 2658 if (ring->irq_get(ring)) { 2659 flags = dev_priv->mm.interruptible ? PCATCH : 0; 2660 while (!i915_seqno_passed(ring->get_seqno(ring), seqno) 2661 && !atomic_load_acq_int(&dev_priv->mm.wedged) && 2662 ret == 0) { 2663 ret = -msleep(ring, &ring->irq_lock, flags, 2664 "915gwr", 0); 2665 } 2666 ring->irq_put(ring); 2667 mtx_unlock(&ring->irq_lock); 2668 } else { 2669 mtx_unlock(&ring->irq_lock); 2670 if (_intel_wait_for(ring->dev, 2671 i915_seqno_passed(ring->get_seqno(ring), seqno) || 2672 atomic_load_acq_int(&dev_priv->mm.wedged), 3000, 2673 0, "i915wrq") != 0) 2674 ret = -EBUSY; 2675 } 2676 ring->waiting_seqno = 0; 2677 2678 CTR3(KTR_DRM, "request_wait_end %s %d %d", ring->name, seqno, 2679 ret); 2680 } 2681 if (atomic_load_acq_int(&dev_priv->mm.wedged)) 2682 ret = -EAGAIN; 2683 2684 /* Directly dispatch request retiring. While we have the work queue 2685 * to handle this, the waiter on a request often wants an associated 2686 * buffer to have made it to the inactive list, and we would need 2687 * a separate wait queue to handle that. 2688 */ 2689 if (ret == 0 && do_retire) 2690 i915_gem_retire_requests_ring(ring); 2691 2692 return (ret); 2693} 2694 2695static u32 2696i915_gem_get_seqno(struct drm_device *dev) 2697{ 2698 drm_i915_private_t *dev_priv = dev->dev_private; 2699 u32 seqno = dev_priv->next_seqno; 2700 2701 /* reserve 0 for non-seqno */ 2702 if (++dev_priv->next_seqno == 0) 2703 dev_priv->next_seqno = 1; 2704 2705 return seqno; 2706} 2707 2708u32 2709i915_gem_next_request_seqno(struct intel_ring_buffer *ring) 2710{ 2711 if (ring->outstanding_lazy_request == 0) 2712 ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev); 2713 2714 return ring->outstanding_lazy_request; 2715} 2716 2717int 2718i915_add_request(struct intel_ring_buffer *ring, struct drm_file *file, 2719 struct drm_i915_gem_request *request) 2720{ 2721 drm_i915_private_t *dev_priv; 2722 struct drm_i915_file_private *file_priv; 2723 uint32_t seqno; 2724 u32 request_ring_position; 2725 int was_empty; 2726 int ret; 2727 2728 KASSERT(request != NULL, ("NULL request in add")); 2729 DRM_LOCK_ASSERT(ring->dev); 2730 dev_priv = ring->dev->dev_private; 2731 2732 seqno = i915_gem_next_request_seqno(ring); 2733 request_ring_position = intel_ring_get_tail(ring); 2734 2735 ret = ring->add_request(ring, &seqno); 2736 if (ret != 0) 2737 return ret; 2738 2739 CTR2(KTR_DRM, "request_add %s %d", ring->name, seqno); 2740 2741 request->seqno = seqno; 2742 request->ring = ring; 2743 request->tail = request_ring_position; 2744 request->emitted_jiffies = ticks; 2745 was_empty = list_empty(&ring->request_list); 2746 list_add_tail(&request->list, &ring->request_list); 2747 2748 if (file != NULL) { 2749 file_priv = file->driver_priv; 2750 2751 mtx_lock(&file_priv->mm.lck); 2752 request->file_priv = file_priv; 2753 list_add_tail(&request->client_list, 2754 &file_priv->mm.request_list); 2755 mtx_unlock(&file_priv->mm.lck); 2756 } 2757 2758 ring->outstanding_lazy_request = 0; 2759 2760 if (!dev_priv->mm.suspended) { 2761 if (i915_enable_hangcheck) { 2762 callout_schedule(&dev_priv->hangcheck_timer, 2763 DRM_I915_HANGCHECK_PERIOD); 2764 } 2765 if (was_empty) 2766 taskqueue_enqueue_timeout(dev_priv->tq, 2767 &dev_priv->mm.retire_task, hz); 2768 } 2769 return (0); 2770} 2771 2772static inline void 2773i915_gem_request_remove_from_client(struct drm_i915_gem_request *request) 2774{ 2775 struct drm_i915_file_private *file_priv = request->file_priv; 2776 2777 if (!file_priv) 2778 return; 2779 2780 DRM_LOCK_ASSERT(request->ring->dev); 2781 2782 mtx_lock(&file_priv->mm.lck); 2783 if (request->file_priv != NULL) { 2784 list_del(&request->client_list); 2785 request->file_priv = NULL; 2786 } 2787 mtx_unlock(&file_priv->mm.lck); 2788} 2789 2790void 2791i915_gem_release(struct drm_device *dev, struct drm_file *file) 2792{ 2793 struct drm_i915_file_private *file_priv; 2794 struct drm_i915_gem_request *request; 2795 2796 file_priv = file->driver_priv; 2797 2798 /* Clean up our request list when the client is going away, so that 2799 * later retire_requests won't dereference our soon-to-be-gone 2800 * file_priv. 2801 */ 2802 mtx_lock(&file_priv->mm.lck); 2803 while (!list_empty(&file_priv->mm.request_list)) { 2804 request = list_first_entry(&file_priv->mm.request_list, 2805 struct drm_i915_gem_request, 2806 client_list); 2807 list_del(&request->client_list); 2808 request->file_priv = NULL; 2809 } 2810 mtx_unlock(&file_priv->mm.lck); 2811} 2812 2813static void 2814i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv, 2815 struct intel_ring_buffer *ring) 2816{ 2817 2818 if (ring->dev != NULL) 2819 DRM_LOCK_ASSERT(ring->dev); 2820 2821 while (!list_empty(&ring->request_list)) { 2822 struct drm_i915_gem_request *request; 2823 2824 request = list_first_entry(&ring->request_list, 2825 struct drm_i915_gem_request, list); 2826 2827 list_del(&request->list); 2828 i915_gem_request_remove_from_client(request); 2829 free(request, DRM_I915_GEM); 2830 } 2831 2832 while (!list_empty(&ring->active_list)) { 2833 struct drm_i915_gem_object *obj; 2834 2835 obj = list_first_entry(&ring->active_list, 2836 struct drm_i915_gem_object, ring_list); 2837 2838 obj->base.write_domain = 0; 2839 list_del_init(&obj->gpu_write_list); 2840 i915_gem_object_move_to_inactive(obj); 2841 } 2842} 2843 2844static void 2845i915_gem_reset_fences(struct drm_device *dev) 2846{ 2847 struct drm_i915_private *dev_priv = dev->dev_private; 2848 int i; 2849 2850 for (i = 0; i < dev_priv->num_fence_regs; i++) { 2851 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; 2852 struct drm_i915_gem_object *obj = reg->obj; 2853 2854 if (!obj) 2855 continue; 2856 2857 if (obj->tiling_mode) 2858 i915_gem_release_mmap(obj); 2859 2860 reg->obj->fence_reg = I915_FENCE_REG_NONE; 2861 reg->obj->fenced_gpu_access = false; 2862 reg->obj->last_fenced_seqno = 0; 2863 reg->obj->last_fenced_ring = NULL; 2864 i915_gem_clear_fence_reg(dev, reg); 2865 } 2866} 2867 2868void 2869i915_gem_reset(struct drm_device *dev) 2870{ 2871 struct drm_i915_private *dev_priv = dev->dev_private; 2872 struct drm_i915_gem_object *obj; 2873 int i; 2874 2875 for (i = 0; i < I915_NUM_RINGS; i++) 2876 i915_gem_reset_ring_lists(dev_priv, &dev_priv->rings[i]); 2877 2878 /* Remove anything from the flushing lists. The GPU cache is likely 2879 * to be lost on reset along with the data, so simply move the 2880 * lost bo to the inactive list. 2881 */ 2882 while (!list_empty(&dev_priv->mm.flushing_list)) { 2883 obj = list_first_entry(&dev_priv->mm.flushing_list, 2884 struct drm_i915_gem_object, 2885 mm_list); 2886 2887 obj->base.write_domain = 0; 2888 list_del_init(&obj->gpu_write_list); 2889 i915_gem_object_move_to_inactive(obj); 2890 } 2891 2892 /* Move everything out of the GPU domains to ensure we do any 2893 * necessary invalidation upon reuse. 2894 */ 2895 list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) { 2896 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 2897 } 2898 2899 /* The fence registers are invalidated so clear them out */ 2900 i915_gem_reset_fences(dev); 2901} 2902 2903/** 2904 * This function clears the request list as sequence numbers are passed. 2905 */ 2906void 2907i915_gem_retire_requests_ring(struct intel_ring_buffer *ring) 2908{ 2909 uint32_t seqno; 2910 int i; 2911 2912 if (list_empty(&ring->request_list)) 2913 return; 2914 2915 seqno = ring->get_seqno(ring); 2916 CTR2(KTR_DRM, "retire_request_ring %s %d", ring->name, seqno); 2917 2918 for (i = 0; i < DRM_ARRAY_SIZE(ring->sync_seqno); i++) 2919 if (seqno >= ring->sync_seqno[i]) 2920 ring->sync_seqno[i] = 0; 2921 2922 while (!list_empty(&ring->request_list)) { 2923 struct drm_i915_gem_request *request; 2924 2925 request = list_first_entry(&ring->request_list, 2926 struct drm_i915_gem_request, 2927 list); 2928 2929 if (!i915_seqno_passed(seqno, request->seqno)) 2930 break; 2931 2932 CTR2(KTR_DRM, "retire_request_seqno_passed %s %d", 2933 ring->name, seqno); 2934 ring->last_retired_head = request->tail; 2935 2936 list_del(&request->list); 2937 i915_gem_request_remove_from_client(request); 2938 free(request, DRM_I915_GEM); 2939 } 2940 2941 /* Move any buffers on the active list that are no longer referenced 2942 * by the ringbuffer to the flushing/inactive lists as appropriate. 2943 */ 2944 while (!list_empty(&ring->active_list)) { 2945 struct drm_i915_gem_object *obj; 2946 2947 obj = list_first_entry(&ring->active_list, 2948 struct drm_i915_gem_object, 2949 ring_list); 2950 2951 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno)) 2952 break; 2953 2954 if (obj->base.write_domain != 0) 2955 i915_gem_object_move_to_flushing(obj); 2956 else 2957 i915_gem_object_move_to_inactive(obj); 2958 } 2959 2960 if (ring->trace_irq_seqno && 2961 i915_seqno_passed(seqno, ring->trace_irq_seqno)) { 2962 mtx_lock(&ring->irq_lock); 2963 ring->irq_put(ring); 2964 mtx_unlock(&ring->irq_lock); 2965 ring->trace_irq_seqno = 0; 2966 } 2967} 2968 2969void 2970i915_gem_retire_requests(struct drm_device *dev) 2971{ 2972 drm_i915_private_t *dev_priv = dev->dev_private; 2973 struct drm_i915_gem_object *obj, *next; 2974 int i; 2975 2976 if (!list_empty(&dev_priv->mm.deferred_free_list)) { 2977 list_for_each_entry_safe(obj, next, 2978 &dev_priv->mm.deferred_free_list, mm_list) 2979 i915_gem_free_object_tail(obj); 2980 } 2981 2982 for (i = 0; i < I915_NUM_RINGS; i++) 2983 i915_gem_retire_requests_ring(&dev_priv->rings[i]); 2984} 2985 2986static int 2987sandybridge_write_fence_reg(struct drm_i915_gem_object *obj, 2988 struct intel_ring_buffer *pipelined) 2989{ 2990 struct drm_device *dev = obj->base.dev; 2991 drm_i915_private_t *dev_priv = dev->dev_private; 2992 u32 size = obj->gtt_space->size; 2993 int regnum = obj->fence_reg; 2994 uint64_t val; 2995 2996 val = (uint64_t)((obj->gtt_offset + size - 4096) & 2997 0xfffff000) << 32; 2998 val |= obj->gtt_offset & 0xfffff000; 2999 val |= (uint64_t)((obj->stride / 128) - 1) << 3000 SANDYBRIDGE_FENCE_PITCH_SHIFT; 3001 3002 if (obj->tiling_mode == I915_TILING_Y) 3003 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 3004 val |= I965_FENCE_REG_VALID; 3005 3006 if (pipelined) { 3007 int ret = intel_ring_begin(pipelined, 6); 3008 if (ret) 3009 return ret; 3010 3011 intel_ring_emit(pipelined, MI_NOOP); 3012 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2)); 3013 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8); 3014 intel_ring_emit(pipelined, (u32)val); 3015 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4); 3016 intel_ring_emit(pipelined, (u32)(val >> 32)); 3017 intel_ring_advance(pipelined); 3018 } else 3019 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val); 3020 3021 return 0; 3022} 3023 3024static int 3025i965_write_fence_reg(struct drm_i915_gem_object *obj, 3026 struct intel_ring_buffer *pipelined) 3027{ 3028 struct drm_device *dev = obj->base.dev; 3029 drm_i915_private_t *dev_priv = dev->dev_private; 3030 u32 size = obj->gtt_space->size; 3031 int regnum = obj->fence_reg; 3032 uint64_t val; 3033 3034 val = (uint64_t)((obj->gtt_offset + size - 4096) & 3035 0xfffff000) << 32; 3036 val |= obj->gtt_offset & 0xfffff000; 3037 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT; 3038 if (obj->tiling_mode == I915_TILING_Y) 3039 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 3040 val |= I965_FENCE_REG_VALID; 3041 3042 if (pipelined) { 3043 int ret = intel_ring_begin(pipelined, 6); 3044 if (ret) 3045 return ret; 3046 3047 intel_ring_emit(pipelined, MI_NOOP); 3048 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2)); 3049 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8); 3050 intel_ring_emit(pipelined, (u32)val); 3051 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4); 3052 intel_ring_emit(pipelined, (u32)(val >> 32)); 3053 intel_ring_advance(pipelined); 3054 } else 3055 I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val); 3056 3057 return 0; 3058} 3059 3060static int 3061i915_write_fence_reg(struct drm_i915_gem_object *obj, 3062 struct intel_ring_buffer *pipelined) 3063{ 3064 struct drm_device *dev = obj->base.dev; 3065 drm_i915_private_t *dev_priv = dev->dev_private; 3066 u32 size = obj->gtt_space->size; 3067 u32 fence_reg, val, pitch_val; 3068 int tile_width; 3069 3070 if ((obj->gtt_offset & ~I915_FENCE_START_MASK) || 3071 (size & -size) != size || (obj->gtt_offset & (size - 1))) { 3072 printf( 3073"object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n", 3074 obj->gtt_offset, obj->map_and_fenceable, size); 3075 return -EINVAL; 3076 } 3077 3078 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)) 3079 tile_width = 128; 3080 else 3081 tile_width = 512; 3082 3083 /* Note: pitch better be a power of two tile widths */ 3084 pitch_val = obj->stride / tile_width; 3085 pitch_val = ffs(pitch_val) - 1; 3086 3087 val = obj->gtt_offset; 3088 if (obj->tiling_mode == I915_TILING_Y) 3089 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 3090 val |= I915_FENCE_SIZE_BITS(size); 3091 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 3092 val |= I830_FENCE_REG_VALID; 3093 3094 fence_reg = obj->fence_reg; 3095 if (fence_reg < 8) 3096 fence_reg = FENCE_REG_830_0 + fence_reg * 4; 3097 else 3098 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4; 3099 3100 if (pipelined) { 3101 int ret = intel_ring_begin(pipelined, 4); 3102 if (ret) 3103 return ret; 3104 3105 intel_ring_emit(pipelined, MI_NOOP); 3106 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1)); 3107 intel_ring_emit(pipelined, fence_reg); 3108 intel_ring_emit(pipelined, val); 3109 intel_ring_advance(pipelined); 3110 } else 3111 I915_WRITE(fence_reg, val); 3112 3113 return 0; 3114} 3115 3116static int 3117i830_write_fence_reg(struct drm_i915_gem_object *obj, 3118 struct intel_ring_buffer *pipelined) 3119{ 3120 struct drm_device *dev = obj->base.dev; 3121 drm_i915_private_t *dev_priv = dev->dev_private; 3122 u32 size = obj->gtt_space->size; 3123 int regnum = obj->fence_reg; 3124 uint32_t val; 3125 uint32_t pitch_val; 3126 3127 if ((obj->gtt_offset & ~I830_FENCE_START_MASK) || 3128 (size & -size) != size || (obj->gtt_offset & (size - 1))) { 3129 printf( 3130"object 0x%08x not 512K or pot-size 0x%08x aligned\n", 3131 obj->gtt_offset, size); 3132 return -EINVAL; 3133 } 3134 3135 pitch_val = obj->stride / 128; 3136 pitch_val = ffs(pitch_val) - 1; 3137 3138 val = obj->gtt_offset; 3139 if (obj->tiling_mode == I915_TILING_Y) 3140 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 3141 val |= I830_FENCE_SIZE_BITS(size); 3142 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 3143 val |= I830_FENCE_REG_VALID; 3144 3145 if (pipelined) { 3146 int ret = intel_ring_begin(pipelined, 4); 3147 if (ret) 3148 return ret; 3149 3150 intel_ring_emit(pipelined, MI_NOOP); 3151 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1)); 3152 intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4); 3153 intel_ring_emit(pipelined, val); 3154 intel_ring_advance(pipelined); 3155 } else 3156 I915_WRITE(FENCE_REG_830_0 + regnum * 4, val); 3157 3158 return 0; 3159} 3160 3161static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno) 3162{ 3163 return i915_seqno_passed(ring->get_seqno(ring), seqno); 3164} 3165 3166static int 3167i915_gem_object_flush_fence(struct drm_i915_gem_object *obj, 3168 struct intel_ring_buffer *pipelined) 3169{ 3170 int ret; 3171 3172 if (obj->fenced_gpu_access) { 3173 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 3174 ret = i915_gem_flush_ring(obj->last_fenced_ring, 0, 3175 obj->base.write_domain); 3176 if (ret) 3177 return ret; 3178 } 3179 3180 obj->fenced_gpu_access = false; 3181 } 3182 3183 if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) { 3184 if (!ring_passed_seqno(obj->last_fenced_ring, 3185 obj->last_fenced_seqno)) { 3186 ret = i915_wait_request(obj->last_fenced_ring, 3187 obj->last_fenced_seqno, 3188 true); 3189 if (ret) 3190 return ret; 3191 } 3192 3193 obj->last_fenced_seqno = 0; 3194 obj->last_fenced_ring = NULL; 3195 } 3196 3197 /* Ensure that all CPU reads are completed before installing a fence 3198 * and all writes before removing the fence. 3199 */ 3200 if (obj->base.read_domains & I915_GEM_DOMAIN_GTT) 3201 mb(); 3202 3203 return 0; 3204} 3205 3206int 3207i915_gem_object_put_fence(struct drm_i915_gem_object *obj) 3208{ 3209 int ret; 3210 3211 if (obj->tiling_mode) 3212 i915_gem_release_mmap(obj); 3213 3214 ret = i915_gem_object_flush_fence(obj, NULL); 3215 if (ret) 3216 return ret; 3217 3218 if (obj->fence_reg != I915_FENCE_REG_NONE) { 3219 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 3220 3221 if (dev_priv->fence_regs[obj->fence_reg].pin_count != 0) 3222 printf("%s: pin_count %d\n", __func__, 3223 dev_priv->fence_regs[obj->fence_reg].pin_count); 3224 i915_gem_clear_fence_reg(obj->base.dev, 3225 &dev_priv->fence_regs[obj->fence_reg]); 3226 3227 obj->fence_reg = I915_FENCE_REG_NONE; 3228 } 3229 3230 return 0; 3231} 3232 3233static struct drm_i915_fence_reg * 3234i915_find_fence_reg(struct drm_device *dev, struct intel_ring_buffer *pipelined) 3235{ 3236 struct drm_i915_private *dev_priv = dev->dev_private; 3237 struct drm_i915_fence_reg *reg, *first, *avail; 3238 int i; 3239 3240 /* First try to find a free reg */ 3241 avail = NULL; 3242 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) { 3243 reg = &dev_priv->fence_regs[i]; 3244 if (!reg->obj) 3245 return reg; 3246 3247 if (!reg->pin_count) 3248 avail = reg; 3249 } 3250 3251 if (avail == NULL) 3252 return NULL; 3253 3254 /* None available, try to steal one or wait for a user to finish */ 3255 avail = first = NULL; 3256 list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) { 3257 if (reg->pin_count) 3258 continue; 3259 3260 if (first == NULL) 3261 first = reg; 3262 3263 if (!pipelined || 3264 !reg->obj->last_fenced_ring || 3265 reg->obj->last_fenced_ring == pipelined) { 3266 avail = reg; 3267 break; 3268 } 3269 } 3270 3271 if (avail == NULL) 3272 avail = first; 3273 3274 return avail; 3275} 3276 3277int 3278i915_gem_object_get_fence(struct drm_i915_gem_object *obj, 3279 struct intel_ring_buffer *pipelined) 3280{ 3281 struct drm_device *dev = obj->base.dev; 3282 struct drm_i915_private *dev_priv = dev->dev_private; 3283 struct drm_i915_fence_reg *reg; 3284 int ret; 3285 3286 pipelined = NULL; 3287 ret = 0; 3288 3289 if (obj->fence_reg != I915_FENCE_REG_NONE) { 3290 reg = &dev_priv->fence_regs[obj->fence_reg]; 3291 list_move_tail(®->lru_list, &dev_priv->mm.fence_list); 3292 3293 if (obj->tiling_changed) { 3294 ret = i915_gem_object_flush_fence(obj, pipelined); 3295 if (ret) 3296 return ret; 3297 3298 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno) 3299 pipelined = NULL; 3300 3301 if (pipelined) { 3302 reg->setup_seqno = 3303 i915_gem_next_request_seqno(pipelined); 3304 obj->last_fenced_seqno = reg->setup_seqno; 3305 obj->last_fenced_ring = pipelined; 3306 } 3307 3308 goto update; 3309 } 3310 3311 if (!pipelined) { 3312 if (reg->setup_seqno) { 3313 if (!ring_passed_seqno(obj->last_fenced_ring, 3314 reg->setup_seqno)) { 3315 ret = i915_wait_request( 3316 obj->last_fenced_ring, 3317 reg->setup_seqno, 3318 true); 3319 if (ret) 3320 return ret; 3321 } 3322 3323 reg->setup_seqno = 0; 3324 } 3325 } else if (obj->last_fenced_ring && 3326 obj->last_fenced_ring != pipelined) { 3327 ret = i915_gem_object_flush_fence(obj, pipelined); 3328 if (ret) 3329 return ret; 3330 } 3331 3332 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno) 3333 pipelined = NULL; 3334 KASSERT(pipelined || reg->setup_seqno == 0, ("!pipelined")); 3335 3336 if (obj->tiling_changed) { 3337 if (pipelined) { 3338 reg->setup_seqno = 3339 i915_gem_next_request_seqno(pipelined); 3340 obj->last_fenced_seqno = reg->setup_seqno; 3341 obj->last_fenced_ring = pipelined; 3342 } 3343 goto update; 3344 } 3345 3346 return 0; 3347 } 3348 3349 reg = i915_find_fence_reg(dev, pipelined); 3350 if (reg == NULL) 3351 return -EDEADLK; 3352 3353 ret = i915_gem_object_flush_fence(obj, pipelined); 3354 if (ret) 3355 return ret; 3356 3357 if (reg->obj) { 3358 struct drm_i915_gem_object *old = reg->obj; 3359 3360 drm_gem_object_reference(&old->base); 3361 3362 if (old->tiling_mode) 3363 i915_gem_release_mmap(old); 3364 3365 ret = i915_gem_object_flush_fence(old, pipelined); 3366 if (ret) { 3367 drm_gem_object_unreference(&old->base); 3368 return ret; 3369 } 3370 3371 if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0) 3372 pipelined = NULL; 3373 3374 old->fence_reg = I915_FENCE_REG_NONE; 3375 old->last_fenced_ring = pipelined; 3376 old->last_fenced_seqno = 3377 pipelined ? i915_gem_next_request_seqno(pipelined) : 0; 3378 3379 drm_gem_object_unreference(&old->base); 3380 } else if (obj->last_fenced_seqno == 0) 3381 pipelined = NULL; 3382 3383 reg->obj = obj; 3384 list_move_tail(®->lru_list, &dev_priv->mm.fence_list); 3385 obj->fence_reg = reg - dev_priv->fence_regs; 3386 obj->last_fenced_ring = pipelined; 3387 3388 reg->setup_seqno = 3389 pipelined ? i915_gem_next_request_seqno(pipelined) : 0; 3390 obj->last_fenced_seqno = reg->setup_seqno; 3391 3392update: 3393 obj->tiling_changed = false; 3394 switch (INTEL_INFO(dev)->gen) { 3395 case 7: 3396 case 6: 3397 ret = sandybridge_write_fence_reg(obj, pipelined); 3398 break; 3399 case 5: 3400 case 4: 3401 ret = i965_write_fence_reg(obj, pipelined); 3402 break; 3403 case 3: 3404 ret = i915_write_fence_reg(obj, pipelined); 3405 break; 3406 case 2: 3407 ret = i830_write_fence_reg(obj, pipelined); 3408 break; 3409 } 3410 3411 return ret; 3412} 3413 3414static void 3415i915_gem_clear_fence_reg(struct drm_device *dev, struct drm_i915_fence_reg *reg) 3416{ 3417 drm_i915_private_t *dev_priv = dev->dev_private; 3418 uint32_t fence_reg = reg - dev_priv->fence_regs; 3419 3420 switch (INTEL_INFO(dev)->gen) { 3421 case 7: 3422 case 6: 3423 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0); 3424 break; 3425 case 5: 3426 case 4: 3427 I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0); 3428 break; 3429 case 3: 3430 if (fence_reg >= 8) 3431 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4; 3432 else 3433 case 2: 3434 fence_reg = FENCE_REG_830_0 + fence_reg * 4; 3435 3436 I915_WRITE(fence_reg, 0); 3437 break; 3438 } 3439 3440 list_del_init(®->lru_list); 3441 reg->obj = NULL; 3442 reg->setup_seqno = 0; 3443 reg->pin_count = 0; 3444} 3445 3446int 3447i915_gem_init_object(struct drm_gem_object *obj) 3448{ 3449 3450 printf("i915_gem_init_object called\n"); 3451 return (0); 3452} 3453 3454static bool 3455i915_gem_object_is_inactive(struct drm_i915_gem_object *obj) 3456{ 3457 3458 return (obj->gtt_space && !obj->active && obj->pin_count == 0); 3459} 3460 3461static void 3462i915_gem_retire_task_handler(void *arg, int pending) 3463{ 3464 drm_i915_private_t *dev_priv; 3465 struct drm_device *dev; 3466 bool idle; 3467 int i; 3468 3469 dev_priv = arg; 3470 dev = dev_priv->dev; 3471 3472 /* Come back later if the device is busy... */ 3473 if (!sx_try_xlock(&dev->dev_struct_lock)) { 3474 taskqueue_enqueue_timeout(dev_priv->tq, 3475 &dev_priv->mm.retire_task, hz); 3476 return; 3477 } 3478 3479 CTR0(KTR_DRM, "retire_task"); 3480 3481 i915_gem_retire_requests(dev); 3482 3483 /* Send a periodic flush down the ring so we don't hold onto GEM 3484 * objects indefinitely. 3485 */ 3486 idle = true; 3487 for (i = 0; i < I915_NUM_RINGS; i++) { 3488 struct intel_ring_buffer *ring = &dev_priv->rings[i]; 3489 3490 if (!list_empty(&ring->gpu_write_list)) { 3491 struct drm_i915_gem_request *request; 3492 int ret; 3493 3494 ret = i915_gem_flush_ring(ring, 3495 0, I915_GEM_GPU_DOMAINS); 3496 request = malloc(sizeof(*request), DRM_I915_GEM, 3497 M_WAITOK | M_ZERO); 3498 if (ret || request == NULL || 3499 i915_add_request(ring, NULL, request)) 3500 free(request, DRM_I915_GEM); 3501 } 3502 3503 idle &= list_empty(&ring->request_list); 3504 } 3505 3506 if (!dev_priv->mm.suspended && !idle) 3507 taskqueue_enqueue_timeout(dev_priv->tq, 3508 &dev_priv->mm.retire_task, hz); 3509 3510 DRM_UNLOCK(dev); 3511} 3512 3513void 3514i915_gem_lastclose(struct drm_device *dev) 3515{ 3516 int ret; 3517 3518 if (drm_core_check_feature(dev, DRIVER_MODESET)) 3519 return; 3520 3521 ret = i915_gem_idle(dev); 3522 if (ret != 0) 3523 DRM_ERROR("failed to idle hardware: %d\n", ret); 3524} 3525 3526static int 3527i915_gem_init_phys_object(struct drm_device *dev, int id, int size, int align) 3528{ 3529 drm_i915_private_t *dev_priv; 3530 struct drm_i915_gem_phys_object *phys_obj; 3531 int ret; 3532 3533 dev_priv = dev->dev_private; 3534 if (dev_priv->mm.phys_objs[id - 1] != NULL || size == 0) 3535 return (0); 3536 3537 phys_obj = malloc(sizeof(struct drm_i915_gem_phys_object), DRM_I915_GEM, 3538 M_WAITOK | M_ZERO); 3539 3540 phys_obj->id = id; 3541 3542 phys_obj->handle = drm_pci_alloc(dev, size, align, ~0); 3543 if (phys_obj->handle == NULL) { 3544 ret = -ENOMEM; 3545 goto free_obj; 3546 } 3547 pmap_change_attr((vm_offset_t)phys_obj->handle->vaddr, 3548 size / PAGE_SIZE, PAT_WRITE_COMBINING); 3549 3550 dev_priv->mm.phys_objs[id - 1] = phys_obj; 3551 3552 return (0); 3553 3554free_obj: 3555 free(phys_obj, DRM_I915_GEM); 3556 return (ret); 3557} 3558 3559static void 3560i915_gem_free_phys_object(struct drm_device *dev, int id) 3561{ 3562 drm_i915_private_t *dev_priv; 3563 struct drm_i915_gem_phys_object *phys_obj; 3564 3565 dev_priv = dev->dev_private; 3566 if (dev_priv->mm.phys_objs[id - 1] == NULL) 3567 return; 3568 3569 phys_obj = dev_priv->mm.phys_objs[id - 1]; 3570 if (phys_obj->cur_obj != NULL) 3571 i915_gem_detach_phys_object(dev, phys_obj->cur_obj); 3572 3573 drm_pci_free(dev, phys_obj->handle); 3574 free(phys_obj, DRM_I915_GEM); 3575 dev_priv->mm.phys_objs[id - 1] = NULL; 3576} 3577 3578void 3579i915_gem_free_all_phys_object(struct drm_device *dev) 3580{ 3581 int i; 3582 3583 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++) 3584 i915_gem_free_phys_object(dev, i); 3585} 3586 3587void 3588i915_gem_detach_phys_object(struct drm_device *dev, 3589 struct drm_i915_gem_object *obj) 3590{ 3591 vm_page_t m; 3592 struct sf_buf *sf; 3593 char *vaddr, *dst; 3594 int i, page_count; 3595 3596 if (obj->phys_obj == NULL) 3597 return; 3598 vaddr = obj->phys_obj->handle->vaddr; 3599 3600 page_count = obj->base.size / PAGE_SIZE; 3601 VM_OBJECT_WLOCK(obj->base.vm_obj); 3602 for (i = 0; i < page_count; i++) { 3603 m = i915_gem_wire_page(obj->base.vm_obj, i); 3604 if (m == NULL) 3605 continue; /* XXX */ 3606 3607 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 3608 sf = sf_buf_alloc(m, 0); 3609 if (sf != NULL) { 3610 dst = (char *)sf_buf_kva(sf); 3611 memcpy(dst, vaddr + IDX_TO_OFF(i), PAGE_SIZE); 3612 sf_buf_free(sf); 3613 } 3614 drm_clflush_pages(&m, 1); 3615 3616 VM_OBJECT_WLOCK(obj->base.vm_obj); 3617 vm_page_reference(m); 3618 vm_page_lock(m); 3619 vm_page_dirty(m); 3620 vm_page_unwire(m, 0); 3621 vm_page_unlock(m); 3622 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 3623 } 3624 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 3625 intel_gtt_chipset_flush(); 3626 3627 obj->phys_obj->cur_obj = NULL; 3628 obj->phys_obj = NULL; 3629} 3630 3631int 3632i915_gem_attach_phys_object(struct drm_device *dev, 3633 struct drm_i915_gem_object *obj, int id, int align) 3634{ 3635 drm_i915_private_t *dev_priv; 3636 vm_page_t m; 3637 struct sf_buf *sf; 3638 char *dst, *src; 3639 int i, page_count, ret; 3640 3641 if (id > I915_MAX_PHYS_OBJECT) 3642 return (-EINVAL); 3643 3644 if (obj->phys_obj != NULL) { 3645 if (obj->phys_obj->id == id) 3646 return (0); 3647 i915_gem_detach_phys_object(dev, obj); 3648 } 3649 3650 dev_priv = dev->dev_private; 3651 if (dev_priv->mm.phys_objs[id - 1] == NULL) { 3652 ret = i915_gem_init_phys_object(dev, id, obj->base.size, align); 3653 if (ret != 0) { 3654 DRM_ERROR("failed to init phys object %d size: %zu\n", 3655 id, obj->base.size); 3656 return (ret); 3657 } 3658 } 3659 3660 /* bind to the object */ 3661 obj->phys_obj = dev_priv->mm.phys_objs[id - 1]; 3662 obj->phys_obj->cur_obj = obj; 3663 3664 page_count = obj->base.size / PAGE_SIZE; 3665 3666 VM_OBJECT_WLOCK(obj->base.vm_obj); 3667 ret = 0; 3668 for (i = 0; i < page_count; i++) { 3669 m = i915_gem_wire_page(obj->base.vm_obj, i); 3670 if (m == NULL) { 3671 ret = -EIO; 3672 break; 3673 } 3674 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 3675 sf = sf_buf_alloc(m, 0); 3676 src = (char *)sf_buf_kva(sf); 3677 dst = (char *)obj->phys_obj->handle->vaddr + IDX_TO_OFF(i); 3678 memcpy(dst, src, PAGE_SIZE); 3679 sf_buf_free(sf); 3680 3681 VM_OBJECT_WLOCK(obj->base.vm_obj); 3682 3683 vm_page_reference(m); 3684 vm_page_lock(m); 3685 vm_page_unwire(m, 0); 3686 vm_page_unlock(m); 3687 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 3688 } 3689 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 3690 3691 return (0); 3692} 3693 3694static int 3695i915_gem_phys_pwrite(struct drm_device *dev, struct drm_i915_gem_object *obj, 3696 uint64_t data_ptr, uint64_t offset, uint64_t size, 3697 struct drm_file *file_priv) 3698{ 3699 char *user_data, *vaddr; 3700 int ret; 3701 3702 vaddr = (char *)obj->phys_obj->handle->vaddr + offset; 3703 user_data = (char *)(uintptr_t)data_ptr; 3704 3705 if (copyin_nofault(user_data, vaddr, size) != 0) { 3706 /* The physical object once assigned is fixed for the lifetime 3707 * of the obj, so we can safely drop the lock and continue 3708 * to access vaddr. 3709 */ 3710 DRM_UNLOCK(dev); 3711 ret = -copyin(user_data, vaddr, size); 3712 DRM_LOCK(dev); 3713 if (ret != 0) 3714 return (ret); 3715 } 3716 3717 intel_gtt_chipset_flush(); 3718 return (0); 3719} 3720 3721static int 3722i915_gpu_is_active(struct drm_device *dev) 3723{ 3724 drm_i915_private_t *dev_priv; 3725 3726 dev_priv = dev->dev_private; 3727 return (!list_empty(&dev_priv->mm.flushing_list) || 3728 !list_empty(&dev_priv->mm.active_list)); 3729} 3730 3731static void 3732i915_gem_lowmem(void *arg) 3733{ 3734 struct drm_device *dev; 3735 struct drm_i915_private *dev_priv; 3736 struct drm_i915_gem_object *obj, *next; 3737 int cnt, cnt_fail, cnt_total; 3738 3739 dev = arg; 3740 dev_priv = dev->dev_private; 3741 3742 if (!sx_try_xlock(&dev->dev_struct_lock)) 3743 return; 3744 3745 CTR0(KTR_DRM, "gem_lowmem"); 3746 3747rescan: 3748 /* first scan for clean buffers */ 3749 i915_gem_retire_requests(dev); 3750 3751 cnt_total = cnt_fail = cnt = 0; 3752 3753 list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list, 3754 mm_list) { 3755 if (i915_gem_object_is_purgeable(obj)) { 3756 if (i915_gem_object_unbind(obj) != 0) 3757 cnt_total++; 3758 } else 3759 cnt_total++; 3760 } 3761 3762 /* second pass, evict/count anything still on the inactive list */ 3763 list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list, 3764 mm_list) { 3765 if (i915_gem_object_unbind(obj) == 0) 3766 cnt++; 3767 else 3768 cnt_fail++; 3769 } 3770 3771 if (cnt_fail > cnt_total / 100 && i915_gpu_is_active(dev)) { 3772 /* 3773 * We are desperate for pages, so as a last resort, wait 3774 * for the GPU to finish and discard whatever we can. 3775 * This has a dramatic impact to reduce the number of 3776 * OOM-killer events whilst running the GPU aggressively. 3777 */ 3778 if (i915_gpu_idle(dev, true) == 0) 3779 goto rescan; 3780 } 3781 DRM_UNLOCK(dev); 3782} 3783 3784void 3785i915_gem_unload(struct drm_device *dev) 3786{ 3787 struct drm_i915_private *dev_priv; 3788 3789 dev_priv = dev->dev_private; 3790 EVENTHANDLER_DEREGISTER(vm_lowmem, dev_priv->mm.i915_lowmem); 3791} 3792