i915_gem.c revision 307672
1/*- 2 * Copyright �� 2008 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eric Anholt <eric@anholt.net> 25 * 26 * Copyright (c) 2011 The FreeBSD Foundation 27 * All rights reserved. 28 * 29 * This software was developed by Konstantin Belousov under sponsorship from 30 * the FreeBSD Foundation. 31 * 32 * Redistribution and use in source and binary forms, with or without 33 * modification, are permitted provided that the following conditions 34 * are met: 35 * 1. Redistributions of source code must retain the above copyright 36 * notice, this list of conditions and the following disclaimer. 37 * 2. Redistributions in binary form must reproduce the above copyright 38 * notice, this list of conditions and the following disclaimer in the 39 * documentation and/or other materials provided with the distribution. 40 * 41 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 42 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 43 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 44 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 45 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 46 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 47 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 48 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 49 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 50 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 51 * SUCH DAMAGE. 52 */ 53 54#include <sys/cdefs.h> 55__FBSDID("$FreeBSD: stable/10/sys/dev/drm2/i915/i915_gem.c 307672 2016-10-20 13:12:19Z kib $"); 56 57#include <dev/drm2/drmP.h> 58#include <dev/drm2/drm.h> 59#include <dev/drm2/i915/i915_drm.h> 60#include <dev/drm2/i915/i915_drv.h> 61#include <dev/drm2/i915/intel_drv.h> 62#include <dev/drm2/i915/intel_ringbuffer.h> 63#include <sys/resourcevar.h> 64#include <sys/sched.h> 65#include <sys/sf_buf.h> 66 67#include <vm/vm.h> 68#include <vm/vm_pageout.h> 69 70#include <machine/md_var.h> 71 72static void i915_gem_object_flush_cpu_write_domain( 73 struct drm_i915_gem_object *obj); 74static uint32_t i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, 75 int tiling_mode); 76static uint32_t i915_gem_get_gtt_alignment(struct drm_device *dev, 77 uint32_t size, int tiling_mode); 78static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, 79 unsigned alignment, bool map_and_fenceable); 80static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, 81 int flags); 82static void i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj); 83static void i915_gem_object_put_pages_range(struct drm_i915_gem_object *obj, 84 off_t start, off_t end); 85static int i915_gem_object_get_pages_range(struct drm_i915_gem_object *obj, 86 off_t start, off_t end); 87static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj); 88static void i915_gem_object_truncate(struct drm_i915_gem_object *obj); 89static int i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj); 90static bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj); 91static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj); 92static vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex, 93 bool *fresh); 94static void i915_gem_process_flushing_list(struct intel_ring_buffer *ring, 95 uint32_t flush_domains); 96static void i915_gem_reset_fences(struct drm_device *dev); 97static void i915_gem_retire_task_handler(void *arg, int pending); 98static void i915_gem_lowmem(void *arg); 99static void i915_gem_write_fence(struct drm_device *dev, int reg, 100 struct drm_i915_gem_object *obj); 101static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno, 102 bool interruptible); 103static int i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno); 104 105MALLOC_DEFINE(DRM_I915_GEM, "i915gem", "Allocations from i915 gem"); 106long i915_gem_wired_pages_cnt; 107 108static bool cpu_cache_is_coherent(struct drm_device *dev, 109 enum i915_cache_level level) 110{ 111 return HAS_LLC(dev) || level != I915_CACHE_NONE; 112} 113 114static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj) 115{ 116 if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) 117 return true; 118 119 return obj->pin_display; 120} 121 122static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj) 123{ 124 if (obj->tiling_mode) 125 i915_gem_release_mmap(obj); 126 127 /* As we do not have an associated fence register, we will force 128 * a tiling change if we ever need to acquire one. 129 */ 130 obj->fence_dirty = false; 131 obj->fence_reg = I915_FENCE_REG_NONE; 132} 133 134static void 135i915_gem_info_add_obj(struct drm_i915_private *dev_priv, size_t size) 136{ 137 138 dev_priv->mm.object_count++; 139 dev_priv->mm.object_memory += size; 140} 141 142static void 143i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, size_t size) 144{ 145 146 dev_priv->mm.object_count--; 147 dev_priv->mm.object_memory -= size; 148} 149 150static int 151i915_gem_wait_for_error(struct drm_device *dev) 152{ 153 struct drm_i915_private *dev_priv; 154 int ret; 155 156 dev_priv = dev->dev_private; 157 if (!atomic_load_acq_int(&dev_priv->mm.wedged)) 158 return (0); 159 160 mtx_lock(&dev_priv->error_completion_lock); 161 while (dev_priv->error_completion == 0) { 162 ret = -msleep(&dev_priv->error_completion, 163 &dev_priv->error_completion_lock, PCATCH, "915wco", 0); 164 if (ret == -ERESTART) 165 ret = -ERESTARTSYS; 166 if (ret != 0) { 167 mtx_unlock(&dev_priv->error_completion_lock); 168 return (ret); 169 } 170 } 171 mtx_unlock(&dev_priv->error_completion_lock); 172 173 if (atomic_load_acq_int(&dev_priv->mm.wedged)) { 174 mtx_lock(&dev_priv->error_completion_lock); 175 dev_priv->error_completion++; 176 mtx_unlock(&dev_priv->error_completion_lock); 177 } 178 return (0); 179} 180 181int 182i915_mutex_lock_interruptible(struct drm_device *dev) 183{ 184 struct drm_i915_private *dev_priv; 185 int ret; 186 187 dev_priv = dev->dev_private; 188 ret = i915_gem_wait_for_error(dev); 189 if (ret != 0) 190 return (ret); 191 192 /* 193 * interruptible shall it be. might indeed be if dev_lock is 194 * changed to sx 195 */ 196 ret = sx_xlock_sig(&dev->dev_struct_lock); 197 if (ret != 0) 198 return (-ret); 199 200 return (0); 201} 202 203 204void 205i915_gem_free_object(struct drm_gem_object *gem_obj) 206{ 207 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); 208 struct drm_device *dev; 209 drm_i915_private_t *dev_priv; 210 211 dev = obj->base.dev; 212 dev_priv = dev->dev_private; 213 214 CTR1(KTR_DRM, "object_destroy_tail %p", obj); 215 216 if (obj->phys_obj) 217 i915_gem_detach_phys_object(dev, obj); 218 219 obj->pin_count = 0; 220 if (i915_gem_object_unbind(obj) == -ERESTARTSYS) { 221 bool was_interruptible; 222 223 was_interruptible = dev_priv->mm.interruptible; 224 dev_priv->mm.interruptible = false; 225 226 if (i915_gem_object_unbind(obj)) 227 printf("i915_gem_free_object: unbind\n"); 228 229 dev_priv->mm.interruptible = was_interruptible; 230 } 231 232 drm_gem_free_mmap_offset(&obj->base); 233 drm_gem_object_release(&obj->base); 234 i915_gem_info_remove_obj(dev_priv, obj->base.size); 235 236 free(obj->bit_17, DRM_I915_GEM); 237 free(obj, DRM_I915_GEM); 238} 239 240static void 241init_ring_lists(struct intel_ring_buffer *ring) 242{ 243 244 INIT_LIST_HEAD(&ring->active_list); 245 INIT_LIST_HEAD(&ring->request_list); 246 INIT_LIST_HEAD(&ring->gpu_write_list); 247} 248 249void 250i915_gem_load(struct drm_device *dev) 251{ 252 drm_i915_private_t *dev_priv; 253 int i; 254 255 dev_priv = dev->dev_private; 256 257 INIT_LIST_HEAD(&dev_priv->mm.active_list); 258 INIT_LIST_HEAD(&dev_priv->mm.flushing_list); 259 INIT_LIST_HEAD(&dev_priv->mm.inactive_list); 260 INIT_LIST_HEAD(&dev_priv->mm.fence_list); 261 INIT_LIST_HEAD(&dev_priv->mm.gtt_list); 262 for (i = 0; i < I915_NUM_RINGS; i++) 263 init_ring_lists(&dev_priv->rings[i]); 264 for (i = 0; i < I915_MAX_NUM_FENCES; i++) 265 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); 266 TIMEOUT_TASK_INIT(dev_priv->tq, &dev_priv->mm.retire_task, 0, 267 i915_gem_retire_task_handler, dev_priv); 268 dev_priv->error_completion = 0; 269 270 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ 271 if (IS_GEN3(dev)) { 272 I915_WRITE(MI_ARB_STATE, 273 _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE)); 274 } 275 276 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL; 277 278 /* Old X drivers will take 0-2 for front, back, depth buffers */ 279 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 280 dev_priv->fence_reg_start = 3; 281 282 if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || 283 IS_G33(dev)) 284 dev_priv->num_fence_regs = 16; 285 else 286 dev_priv->num_fence_regs = 8; 287 288 /* Initialize fence registers to zero */ 289 i915_gem_reset_fences(dev); 290 291 i915_gem_detect_bit_6_swizzle(dev); 292 dev_priv->mm.interruptible = true; 293 294 dev_priv->mm.i915_lowmem = EVENTHANDLER_REGISTER(vm_lowmem, 295 i915_gem_lowmem, dev, EVENTHANDLER_PRI_ANY); 296} 297 298int 299i915_gem_init_ioctl(struct drm_device *dev, void *data, 300 struct drm_file *file) 301{ 302 struct drm_i915_gem_init *args; 303 drm_i915_private_t *dev_priv; 304 int error; 305 306 if (drm_core_check_feature(dev, DRIVER_MODESET)) 307 return -ENODEV; 308 309 dev_priv = dev->dev_private; 310 args = data; 311 312 if (args->gtt_start >= args->gtt_end || 313 (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1)) 314 return (-EINVAL); 315 316 if (mtx_initialized(&dev_priv->mm.gtt_space.unused_lock)) 317 return (-EBUSY); 318 319 /* GEM with user mode setting was never supported on ilk and later. */ 320 if (INTEL_INFO(dev)->gen >= 5) 321 return -ENODEV; 322 323 /* 324 * XXXKIB. The second-time initialization should be guarded 325 * against. 326 */ 327 DRM_LOCK(dev); 328 error = i915_gem_init_global_gtt(dev, args->gtt_start, 329 args->gtt_end, args->gtt_end); 330 DRM_UNLOCK(dev); 331 return (error); 332} 333 334int 335i915_gem_idle(struct drm_device *dev) 336{ 337 drm_i915_private_t *dev_priv; 338 int ret; 339 340 DRM_LOCK(dev); 341 342 dev_priv = dev->dev_private; 343 if (dev_priv->mm.suspended) { 344 DRM_UNLOCK(dev); 345 return (0); 346 } 347 348 ret = i915_gpu_idle(dev); 349 if (ret != 0) { 350 DRM_UNLOCK(dev); 351 return (ret); 352 } 353 i915_gem_retire_requests(dev); 354 355 /* Under UMS, be paranoid and evict. */ 356 if (!drm_core_check_feature(dev, DRIVER_MODESET)) { 357 ret = i915_gem_evict_everything(dev, false); 358 if (ret != 0) { 359 DRM_UNLOCK(dev); 360 return ret; 361 } 362 } 363 364 i915_gem_reset_fences(dev); 365 366 /* Hack! Don't let anybody do execbuf while we don't control the chip. 367 * We need to replace this with a semaphore, or something. 368 * And not confound mm.suspended! 369 */ 370 dev_priv->mm.suspended = 1; 371 callout_stop(&dev_priv->hangcheck_timer); 372 373 i915_kernel_lost_context(dev); 374 i915_gem_cleanup_ringbuffer(dev); 375 376 DRM_UNLOCK(dev); 377 378 /* Cancel the retire work handler, which should be idle now. */ 379 taskqueue_cancel_timeout(dev_priv->tq, &dev_priv->mm.retire_task, NULL); 380 return (ret); 381} 382 383void 384i915_gem_init_swizzling(struct drm_device *dev) 385{ 386 drm_i915_private_t *dev_priv; 387 388 dev_priv = dev->dev_private; 389 390 if (INTEL_INFO(dev)->gen < 5 || 391 dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) 392 return; 393 394 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | 395 DISP_TILE_SURFACE_SWIZZLING); 396 397 if (IS_GEN5(dev)) 398 return; 399 400 401 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL); 402 if (IS_GEN6(dev)) 403 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB)); 404 else 405 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB)); 406} 407 408void i915_gem_init_ppgtt(struct drm_device *dev) 409{ 410 drm_i915_private_t *dev_priv; 411 struct i915_hw_ppgtt *ppgtt; 412 uint32_t pd_offset, pd_entry; 413 vm_paddr_t pt_addr; 414 struct intel_ring_buffer *ring; 415 u_int first_pd_entry_in_global_pt, i; 416 417 dev_priv = dev->dev_private; 418 ppgtt = dev_priv->mm.aliasing_ppgtt; 419 if (ppgtt == NULL) 420 return; 421 422 first_pd_entry_in_global_pt = 512 * 1024 - I915_PPGTT_PD_ENTRIES; 423 for (i = 0; i < ppgtt->num_pd_entries; i++) { 424 pt_addr = VM_PAGE_TO_PHYS(ppgtt->pt_pages[i]); 425 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr); 426 pd_entry |= GEN6_PDE_VALID; 427 intel_gtt_write(first_pd_entry_in_global_pt + i, pd_entry); 428 } 429 intel_gtt_read_pte(first_pd_entry_in_global_pt); 430 431 pd_offset = ppgtt->pd_offset; 432 pd_offset /= 64; /* in cachelines, */ 433 pd_offset <<= 16; 434 435 if (INTEL_INFO(dev)->gen == 6) { 436 uint32_t ecochk, gab_ctl, ecobits; 437 438 ecobits = I915_READ(GAC_ECO_BITS); 439 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B); 440 441 gab_ctl = I915_READ(GAB_CTL); 442 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT); 443 444 ecochk = I915_READ(GAM_ECOCHK); 445 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | 446 ECOCHK_PPGTT_CACHE64B); 447 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); 448 } else if (INTEL_INFO(dev)->gen >= 7) { 449 I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B); 450 /* GFX_MODE is per-ring on gen7+ */ 451 } 452 453 for_each_ring(ring, dev_priv, i) { 454 if (INTEL_INFO(dev)->gen >= 7) 455 I915_WRITE(RING_MODE_GEN7(ring), 456 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); 457 458 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); 459 I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset); 460 } 461} 462 463int 464i915_gem_init_hw(struct drm_device *dev) 465{ 466 drm_i915_private_t *dev_priv; 467 int ret; 468 469 dev_priv = dev->dev_private; 470 471 i915_gem_init_swizzling(dev); 472 473 ret = intel_init_render_ring_buffer(dev); 474 if (ret != 0) 475 return (ret); 476 477 if (HAS_BSD(dev)) { 478 ret = intel_init_bsd_ring_buffer(dev); 479 if (ret != 0) 480 goto cleanup_render_ring; 481 } 482 483 if (HAS_BLT(dev)) { 484 ret = intel_init_blt_ring_buffer(dev); 485 if (ret != 0) 486 goto cleanup_bsd_ring; 487 } 488 489 dev_priv->next_seqno = 1; 490 i915_gem_context_init(dev); 491 i915_gem_init_ppgtt(dev); 492 return (0); 493 494cleanup_bsd_ring: 495 intel_cleanup_ring_buffer(&dev_priv->rings[VCS]); 496cleanup_render_ring: 497 intel_cleanup_ring_buffer(&dev_priv->rings[RCS]); 498 return (ret); 499} 500 501static bool 502intel_enable_ppgtt(struct drm_device *dev) 503{ 504 if (i915_enable_ppgtt >= 0) 505 return i915_enable_ppgtt; 506 507 /* Disable ppgtt on SNB if VT-d is on. */ 508 if (INTEL_INFO(dev)->gen == 6 && intel_iommu_enabled) 509 return false; 510 511 return true; 512} 513 514int i915_gem_init(struct drm_device *dev) 515{ 516 struct drm_i915_private *dev_priv = dev->dev_private; 517 unsigned long gtt_size, mappable_size; 518 int ret; 519 520 gtt_size = dev_priv->mm.gtt.gtt_total_entries << PAGE_SHIFT; 521 mappable_size = dev_priv->mm.gtt.gtt_mappable_entries << PAGE_SHIFT; 522 523 DRM_LOCK(dev); 524 if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) { 525 /* PPGTT pdes are stolen from global gtt ptes, so shrink the 526 * aperture accordingly when using aliasing ppgtt. */ 527 gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE; 528 529 i915_gem_init_global_gtt(dev, 0, mappable_size, gtt_size); 530 531 ret = i915_gem_init_aliasing_ppgtt(dev); 532 if (ret) { 533 DRM_UNLOCK(dev); 534 return ret; 535 } 536 } else { 537 /* Let GEM Manage all of the aperture. 538 * 539 * However, leave one page at the end still bound to the scratch 540 * page. There are a number of places where the hardware 541 * apparently prefetches past the end of the object, and we've 542 * seen multiple hangs with the GPU head pointer stuck in a 543 * batchbuffer bound at the last page of the aperture. One page 544 * should be enough to keep any prefetching inside of the 545 * aperture. 546 */ 547 i915_gem_init_global_gtt(dev, 0, mappable_size, 548 gtt_size); 549 } 550 551 ret = i915_gem_init_hw(dev); 552 DRM_UNLOCK(dev); 553 if (ret != 0) { 554 i915_gem_cleanup_aliasing_ppgtt(dev); 555 return (ret); 556 } 557 558 /* Allow hardware batchbuffers unless told otherwise, but not for KMS. */ 559 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 560 dev_priv->dri1.allow_batchbuffer = 1; 561 return 0; 562} 563 564int 565i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, 566 struct drm_file *file) 567{ 568 struct drm_i915_private *dev_priv; 569 struct drm_i915_gem_get_aperture *args; 570 struct drm_i915_gem_object *obj; 571 size_t pinned; 572 573 dev_priv = dev->dev_private; 574 args = data; 575 576 pinned = 0; 577 DRM_LOCK(dev); 578 list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) 579 if (obj->pin_count) 580 pinned += obj->gtt_space->size; 581 DRM_UNLOCK(dev); 582 583 args->aper_size = dev_priv->mm.gtt_total; 584 args->aper_available_size = args->aper_size - pinned; 585 586 return (0); 587} 588 589int 590i915_gem_object_pin(struct drm_i915_gem_object *obj, uint32_t alignment, 591 bool map_and_fenceable) 592{ 593 int ret; 594 595 if (obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT) 596 return (-EBUSY); 597 598 if (obj->gtt_space != NULL) { 599 if ((alignment && obj->gtt_offset & (alignment - 1)) || 600 (map_and_fenceable && !obj->map_and_fenceable)) { 601 DRM_DEBUG("bo is already pinned with incorrect alignment:" 602 " offset=%x, req.alignment=%x, req.map_and_fenceable=%d," 603 " obj->map_and_fenceable=%d\n", 604 obj->gtt_offset, alignment, 605 map_and_fenceable, 606 obj->map_and_fenceable); 607 ret = i915_gem_object_unbind(obj); 608 if (ret != 0) 609 return (ret); 610 } 611 } 612 613 if (obj->gtt_space == NULL) { 614 ret = i915_gem_object_bind_to_gtt(obj, alignment, 615 map_and_fenceable); 616 if (ret) 617 return (ret); 618 } 619 620 if (!obj->has_global_gtt_mapping && map_and_fenceable) 621 i915_gem_gtt_bind_object(obj, obj->cache_level); 622 623 obj->pin_count++; 624 obj->pin_mappable |= map_and_fenceable; 625 626 return 0; 627} 628 629void 630i915_gem_object_unpin(struct drm_i915_gem_object *obj) 631{ 632 633 KASSERT(obj->pin_count != 0, ("zero pin count")); 634 KASSERT(obj->gtt_space != NULL, ("No gtt mapping")); 635 636 if (--obj->pin_count == 0) 637 obj->pin_mappable = false; 638} 639 640int 641i915_gem_pin_ioctl(struct drm_device *dev, void *data, 642 struct drm_file *file) 643{ 644 struct drm_i915_gem_pin *args; 645 struct drm_i915_gem_object *obj; 646 struct drm_gem_object *gobj; 647 int ret; 648 649 args = data; 650 651 ret = i915_mutex_lock_interruptible(dev); 652 if (ret != 0) 653 return ret; 654 655 gobj = drm_gem_object_lookup(dev, file, args->handle); 656 if (gobj == NULL) { 657 ret = -ENOENT; 658 goto unlock; 659 } 660 obj = to_intel_bo(gobj); 661 662 if (obj->madv != I915_MADV_WILLNEED) { 663 DRM_ERROR("Attempting to pin a purgeable buffer\n"); 664 ret = -EINVAL; 665 goto out; 666 } 667 668 if (obj->pin_filp != NULL && obj->pin_filp != file) { 669 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n", 670 args->handle); 671 ret = -EINVAL; 672 goto out; 673 } 674 675 obj->user_pin_count++; 676 obj->pin_filp = file; 677 if (obj->user_pin_count == 1) { 678 ret = i915_gem_object_pin(obj, args->alignment, true); 679 if (ret != 0) 680 goto out; 681 } 682 683 /* XXX - flush the CPU caches for pinned objects 684 * as the X server doesn't manage domains yet 685 */ 686 i915_gem_object_flush_cpu_write_domain(obj); 687 args->offset = obj->gtt_offset; 688out: 689 drm_gem_object_unreference(&obj->base); 690unlock: 691 DRM_UNLOCK(dev); 692 return (ret); 693} 694 695int 696i915_gem_unpin_ioctl(struct drm_device *dev, void *data, 697 struct drm_file *file) 698{ 699 struct drm_i915_gem_pin *args; 700 struct drm_i915_gem_object *obj; 701 int ret; 702 703 args = data; 704 ret = i915_mutex_lock_interruptible(dev); 705 if (ret != 0) 706 return (ret); 707 708 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 709 if (&obj->base == NULL) { 710 ret = -ENOENT; 711 goto unlock; 712 } 713 714 if (obj->pin_filp != file) { 715 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", 716 args->handle); 717 ret = -EINVAL; 718 goto out; 719 } 720 obj->user_pin_count--; 721 if (obj->user_pin_count == 0) { 722 obj->pin_filp = NULL; 723 i915_gem_object_unpin(obj); 724 } 725 726out: 727 drm_gem_object_unreference(&obj->base); 728unlock: 729 DRM_UNLOCK(dev); 730 return (ret); 731} 732 733int 734i915_gem_busy_ioctl(struct drm_device *dev, void *data, 735 struct drm_file *file) 736{ 737 struct drm_i915_gem_busy *args; 738 struct drm_i915_gem_object *obj; 739 int ret; 740 741 args = data; 742 743 ret = i915_mutex_lock_interruptible(dev); 744 if (ret != 0) 745 return ret; 746 747 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 748 if (&obj->base == NULL) { 749 ret = -ENOENT; 750 goto unlock; 751 } 752 753 args->busy = obj->active; 754 if (args->busy) { 755 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 756 ret = i915_gem_flush_ring(obj->ring, 757 0, obj->base.write_domain); 758 } else { 759 ret = i915_gem_check_olr(obj->ring, 760 obj->last_rendering_seqno); 761 } 762 763 i915_gem_retire_requests_ring(obj->ring); 764 args->busy = obj->active; 765 } 766 767 drm_gem_object_unreference(&obj->base); 768unlock: 769 DRM_UNLOCK(dev); 770 return (ret); 771} 772 773static int 774i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) 775{ 776 struct drm_i915_private *dev_priv; 777 struct drm_i915_file_private *file_priv; 778 unsigned long recent_enough; 779 struct drm_i915_gem_request *request; 780 struct intel_ring_buffer *ring; 781 u32 seqno; 782 int ret; 783 784 dev_priv = dev->dev_private; 785 if (atomic_load_acq_int(&dev_priv->mm.wedged)) 786 return (-EIO); 787 788 file_priv = file->driver_priv; 789 recent_enough = ticks - (20 * hz / 1000); 790 ring = NULL; 791 seqno = 0; 792 793 mtx_lock(&file_priv->mm.lck); 794 list_for_each_entry(request, &file_priv->mm.request_list, client_list) { 795 if (time_after_eq(request->emitted_jiffies, recent_enough)) 796 break; 797 ring = request->ring; 798 seqno = request->seqno; 799 } 800 mtx_unlock(&file_priv->mm.lck); 801 if (seqno == 0) 802 return (0); 803 804 ret = __wait_seqno(ring, seqno, true); 805 if (ret == 0) 806 taskqueue_enqueue_timeout(dev_priv->tq, 807 &dev_priv->mm.retire_task, 0); 808 809 return (ret); 810} 811 812int 813i915_gem_throttle_ioctl(struct drm_device *dev, void *data, 814 struct drm_file *file_priv) 815{ 816 817 return (i915_gem_ring_throttle(dev, file_priv)); 818} 819 820int 821i915_gem_madvise_ioctl(struct drm_device *dev, void *data, 822 struct drm_file *file_priv) 823{ 824 struct drm_i915_gem_madvise *args; 825 struct drm_i915_gem_object *obj; 826 int ret; 827 828 args = data; 829 switch (args->madv) { 830 case I915_MADV_DONTNEED: 831 case I915_MADV_WILLNEED: 832 break; 833 default: 834 return (-EINVAL); 835 } 836 837 ret = i915_mutex_lock_interruptible(dev); 838 if (ret != 0) 839 return (ret); 840 841 obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle)); 842 if (&obj->base == NULL) { 843 ret = -ENOENT; 844 goto unlock; 845 } 846 847 if (obj->pin_count != 0) { 848 ret = -EINVAL; 849 goto out; 850 } 851 852 if (obj->madv != I915_MADV_PURGED_INTERNAL) 853 obj->madv = args->madv; 854 if (i915_gem_object_is_purgeable(obj) && obj->gtt_space == NULL) 855 i915_gem_object_truncate(obj); 856 args->retained = obj->madv != I915_MADV_PURGED_INTERNAL; 857 858out: 859 drm_gem_object_unreference(&obj->base); 860unlock: 861 DRM_UNLOCK(dev); 862 return (ret); 863} 864 865void 866i915_gem_cleanup_ringbuffer(struct drm_device *dev) 867{ 868 drm_i915_private_t *dev_priv; 869 struct intel_ring_buffer *ring; 870 int i; 871 872 dev_priv = dev->dev_private; 873 for_each_ring(ring, dev_priv, i) 874 intel_cleanup_ring_buffer(ring); 875} 876 877int 878i915_gem_entervt_ioctl(struct drm_device *dev, void *data, 879 struct drm_file *file_priv) 880{ 881 drm_i915_private_t *dev_priv; 882 int ret; 883 884 if (drm_core_check_feature(dev, DRIVER_MODESET)) 885 return (0); 886 dev_priv = dev->dev_private; 887 if (atomic_load_acq_int(&dev_priv->mm.wedged) != 0) { 888 DRM_ERROR("Reenabling wedged hardware, good luck\n"); 889 atomic_store_rel_int(&dev_priv->mm.wedged, 0); 890 } 891 892 DRM_LOCK(dev); 893 dev_priv->mm.suspended = 0; 894 895 ret = i915_gem_init_hw(dev); 896 if (ret != 0) { 897 DRM_UNLOCK(dev); 898 return (ret); 899 } 900 901 KASSERT(list_empty(&dev_priv->mm.active_list), ("active list")); 902 KASSERT(list_empty(&dev_priv->mm.flushing_list), ("flushing list")); 903 KASSERT(list_empty(&dev_priv->mm.inactive_list), ("inactive list")); 904 DRM_UNLOCK(dev); 905 906 ret = drm_irq_install(dev); 907 if (ret) 908 goto cleanup_ringbuffer; 909 910 return (0); 911 912cleanup_ringbuffer: 913 DRM_LOCK(dev); 914 i915_gem_cleanup_ringbuffer(dev); 915 dev_priv->mm.suspended = 1; 916 DRM_UNLOCK(dev); 917 918 return (ret); 919} 920 921int 922i915_gem_leavevt_ioctl(struct drm_device *dev, void *data, 923 struct drm_file *file_priv) 924{ 925 926 if (drm_core_check_feature(dev, DRIVER_MODESET)) 927 return 0; 928 929 drm_irq_uninstall(dev); 930 return (i915_gem_idle(dev)); 931} 932 933int 934i915_gem_create(struct drm_file *file, struct drm_device *dev, uint64_t size, 935 uint32_t *handle_p) 936{ 937 struct drm_i915_gem_object *obj; 938 uint32_t handle; 939 int ret; 940 941 size = roundup(size, PAGE_SIZE); 942 if (size == 0) 943 return (-EINVAL); 944 945 obj = i915_gem_alloc_object(dev, size); 946 if (obj == NULL) 947 return (-ENOMEM); 948 949 ret = drm_gem_handle_create(file, &obj->base, &handle); 950 if (ret != 0) { 951 drm_gem_object_release(&obj->base); 952 i915_gem_info_remove_obj(dev->dev_private, obj->base.size); 953 free(obj, DRM_I915_GEM); 954 return (ret); 955 } 956 957 /* drop reference from allocate - handle holds it now */ 958 drm_gem_object_unreference(&obj->base); 959 CTR2(KTR_DRM, "object_create %p %x", obj, size); 960 *handle_p = handle; 961 return (0); 962} 963 964int 965i915_gem_dumb_create(struct drm_file *file, struct drm_device *dev, 966 struct drm_mode_create_dumb *args) 967{ 968 969 /* have to work out size/pitch and return them */ 970 args->pitch = roundup2(args->width * ((args->bpp + 7) / 8), 64); 971 args->size = args->pitch * args->height; 972 return (i915_gem_create(file, dev, args->size, &args->handle)); 973} 974 975int 976i915_gem_dumb_destroy(struct drm_file *file, struct drm_device *dev, 977 uint32_t handle) 978{ 979 980 return (drm_gem_handle_delete(file, handle)); 981} 982 983int 984i915_gem_create_ioctl(struct drm_device *dev, void *data, 985 struct drm_file *file) 986{ 987 struct drm_i915_gem_create *args = data; 988 989 return (i915_gem_create(file, dev, args->size, &args->handle)); 990} 991 992#define __user 993#define __force 994#define __iomem 995#define to_user_ptr(x) ((void *)(uintptr_t)(x)) 996#define offset_in_page(x) ((x) & PAGE_MASK) 997#define page_to_phys(x) VM_PAGE_TO_PHYS(x) 998static inline int 999__copy_to_user_inatomic(void __user *to, const void *from, unsigned n) 1000{ 1001 return (copyout_nofault(from, to, n) != 0 ? n : 0); 1002} 1003static inline unsigned long 1004__copy_from_user_inatomic_nocache(void *to, const void __user *from, 1005 unsigned long n) 1006{ 1007 1008 /* 1009 * XXXKIB. Equivalent Linux function is implemented using 1010 * MOVNTI for aligned moves. For unaligned head and tail, 1011 * normal move is performed. As such, it is not incorrect, if 1012 * only somewhat slower, to use normal copyin. All uses 1013 * except shmem_pwrite_fast() have the destination mapped WC. 1014 */ 1015 return ((copyin_nofault(__DECONST(void *, from), to, n) != 0 ? n : 0)); 1016} 1017static inline int 1018fault_in_multipages_readable(const char __user *uaddr, int size) 1019{ 1020 char c; 1021 int ret = 0; 1022 const char __user *end = uaddr + size - 1; 1023 1024 if (unlikely(size == 0)) 1025 return ret; 1026 1027 while (uaddr <= end) { 1028 ret = -copyin(uaddr, &c, 1); 1029 if (ret != 0) 1030 return -EFAULT; 1031 uaddr += PAGE_SIZE; 1032 } 1033 1034 /* Check whether the range spilled into the next page. */ 1035 if (((unsigned long)uaddr & ~PAGE_MASK) == 1036 ((unsigned long)end & ~PAGE_MASK)) { 1037 ret = -copyin(end, &c, 1); 1038 } 1039 1040 return ret; 1041} 1042 1043static inline int 1044fault_in_multipages_writeable(char __user *uaddr, int size) 1045{ 1046 int ret = 0; 1047 char __user *end = uaddr + size - 1; 1048 1049 if (unlikely(size == 0)) 1050 return ret; 1051 1052 /* 1053 * Writing zeroes into userspace here is OK, because we know that if 1054 * the zero gets there, we'll be overwriting it. 1055 */ 1056 while (uaddr <= end) { 1057 ret = subyte(uaddr, 0); 1058 if (ret != 0) 1059 return -EFAULT; 1060 uaddr += PAGE_SIZE; 1061 } 1062 1063 /* Check whether the range spilled into the next page. */ 1064 if (((unsigned long)uaddr & ~PAGE_MASK) == 1065 ((unsigned long)end & ~PAGE_MASK)) 1066 ret = subyte(end, 0); 1067 1068 return ret; 1069} 1070 1071static inline int 1072__copy_to_user_swizzled(char __user *cpu_vaddr, 1073 const char *gpu_vaddr, int gpu_offset, 1074 int length) 1075{ 1076 int ret, cpu_offset = 0; 1077 1078 while (length > 0) { 1079 int cacheline_end = roundup2(gpu_offset + 1, 64); 1080 int this_length = min(cacheline_end - gpu_offset, length); 1081 int swizzled_gpu_offset = gpu_offset ^ 64; 1082 1083 ret = __copy_to_user(cpu_vaddr + cpu_offset, 1084 gpu_vaddr + swizzled_gpu_offset, 1085 this_length); 1086 if (ret) 1087 return ret + length; 1088 1089 cpu_offset += this_length; 1090 gpu_offset += this_length; 1091 length -= this_length; 1092 } 1093 1094 return 0; 1095} 1096 1097static inline int 1098__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset, 1099 const char __user *cpu_vaddr, 1100 int length) 1101{ 1102 int ret, cpu_offset = 0; 1103 1104 while (length > 0) { 1105 int cacheline_end = roundup2(gpu_offset + 1, 64); 1106 int this_length = min(cacheline_end - gpu_offset, length); 1107 int swizzled_gpu_offset = gpu_offset ^ 64; 1108 1109 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset, 1110 cpu_vaddr + cpu_offset, 1111 this_length); 1112 if (ret) 1113 return ret + length; 1114 1115 cpu_offset += this_length; 1116 gpu_offset += this_length; 1117 length -= this_length; 1118 } 1119 1120 return 0; 1121} 1122 1123static int 1124i915_gem_phys_pwrite(struct drm_device *dev, 1125 struct drm_i915_gem_object *obj, 1126 struct drm_i915_gem_pwrite *args, 1127 struct drm_file *file_priv) 1128{ 1129 void *vaddr = (char *)obj->phys_obj->handle->vaddr + args->offset; 1130 char __user *user_data = to_user_ptr(args->data_ptr); 1131 1132 if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) { 1133 unsigned long unwritten; 1134 1135 /* The physical object once assigned is fixed for the lifetime 1136 * of the obj, so we can safely drop the lock and continue 1137 * to access vaddr. 1138 */ 1139 DRM_UNLOCK(dev); 1140 unwritten = copy_from_user(vaddr, user_data, args->size); 1141 DRM_LOCK(dev); 1142 if (unwritten) 1143 return -EFAULT; 1144 } 1145 1146 i915_gem_chipset_flush(dev); 1147 return 0; 1148} 1149 1150/* Per-page copy function for the shmem pread fastpath. 1151 * Flushes invalid cachelines before reading the target if 1152 * needs_clflush is set. */ 1153static int 1154shmem_pread_fast(vm_page_t page, int shmem_page_offset, int page_length, 1155 char __user *user_data, 1156 bool page_do_bit17_swizzling, bool needs_clflush) 1157{ 1158 char *vaddr; 1159 struct sf_buf *sf; 1160 int ret; 1161 1162 if (unlikely(page_do_bit17_swizzling)) 1163 return -EINVAL; 1164 1165 sched_pin(); 1166 sf = sf_buf_alloc(page, SFB_NOWAIT | SFB_CPUPRIVATE); 1167 if (sf == NULL) { 1168 sched_unpin(); 1169 return (-EFAULT); 1170 } 1171 vaddr = (char *)sf_buf_kva(sf); 1172 if (needs_clflush) 1173 drm_clflush_virt_range(vaddr + shmem_page_offset, 1174 page_length); 1175 ret = __copy_to_user_inatomic(user_data, 1176 vaddr + shmem_page_offset, 1177 page_length); 1178 sf_buf_free(sf); 1179 sched_unpin(); 1180 1181 return ret ? -EFAULT : 0; 1182} 1183 1184static void 1185shmem_clflush_swizzled_range(char *addr, unsigned long length, 1186 bool swizzled) 1187{ 1188 if (unlikely(swizzled)) { 1189 unsigned long start = (unsigned long) addr; 1190 unsigned long end = (unsigned long) addr + length; 1191 1192 /* For swizzling simply ensure that we always flush both 1193 * channels. Lame, but simple and it works. Swizzled 1194 * pwrite/pread is far from a hotpath - current userspace 1195 * doesn't use it at all. */ 1196 start = rounddown2(start, 128); 1197 end = roundup2(end, 128); 1198 1199 drm_clflush_virt_range((void *)start, end - start); 1200 } else { 1201 drm_clflush_virt_range(addr, length); 1202 } 1203 1204} 1205 1206/* Only difference to the fast-path function is that this can handle bit17 1207 * and uses non-atomic copy and kmap functions. */ 1208static int 1209shmem_pread_slow(vm_page_t page, int shmem_page_offset, int page_length, 1210 char __user *user_data, 1211 bool page_do_bit17_swizzling, bool needs_clflush) 1212{ 1213 char *vaddr; 1214 struct sf_buf *sf; 1215 int ret; 1216 1217 sf = sf_buf_alloc(page, 0); 1218 vaddr = (char *)sf_buf_kva(sf); 1219 if (needs_clflush) 1220 shmem_clflush_swizzled_range(vaddr + shmem_page_offset, 1221 page_length, 1222 page_do_bit17_swizzling); 1223 1224 if (page_do_bit17_swizzling) 1225 ret = __copy_to_user_swizzled(user_data, 1226 vaddr, shmem_page_offset, 1227 page_length); 1228 else 1229 ret = __copy_to_user(user_data, 1230 vaddr + shmem_page_offset, 1231 page_length); 1232 sf_buf_free(sf); 1233 1234 return ret ? - EFAULT : 0; 1235} 1236 1237static int 1238i915_gem_shmem_pread(struct drm_device *dev, 1239 struct drm_i915_gem_object *obj, 1240 struct drm_i915_gem_pread *args, 1241 struct drm_file *file) 1242{ 1243 char __user *user_data; 1244 ssize_t remain, sremain; 1245 off_t offset, soffset; 1246 int shmem_page_offset, page_length, ret = 0; 1247 int obj_do_bit17_swizzling, page_do_bit17_swizzling; 1248 int prefaulted = 0; 1249 int needs_clflush = 0; 1250 1251 user_data = to_user_ptr(args->data_ptr); 1252 sremain = remain = args->size; 1253 1254 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 1255 1256 if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) { 1257 /* If we're not in the cpu read domain, set ourself into the gtt 1258 * read domain and manually flush cachelines (if required). This 1259 * optimizes for the case when the gpu will dirty the data 1260 * anyway again before the next pread happens. */ 1261 needs_clflush = !cpu_cache_is_coherent(dev, obj->cache_level); 1262 ret = i915_gem_object_set_to_gtt_domain(obj, false); 1263 if (ret) 1264 return ret; 1265 } 1266 1267 soffset = offset = args->offset; 1268 ret = i915_gem_object_get_pages_range(obj, soffset, soffset + sremain); 1269 if (ret) 1270 return ret; 1271 1272 i915_gem_object_pin_pages(obj); 1273 1274 VM_OBJECT_WLOCK(obj->base.vm_obj); 1275 for (vm_page_t page = vm_page_find_least(obj->base.vm_obj, 1276 OFF_TO_IDX(offset));; page = vm_page_next(page)) { 1277 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 1278 1279 if (remain <= 0) 1280 break; 1281 1282 /* Operation in this page 1283 * 1284 * shmem_page_offset = offset within page in shmem file 1285 * page_length = bytes to copy for this page 1286 */ 1287 shmem_page_offset = offset_in_page(offset); 1288 page_length = remain; 1289 if ((shmem_page_offset + page_length) > PAGE_SIZE) 1290 page_length = PAGE_SIZE - shmem_page_offset; 1291 1292 page_do_bit17_swizzling = obj_do_bit17_swizzling && 1293 (page_to_phys(page) & (1 << 17)) != 0; 1294 1295 ret = shmem_pread_fast(page, shmem_page_offset, page_length, 1296 user_data, page_do_bit17_swizzling, 1297 needs_clflush); 1298 if (ret == 0) 1299 goto next_page; 1300 1301 DRM_UNLOCK(dev); 1302 1303 if (likely(!i915_prefault_disable) && !prefaulted) { 1304 ret = fault_in_multipages_writeable(user_data, remain); 1305 /* Userspace is tricking us, but we've already clobbered 1306 * its pages with the prefault and promised to write the 1307 * data up to the first fault. Hence ignore any errors 1308 * and just continue. */ 1309 (void)ret; 1310 prefaulted = 1; 1311 } 1312 1313 ret = shmem_pread_slow(page, shmem_page_offset, page_length, 1314 user_data, page_do_bit17_swizzling, 1315 needs_clflush); 1316 1317 DRM_LOCK(dev); 1318 1319next_page: 1320 vm_page_reference(page); 1321 1322 if (ret) 1323 goto out; 1324 1325 remain -= page_length; 1326 user_data += page_length; 1327 offset += page_length; 1328 VM_OBJECT_WLOCK(obj->base.vm_obj); 1329 } 1330 1331out: 1332 i915_gem_object_unpin_pages(obj); 1333 i915_gem_object_put_pages_range(obj, soffset, soffset + sremain); 1334 1335 return ret; 1336} 1337 1338/** 1339 * Reads data from the object referenced by handle. 1340 * 1341 * On error, the contents of *data are undefined. 1342 */ 1343int 1344i915_gem_pread_ioctl(struct drm_device *dev, void *data, 1345 struct drm_file *file) 1346{ 1347 struct drm_i915_gem_pread *args = data; 1348 struct drm_i915_gem_object *obj; 1349 int ret = 0; 1350 1351 if (args->size == 0) 1352 return 0; 1353 1354 if (!useracc(to_user_ptr(args->data_ptr), args->size, VM_PROT_WRITE)) 1355 return -EFAULT; 1356 1357 ret = i915_mutex_lock_interruptible(dev); 1358 if (ret) 1359 return ret; 1360 1361 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1362 if (&obj->base == NULL) { 1363 ret = -ENOENT; 1364 goto unlock; 1365 } 1366 1367 /* Bounds check source. */ 1368 if (args->offset > obj->base.size || 1369 args->size > obj->base.size - args->offset) { 1370 ret = -EINVAL; 1371 goto out; 1372 } 1373 1374#if 1 1375 KIB_NOTYET(); 1376#else 1377 /* prime objects have no backing filp to GEM pread/pwrite 1378 * pages from. 1379 */ 1380 if (!obj->base.filp) { 1381 ret = -EINVAL; 1382 goto out; 1383 } 1384#endif 1385 1386 CTR3(KTR_DRM, "pread %p %jx %jx", obj, args->offset, args->size); 1387 1388 ret = i915_gem_shmem_pread(dev, obj, args, file); 1389 1390out: 1391 drm_gem_object_unreference(&obj->base); 1392unlock: 1393 DRM_UNLOCK(dev); 1394 return ret; 1395} 1396 1397/* This is the fast write path which cannot handle 1398 * page faults in the source data 1399 */ 1400 1401static inline int 1402fast_user_write(struct drm_device *dev, 1403 off_t page_base, int page_offset, 1404 char __user *user_data, 1405 int length) 1406{ 1407 void __iomem *vaddr_atomic; 1408 void *vaddr; 1409 unsigned long unwritten; 1410 1411 vaddr_atomic = pmap_mapdev_attr(dev->agp->base + page_base, 1412 length, PAT_WRITE_COMBINING); 1413 /* We can use the cpu mem copy function because this is X86. */ 1414 vaddr = (char *)vaddr_atomic + page_offset; 1415 unwritten = __copy_from_user_inatomic_nocache(vaddr, 1416 user_data, length); 1417 pmap_unmapdev((vm_offset_t)vaddr_atomic, length); 1418 return unwritten; 1419} 1420 1421/** 1422 * This is the fast pwrite path, where we copy the data directly from the 1423 * user into the GTT, uncached. 1424 */ 1425static int 1426i915_gem_gtt_pwrite_fast(struct drm_device *dev, 1427 struct drm_i915_gem_object *obj, 1428 struct drm_i915_gem_pwrite *args, 1429 struct drm_file *file) 1430{ 1431 ssize_t remain; 1432 off_t offset, page_base; 1433 char __user *user_data; 1434 int page_offset, page_length, ret; 1435 1436 ret = i915_gem_object_pin(obj, 0, true); 1437 /* XXXKIB ret = i915_gem_obj_ggtt_pin(obj, 0, true, true); */ 1438 if (ret != 0) 1439 goto out; 1440 1441 ret = i915_gem_object_set_to_gtt_domain(obj, true); 1442 if (ret) 1443 goto out_unpin; 1444 1445 ret = i915_gem_object_put_fence(obj); 1446 if (ret) 1447 goto out_unpin; 1448 1449 user_data = to_user_ptr(args->data_ptr); 1450 remain = args->size; 1451 1452 offset = obj->gtt_offset + args->offset; 1453 1454 while (remain > 0) { 1455 /* Operation in this page 1456 * 1457 * page_base = page offset within aperture 1458 * page_offset = offset within page 1459 * page_length = bytes to copy for this page 1460 */ 1461 page_base = offset & ~PAGE_MASK; 1462 page_offset = offset_in_page(offset); 1463 page_length = remain; 1464 if ((page_offset + remain) > PAGE_SIZE) 1465 page_length = PAGE_SIZE - page_offset; 1466 1467 /* If we get a fault while copying data, then (presumably) our 1468 * source page isn't available. Return the error and we'll 1469 * retry in the slow path. 1470 */ 1471 if (fast_user_write(dev, page_base, 1472 page_offset, user_data, page_length)) { 1473 ret = -EFAULT; 1474 goto out_unpin; 1475 } 1476 1477 remain -= page_length; 1478 user_data += page_length; 1479 offset += page_length; 1480 } 1481 1482out_unpin: 1483 i915_gem_object_unpin(obj); 1484out: 1485 return ret; 1486} 1487 1488/* Per-page copy function for the shmem pwrite fastpath. 1489 * Flushes invalid cachelines before writing to the target if 1490 * needs_clflush_before is set and flushes out any written cachelines after 1491 * writing if needs_clflush is set. */ 1492static int 1493shmem_pwrite_fast(vm_page_t page, int shmem_page_offset, int page_length, 1494 char __user *user_data, 1495 bool page_do_bit17_swizzling, 1496 bool needs_clflush_before, 1497 bool needs_clflush_after) 1498{ 1499 char *vaddr; 1500 struct sf_buf *sf; 1501 int ret; 1502 1503 if (unlikely(page_do_bit17_swizzling)) 1504 return -EINVAL; 1505 1506 sched_pin(); 1507 sf = sf_buf_alloc(page, SFB_NOWAIT | SFB_CPUPRIVATE); 1508 if (sf == NULL) { 1509 sched_unpin(); 1510 return (-EFAULT); 1511 } 1512 vaddr = (char *)sf_buf_kva(sf); 1513 if (needs_clflush_before) 1514 drm_clflush_virt_range(vaddr + shmem_page_offset, 1515 page_length); 1516 ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset, 1517 user_data, 1518 page_length); 1519 if (needs_clflush_after) 1520 drm_clflush_virt_range(vaddr + shmem_page_offset, 1521 page_length); 1522 sf_buf_free(sf); 1523 sched_unpin(); 1524 1525 return ret ? -EFAULT : 0; 1526} 1527 1528/* Only difference to the fast-path function is that this can handle bit17 1529 * and uses non-atomic copy and kmap functions. */ 1530static int 1531shmem_pwrite_slow(vm_page_t page, int shmem_page_offset, int page_length, 1532 char __user *user_data, 1533 bool page_do_bit17_swizzling, 1534 bool needs_clflush_before, 1535 bool needs_clflush_after) 1536{ 1537 char *vaddr; 1538 struct sf_buf *sf; 1539 int ret; 1540 1541 sf = sf_buf_alloc(page, 0); 1542 vaddr = (char *)sf_buf_kva(sf); 1543 if (unlikely(needs_clflush_before || page_do_bit17_swizzling)) 1544 shmem_clflush_swizzled_range(vaddr + shmem_page_offset, 1545 page_length, 1546 page_do_bit17_swizzling); 1547 if (page_do_bit17_swizzling) 1548 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset, 1549 user_data, 1550 page_length); 1551 else 1552 ret = __copy_from_user(vaddr + shmem_page_offset, 1553 user_data, 1554 page_length); 1555 if (needs_clflush_after) 1556 shmem_clflush_swizzled_range(vaddr + shmem_page_offset, 1557 page_length, 1558 page_do_bit17_swizzling); 1559 sf_buf_free(sf); 1560 1561 return ret ? -EFAULT : 0; 1562} 1563 1564static int 1565i915_gem_shmem_pwrite(struct drm_device *dev, 1566 struct drm_i915_gem_object *obj, 1567 struct drm_i915_gem_pwrite *args, 1568 struct drm_file *file) 1569{ 1570 ssize_t remain, sremain; 1571 off_t offset, soffset; 1572 char __user *user_data; 1573 int shmem_page_offset, page_length, ret = 0; 1574 int obj_do_bit17_swizzling, page_do_bit17_swizzling; 1575 int hit_slowpath = 0; 1576 int needs_clflush_after = 0; 1577 int needs_clflush_before = 0; 1578 1579 user_data = to_user_ptr(args->data_ptr); 1580 sremain = remain = args->size; 1581 1582 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 1583 1584 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) { 1585 /* If we're not in the cpu write domain, set ourself into the gtt 1586 * write domain and manually flush cachelines (if required). This 1587 * optimizes for the case when the gpu will use the data 1588 * right away and we therefore have to clflush anyway. */ 1589 needs_clflush_after = cpu_write_needs_clflush(obj); 1590 ret = i915_gem_object_set_to_gtt_domain(obj, true); 1591 if (ret) 1592 return ret; 1593 } 1594 /* Same trick applies to invalidate partially written cachelines read 1595 * before writing. */ 1596 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) 1597 needs_clflush_before = 1598 !cpu_cache_is_coherent(dev, obj->cache_level); 1599 1600 soffset = offset = args->offset; 1601 ret = i915_gem_object_get_pages_range(obj, soffset, soffset + sremain); 1602 if (ret) 1603 return ret; 1604 1605 i915_gem_object_pin_pages(obj); 1606 1607 obj->dirty = 1; 1608 1609 VM_OBJECT_WLOCK(obj->base.vm_obj); 1610 for (vm_page_t page = vm_page_find_least(obj->base.vm_obj, 1611 OFF_TO_IDX(offset));; page = vm_page_next(page)) { 1612 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 1613 int partial_cacheline_write; 1614 1615 if (remain <= 0) 1616 break; 1617 1618 /* Operation in this page 1619 * 1620 * shmem_page_offset = offset within page in shmem file 1621 * page_length = bytes to copy for this page 1622 */ 1623 shmem_page_offset = offset_in_page(offset); 1624 1625 page_length = remain; 1626 if ((shmem_page_offset + page_length) > PAGE_SIZE) 1627 page_length = PAGE_SIZE - shmem_page_offset; 1628 1629 /* If we don't overwrite a cacheline completely we need to be 1630 * careful to have up-to-date data by first clflushing. Don't 1631 * overcomplicate things and flush the entire patch. */ 1632 partial_cacheline_write = needs_clflush_before && 1633 ((shmem_page_offset | page_length) 1634 & (cpu_clflush_line_size - 1)); 1635 1636 page_do_bit17_swizzling = obj_do_bit17_swizzling && 1637 (page_to_phys(page) & (1 << 17)) != 0; 1638 1639 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length, 1640 user_data, page_do_bit17_swizzling, 1641 partial_cacheline_write, 1642 needs_clflush_after); 1643 if (ret == 0) 1644 goto next_page; 1645 1646 hit_slowpath = 1; 1647 DRM_UNLOCK(dev); 1648 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length, 1649 user_data, page_do_bit17_swizzling, 1650 partial_cacheline_write, 1651 needs_clflush_after); 1652 1653 DRM_LOCK(dev); 1654 1655next_page: 1656 vm_page_dirty(page); 1657 vm_page_reference(page); 1658 1659 if (ret) 1660 goto out; 1661 1662 remain -= page_length; 1663 user_data += page_length; 1664 offset += page_length; 1665 VM_OBJECT_WLOCK(obj->base.vm_obj); 1666 } 1667 1668out: 1669 i915_gem_object_unpin_pages(obj); 1670 i915_gem_object_put_pages_range(obj, soffset, soffset + sremain); 1671 1672 if (hit_slowpath) { 1673 /* 1674 * Fixup: Flush cpu caches in case we didn't flush the dirty 1675 * cachelines in-line while writing and the object moved 1676 * out of the cpu write domain while we've dropped the lock. 1677 */ 1678 if (!needs_clflush_after && 1679 obj->base.write_domain != I915_GEM_DOMAIN_CPU) { 1680 i915_gem_clflush_object(obj); 1681 i915_gem_chipset_flush(dev); 1682 } 1683 } 1684 1685 if (needs_clflush_after) 1686 i915_gem_chipset_flush(dev); 1687 1688 return ret; 1689} 1690 1691/** 1692 * Writes data to the object referenced by handle. 1693 * 1694 * On error, the contents of the buffer that were to be modified are undefined. 1695 */ 1696int 1697i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, 1698 struct drm_file *file) 1699{ 1700 struct drm_i915_gem_pwrite *args = data; 1701 struct drm_i915_gem_object *obj; 1702 int ret; 1703 1704 if (args->size == 0) 1705 return 0; 1706 1707 if (!useracc(to_user_ptr(args->data_ptr), args->size, VM_PROT_READ)) 1708 return -EFAULT; 1709 1710 if (likely(!i915_prefault_disable)) { 1711 ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr), 1712 args->size); 1713 if (ret) 1714 return -EFAULT; 1715 } 1716 1717 ret = i915_mutex_lock_interruptible(dev); 1718 if (ret) 1719 return ret; 1720 1721 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1722 if (&obj->base == NULL) { 1723 ret = -ENOENT; 1724 goto unlock; 1725 } 1726 1727 /* Bounds check destination. */ 1728 if (args->offset > obj->base.size || 1729 args->size > obj->base.size - args->offset) { 1730 ret = -EINVAL; 1731 goto out; 1732 } 1733 1734#if 1 1735 KIB_NOTYET(); 1736#else 1737 /* prime objects have no backing filp to GEM pread/pwrite 1738 * pages from. 1739 */ 1740 if (!obj->base.filp) { 1741 ret = -EINVAL; 1742 goto out; 1743 } 1744#endif 1745 1746 CTR3(KTR_DRM, "pwrite %p %jx %jx", obj, args->offset, args->size); 1747 1748 ret = -EFAULT; 1749 /* We can only do the GTT pwrite on untiled buffers, as otherwise 1750 * it would end up going through the fenced access, and we'll get 1751 * different detiling behavior between reading and writing. 1752 * pread/pwrite currently are reading and writing from the CPU 1753 * perspective, requiring manual detiling by the client. 1754 */ 1755 if (obj->phys_obj) { 1756 ret = i915_gem_phys_pwrite(dev, obj, args, file); 1757 goto out; 1758 } 1759 1760 if (obj->tiling_mode == I915_TILING_NONE && 1761 obj->base.write_domain != I915_GEM_DOMAIN_CPU && 1762 cpu_write_needs_clflush(obj)) { 1763 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file); 1764 /* Note that the gtt paths might fail with non-page-backed user 1765 * pointers (e.g. gtt mappings when moving data between 1766 * textures). Fallback to the shmem path in that case. */ 1767 } 1768 1769 if (ret == -EFAULT || ret == -ENOSPC) 1770 ret = i915_gem_shmem_pwrite(dev, obj, args, file); 1771 1772out: 1773 drm_gem_object_unreference(&obj->base); 1774unlock: 1775 DRM_UNLOCK(dev); 1776 return ret; 1777} 1778#undef __user 1779#undef __force 1780#undef __iomem 1781#undef to_user_ptr 1782#undef offset_in_page 1783#undef page_to_phys 1784 1785int 1786i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, 1787 struct drm_file *file) 1788{ 1789 struct drm_i915_gem_set_domain *args; 1790 struct drm_i915_gem_object *obj; 1791 uint32_t read_domains; 1792 uint32_t write_domain; 1793 int ret; 1794 1795 args = data; 1796 read_domains = args->read_domains; 1797 write_domain = args->write_domain; 1798 1799 if ((write_domain & I915_GEM_GPU_DOMAINS) != 0 || 1800 (read_domains & I915_GEM_GPU_DOMAINS) != 0 || 1801 (write_domain != 0 && read_domains != write_domain)) 1802 return (-EINVAL); 1803 1804 ret = i915_mutex_lock_interruptible(dev); 1805 if (ret != 0) 1806 return (ret); 1807 1808 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1809 if (&obj->base == NULL) { 1810 ret = -ENOENT; 1811 goto unlock; 1812 } 1813 1814 if ((read_domains & I915_GEM_DOMAIN_GTT) != 0) { 1815 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); 1816 if (ret == -EINVAL) 1817 ret = 0; 1818 } else 1819 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); 1820 1821 drm_gem_object_unreference(&obj->base); 1822unlock: 1823 DRM_UNLOCK(dev); 1824 return (ret); 1825} 1826 1827int 1828i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, 1829 struct drm_file *file) 1830{ 1831 struct drm_i915_gem_sw_finish *args; 1832 struct drm_i915_gem_object *obj; 1833 int ret; 1834 1835 args = data; 1836 1837 ret = i915_mutex_lock_interruptible(dev); 1838 if (ret != 0) 1839 return (ret); 1840 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1841 if (&obj->base == NULL) { 1842 ret = -ENOENT; 1843 goto unlock; 1844 } 1845 if (obj->pin_count != 0) 1846 i915_gem_object_flush_cpu_write_domain(obj); 1847 drm_gem_object_unreference(&obj->base); 1848unlock: 1849 DRM_UNLOCK(dev); 1850 return (ret); 1851} 1852 1853int 1854i915_gem_mmap_ioctl(struct drm_device *dev, void *data, 1855 struct drm_file *file) 1856{ 1857 struct drm_i915_gem_mmap *args; 1858 struct drm_gem_object *obj; 1859 struct proc *p; 1860 vm_map_t map; 1861 vm_offset_t addr; 1862 vm_size_t size; 1863 int error, rv; 1864 1865 args = data; 1866 1867 obj = drm_gem_object_lookup(dev, file, args->handle); 1868 if (obj == NULL) 1869 return (-ENOENT); 1870 error = 0; 1871 if (args->size == 0) 1872 goto out; 1873 p = curproc; 1874 map = &p->p_vmspace->vm_map; 1875 size = round_page(args->size); 1876 PROC_LOCK(p); 1877 if (map->size + size > lim_cur(p, RLIMIT_VMEM)) { 1878 PROC_UNLOCK(p); 1879 error = -ENOMEM; 1880 goto out; 1881 } 1882 PROC_UNLOCK(p); 1883 1884 addr = 0; 1885 vm_object_reference(obj->vm_obj); 1886 rv = vm_map_find(map, obj->vm_obj, args->offset, &addr, args->size, 0, 1887 VMFS_OPTIMAL_SPACE, VM_PROT_READ | VM_PROT_WRITE, 1888 VM_PROT_READ | VM_PROT_WRITE, MAP_INHERIT_SHARE); 1889 if (rv != KERN_SUCCESS) { 1890 vm_object_deallocate(obj->vm_obj); 1891 error = -vm_mmap_to_errno(rv); 1892 } else { 1893 args->addr_ptr = (uint64_t)addr; 1894 } 1895out: 1896 drm_gem_object_unreference(obj); 1897 return (error); 1898} 1899 1900static int 1901i915_gem_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 1902 vm_ooffset_t foff, struct ucred *cred, u_short *color) 1903{ 1904 1905 *color = 0; /* XXXKIB */ 1906 return (0); 1907} 1908 1909int i915_intr_pf; 1910 1911static int 1912i915_gem_pager_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot, 1913 vm_page_t *mres) 1914{ 1915 struct drm_gem_object *gem_obj; 1916 struct drm_i915_gem_object *obj; 1917 struct drm_device *dev; 1918 drm_i915_private_t *dev_priv; 1919 vm_page_t m, oldm; 1920 int cause, ret; 1921 bool write; 1922 1923 gem_obj = vm_obj->handle; 1924 obj = to_intel_bo(gem_obj); 1925 dev = obj->base.dev; 1926 dev_priv = dev->dev_private; 1927#if 0 1928 write = (prot & VM_PROT_WRITE) != 0; 1929#else 1930 write = true; 1931#endif 1932 vm_object_pip_add(vm_obj, 1); 1933 1934 /* 1935 * Remove the placeholder page inserted by vm_fault() from the 1936 * object before dropping the object lock. If 1937 * i915_gem_release_mmap() is active in parallel on this gem 1938 * object, then it owns the drm device sx and might find the 1939 * placeholder already. Then, since the page is busy, 1940 * i915_gem_release_mmap() sleeps waiting for the busy state 1941 * of the page cleared. We will be not able to acquire drm 1942 * device lock until i915_gem_release_mmap() is able to make a 1943 * progress. 1944 */ 1945 if (*mres != NULL) { 1946 oldm = *mres; 1947 vm_page_lock(oldm); 1948 vm_page_remove(oldm); 1949 vm_page_unlock(oldm); 1950 *mres = NULL; 1951 } else 1952 oldm = NULL; 1953 VM_OBJECT_WUNLOCK(vm_obj); 1954retry: 1955 cause = ret = 0; 1956 m = NULL; 1957 1958 if (i915_intr_pf) { 1959 ret = i915_mutex_lock_interruptible(dev); 1960 if (ret != 0) { 1961 cause = 10; 1962 goto out; 1963 } 1964 } else 1965 DRM_LOCK(dev); 1966 1967 /* 1968 * Since the object lock was dropped, other thread might have 1969 * faulted on the same GTT address and instantiated the 1970 * mapping for the page. Recheck. 1971 */ 1972 VM_OBJECT_WLOCK(vm_obj); 1973 m = vm_page_lookup(vm_obj, OFF_TO_IDX(offset)); 1974 if (m != NULL) { 1975 if (vm_page_busied(m)) { 1976 DRM_UNLOCK(dev); 1977 vm_page_lock(m); 1978 VM_OBJECT_WUNLOCK(vm_obj); 1979 vm_page_busy_sleep(m, "915pee", false); 1980 goto retry; 1981 } 1982 goto have_page; 1983 } else 1984 VM_OBJECT_WUNLOCK(vm_obj); 1985 1986 /* Now bind it into the GTT if needed */ 1987 if (!obj->map_and_fenceable) { 1988 ret = i915_gem_object_unbind(obj); 1989 if (ret != 0) { 1990 cause = 20; 1991 goto unlock; 1992 } 1993 } 1994 if (!obj->gtt_space) { 1995 ret = i915_gem_object_bind_to_gtt(obj, 0, true); 1996 if (ret != 0) { 1997 cause = 30; 1998 goto unlock; 1999 } 2000 2001 ret = i915_gem_object_set_to_gtt_domain(obj, write); 2002 if (ret != 0) { 2003 cause = 40; 2004 goto unlock; 2005 } 2006 } 2007 2008 if (!obj->has_global_gtt_mapping) 2009 i915_gem_gtt_bind_object(obj, obj->cache_level); 2010 2011 ret = i915_gem_object_get_fence(obj); 2012 if (ret != 0) { 2013 cause = 50; 2014 goto unlock; 2015 } 2016 2017 if (i915_gem_object_is_inactive(obj)) 2018 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 2019 2020 obj->fault_mappable = true; 2021 VM_OBJECT_WLOCK(vm_obj); 2022 m = PHYS_TO_VM_PAGE(dev->agp->base + obj->gtt_offset + offset); 2023 KASSERT((m->flags & PG_FICTITIOUS) != 0, 2024 ("physical address %#jx not fictitious", 2025 (uintmax_t)(dev->agp->base + obj->gtt_offset + offset))); 2026 if (m == NULL) { 2027 VM_OBJECT_WUNLOCK(vm_obj); 2028 cause = 60; 2029 ret = -EFAULT; 2030 goto unlock; 2031 } 2032 KASSERT((m->flags & PG_FICTITIOUS) != 0, 2033 ("not fictitious %p", m)); 2034 KASSERT(m->wire_count == 1, ("wire_count not 1 %p", m)); 2035 2036 if (vm_page_busied(m)) { 2037 DRM_UNLOCK(dev); 2038 vm_page_lock(m); 2039 VM_OBJECT_WUNLOCK(vm_obj); 2040 vm_page_busy_sleep(m, "915pbs", false); 2041 goto retry; 2042 } 2043 if (vm_page_insert(m, vm_obj, OFF_TO_IDX(offset))) { 2044 DRM_UNLOCK(dev); 2045 VM_OBJECT_WUNLOCK(vm_obj); 2046 VM_WAIT; 2047 goto retry; 2048 } 2049 m->valid = VM_PAGE_BITS_ALL; 2050have_page: 2051 *mres = m; 2052 vm_page_xbusy(m); 2053 2054 CTR4(KTR_DRM, "fault %p %jx %x phys %x", gem_obj, offset, prot, 2055 m->phys_addr); 2056 DRM_UNLOCK(dev); 2057 if (oldm != NULL) { 2058 vm_page_lock(oldm); 2059 vm_page_free(oldm); 2060 vm_page_unlock(oldm); 2061 } 2062 vm_object_pip_wakeup(vm_obj); 2063 return (VM_PAGER_OK); 2064 2065unlock: 2066 DRM_UNLOCK(dev); 2067out: 2068 KASSERT(ret != 0, ("i915_gem_pager_fault: wrong return")); 2069 CTR5(KTR_DRM, "fault_fail %p %jx %x err %d %d", gem_obj, offset, prot, 2070 -ret, cause); 2071 if (ret == -EAGAIN || ret == -EIO || ret == -EINTR) { 2072 kern_yield(PRI_USER); 2073 goto retry; 2074 } 2075 VM_OBJECT_WLOCK(vm_obj); 2076 vm_object_pip_wakeup(vm_obj); 2077 return (VM_PAGER_ERROR); 2078} 2079 2080static void 2081i915_gem_pager_dtor(void *handle) 2082{ 2083 struct drm_gem_object *obj; 2084 struct drm_device *dev; 2085 2086 obj = handle; 2087 dev = obj->dev; 2088 2089 DRM_LOCK(dev); 2090 drm_gem_free_mmap_offset(obj); 2091 i915_gem_release_mmap(to_intel_bo(obj)); 2092 drm_gem_object_unreference(obj); 2093 DRM_UNLOCK(dev); 2094} 2095 2096struct cdev_pager_ops i915_gem_pager_ops = { 2097 .cdev_pg_fault = i915_gem_pager_fault, 2098 .cdev_pg_ctor = i915_gem_pager_ctor, 2099 .cdev_pg_dtor = i915_gem_pager_dtor 2100}; 2101 2102int 2103i915_gem_mmap_gtt(struct drm_file *file, struct drm_device *dev, 2104 uint32_t handle, uint64_t *offset) 2105{ 2106 struct drm_i915_private *dev_priv; 2107 struct drm_i915_gem_object *obj; 2108 int ret; 2109 2110 dev_priv = dev->dev_private; 2111 2112 ret = i915_mutex_lock_interruptible(dev); 2113 if (ret != 0) 2114 return (ret); 2115 2116 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); 2117 if (&obj->base == NULL) { 2118 ret = -ENOENT; 2119 goto unlock; 2120 } 2121 2122 if (obj->base.size > dev_priv->mm.gtt_mappable_end) { 2123 ret = -E2BIG; 2124 goto out; 2125 } 2126 2127 if (obj->madv != I915_MADV_WILLNEED) { 2128 DRM_ERROR("Attempting to mmap a purgeable buffer\n"); 2129 ret = -EINVAL; 2130 goto out; 2131 } 2132 2133 ret = drm_gem_create_mmap_offset(&obj->base); 2134 if (ret != 0) 2135 goto out; 2136 2137 *offset = DRM_GEM_MAPPING_OFF(obj->base.map_list.key) | 2138 DRM_GEM_MAPPING_KEY; 2139out: 2140 drm_gem_object_unreference(&obj->base); 2141unlock: 2142 DRM_UNLOCK(dev); 2143 return (ret); 2144} 2145 2146int 2147i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, 2148 struct drm_file *file) 2149{ 2150 struct drm_i915_private *dev_priv; 2151 struct drm_i915_gem_mmap_gtt *args; 2152 2153 dev_priv = dev->dev_private; 2154 args = data; 2155 2156 return (i915_gem_mmap_gtt(file, dev, args->handle, &args->offset)); 2157} 2158 2159struct drm_i915_gem_object * 2160i915_gem_alloc_object(struct drm_device *dev, size_t size) 2161{ 2162 struct drm_i915_private *dev_priv; 2163 struct drm_i915_gem_object *obj; 2164 2165 dev_priv = dev->dev_private; 2166 2167 obj = malloc(sizeof(*obj), DRM_I915_GEM, M_WAITOK | M_ZERO); 2168 2169 if (drm_gem_object_init(dev, &obj->base, size) != 0) { 2170 free(obj, DRM_I915_GEM); 2171 return (NULL); 2172 } 2173 2174 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 2175 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 2176 2177 if (HAS_LLC(dev)) 2178 obj->cache_level = I915_CACHE_LLC; 2179 else 2180 obj->cache_level = I915_CACHE_NONE; 2181 obj->base.driver_private = NULL; 2182 obj->fence_reg = I915_FENCE_REG_NONE; 2183 INIT_LIST_HEAD(&obj->mm_list); 2184 INIT_LIST_HEAD(&obj->gtt_list); 2185 INIT_LIST_HEAD(&obj->ring_list); 2186 INIT_LIST_HEAD(&obj->exec_list); 2187 INIT_LIST_HEAD(&obj->gpu_write_list); 2188 obj->madv = I915_MADV_WILLNEED; 2189 /* Avoid an unnecessary call to unbind on the first bind. */ 2190 obj->map_and_fenceable = true; 2191 2192 i915_gem_info_add_obj(dev_priv, size); 2193 2194 return (obj); 2195} 2196 2197void 2198i915_gem_clflush_object(struct drm_i915_gem_object *obj) 2199{ 2200 2201 /* If we don't have a page list set up, then we're not pinned 2202 * to GPU, and we can ignore the cache flush because it'll happen 2203 * again at bind time. 2204 */ 2205 if (obj->pages == NULL) 2206 return; 2207 2208 /* If the GPU is snooping the contents of the CPU cache, 2209 * we do not need to manually clear the CPU cache lines. However, 2210 * the caches are only snooped when the render cache is 2211 * flushed/invalidated. As we always have to emit invalidations 2212 * and flushes when moving into and out of the RENDER domain, correct 2213 * snooping behaviour occurs naturally as the result of our domain 2214 * tracking. 2215 */ 2216 if (obj->cache_level != I915_CACHE_NONE) 2217 return; 2218 2219 CTR1(KTR_DRM, "object_clflush %p", obj); 2220 drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE); 2221} 2222 2223static void 2224i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) 2225{ 2226 uint32_t old_write_domain; 2227 2228 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) 2229 return; 2230 2231 i915_gem_clflush_object(obj); 2232 intel_gtt_chipset_flush(); 2233 old_write_domain = obj->base.write_domain; 2234 obj->base.write_domain = 0; 2235 2236 CTR3(KTR_DRM, "object_change_domain flush_cpu_write %p %x %x", obj, 2237 obj->base.read_domains, old_write_domain); 2238} 2239 2240static int 2241i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj) 2242{ 2243 2244 if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0) 2245 return (0); 2246 return (i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain)); 2247} 2248 2249static void 2250i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj) 2251{ 2252 uint32_t old_write_domain; 2253 2254 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT) 2255 return; 2256 2257 wmb(); 2258 2259 old_write_domain = obj->base.write_domain; 2260 obj->base.write_domain = 0; 2261 2262 CTR3(KTR_DRM, "object_change_domain flush gtt_write %p %x %x", obj, 2263 obj->base.read_domains, old_write_domain); 2264} 2265 2266int 2267i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) 2268{ 2269 drm_i915_private_t *dev_priv = obj->base.dev->dev_private; 2270 uint32_t old_write_domain, old_read_domains; 2271 int ret; 2272 2273 if (obj->gtt_space == NULL) 2274 return (-EINVAL); 2275 2276 if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) 2277 return 0; 2278 2279 ret = i915_gem_object_flush_gpu_write_domain(obj); 2280 if (ret != 0) 2281 return (ret); 2282 2283 if (obj->pending_gpu_write || write) { 2284 ret = i915_gem_object_wait_rendering(obj); 2285 if (ret != 0) 2286 return (ret); 2287 } 2288 2289 i915_gem_object_flush_cpu_write_domain(obj); 2290 2291 old_write_domain = obj->base.write_domain; 2292 old_read_domains = obj->base.read_domains; 2293 2294 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0, 2295 ("In GTT write domain")); 2296 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 2297 if (write) { 2298 obj->base.read_domains = I915_GEM_DOMAIN_GTT; 2299 obj->base.write_domain = I915_GEM_DOMAIN_GTT; 2300 obj->dirty = 1; 2301 } 2302 2303 /* And bump the LRU for this access */ 2304 if (i915_gem_object_is_inactive(obj)) 2305 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 2306 2307 CTR3(KTR_DRM, "object_change_domain set_to_gtt %p %x %x", obj, 2308 old_read_domains, old_write_domain); 2309 return (0); 2310} 2311 2312int 2313i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, 2314 enum i915_cache_level cache_level) 2315{ 2316 struct drm_device *dev; 2317 drm_i915_private_t *dev_priv; 2318 int ret; 2319 2320 if (obj->cache_level == cache_level) 2321 return 0; 2322 2323 if (obj->pin_count) { 2324 DRM_DEBUG("can not change the cache level of pinned objects\n"); 2325 return (-EBUSY); 2326 } 2327 2328 dev = obj->base.dev; 2329 dev_priv = dev->dev_private; 2330 if (obj->gtt_space) { 2331 ret = i915_gem_object_finish_gpu(obj); 2332 if (ret != 0) 2333 return (ret); 2334 2335 i915_gem_object_finish_gtt(obj); 2336 2337 /* Before SandyBridge, you could not use tiling or fence 2338 * registers with snooped memory, so relinquish any fences 2339 * currently pointing to our region in the aperture. 2340 */ 2341 if (INTEL_INFO(obj->base.dev)->gen < 6) { 2342 ret = i915_gem_object_put_fence(obj); 2343 if (ret != 0) 2344 return (ret); 2345 } 2346 2347 if (obj->has_global_gtt_mapping) 2348 i915_gem_gtt_bind_object(obj, cache_level); 2349 if (obj->has_aliasing_ppgtt_mapping) 2350 i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt, 2351 obj, cache_level); 2352 } 2353 2354 if (cache_level == I915_CACHE_NONE) { 2355 u32 old_read_domains, old_write_domain; 2356 2357 /* If we're coming from LLC cached, then we haven't 2358 * actually been tracking whether the data is in the 2359 * CPU cache or not, since we only allow one bit set 2360 * in obj->write_domain and have been skipping the clflushes. 2361 * Just set it to the CPU cache for now. 2362 */ 2363 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 2364 ("obj %p in CPU write domain", obj)); 2365 KASSERT((obj->base.read_domains & ~I915_GEM_DOMAIN_CPU) == 0, 2366 ("obj %p in CPU read domain", obj)); 2367 2368 old_read_domains = obj->base.read_domains; 2369 old_write_domain = obj->base.write_domain; 2370 2371 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 2372 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 2373 2374 CTR3(KTR_DRM, "object_change_domain set_cache_level %p %x %x", 2375 obj, old_read_domains, old_write_domain); 2376 } 2377 2378 obj->cache_level = cache_level; 2379 return (0); 2380} 2381 2382static bool is_pin_display(struct drm_i915_gem_object *obj) 2383{ 2384 /* There are 3 sources that pin objects: 2385 * 1. The display engine (scanouts, sprites, cursors); 2386 * 2. Reservations for execbuffer; 2387 * 3. The user. 2388 * 2389 * We can ignore reservations as we hold the struct_mutex and 2390 * are only called outside of the reservation path. The user 2391 * can only increment pin_count once, and so if after 2392 * subtracting the potential reference by the user, any pin_count 2393 * remains, it must be due to another use by the display engine. 2394 */ 2395 return obj->pin_count - !!obj->user_pin_count; 2396} 2397 2398int 2399i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, 2400 u32 alignment, struct intel_ring_buffer *pipelined) 2401{ 2402 u32 old_read_domains, old_write_domain; 2403 int ret; 2404 2405 ret = i915_gem_object_flush_gpu_write_domain(obj); 2406 if (ret != 0) 2407 return (ret); 2408 2409 if (pipelined != obj->ring) { 2410 ret = i915_gem_object_sync(obj, pipelined); 2411 if (ret) 2412 return (ret); 2413 } 2414 2415 obj->pin_display = true; 2416 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE); 2417 if (ret != 0) 2418 goto err_unpin_display; 2419 2420 ret = i915_gem_object_pin(obj, alignment, true); 2421 if (ret != 0) 2422 goto err_unpin_display; 2423 2424 i915_gem_object_flush_cpu_write_domain(obj); 2425 2426 old_write_domain = obj->base.write_domain; 2427 old_read_domains = obj->base.read_domains; 2428 2429 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0, 2430 ("obj %p in GTT write domain", obj)); 2431 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 2432 2433 CTR3(KTR_DRM, "object_change_domain pin_to_display_plan %p %x %x", 2434 obj, old_read_domains, obj->base.write_domain); 2435 return (0); 2436 2437err_unpin_display: 2438 obj->pin_display = is_pin_display(obj); 2439 return ret; 2440} 2441 2442void 2443i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj) 2444{ 2445 i915_gem_object_unpin(obj); 2446 obj->pin_display = is_pin_display(obj); 2447} 2448 2449int 2450i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj) 2451{ 2452 int ret; 2453 2454 if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0) 2455 return (0); 2456 2457 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 2458 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain); 2459 if (ret != 0) 2460 return (ret); 2461 } 2462 2463 ret = i915_gem_object_wait_rendering(obj); 2464 if (ret != 0) 2465 return (ret); 2466 2467 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 2468 2469 return (0); 2470} 2471 2472int 2473i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) 2474{ 2475 uint32_t old_write_domain, old_read_domains; 2476 int ret; 2477 2478 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) 2479 return 0; 2480 2481 ret = i915_gem_object_flush_gpu_write_domain(obj); 2482 if (ret != 0) 2483 return (ret); 2484 2485 if (write || obj->pending_gpu_write) { 2486 ret = i915_gem_object_wait_rendering(obj); 2487 if (ret != 0) 2488 return (ret); 2489 } 2490 2491 i915_gem_object_flush_gtt_write_domain(obj); 2492 2493 old_write_domain = obj->base.write_domain; 2494 old_read_domains = obj->base.read_domains; 2495 2496 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) { 2497 i915_gem_clflush_object(obj); 2498 obj->base.read_domains |= I915_GEM_DOMAIN_CPU; 2499 } 2500 2501 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 2502 ("In cpu write domain")); 2503 2504 if (write) { 2505 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 2506 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 2507 } 2508 2509 CTR3(KTR_DRM, "object_change_domain set_to_cpu %p %x %x", obj, 2510 old_read_domains, old_write_domain); 2511 return (0); 2512} 2513 2514static uint32_t 2515i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode) 2516{ 2517 uint32_t gtt_size; 2518 2519 if (INTEL_INFO(dev)->gen >= 4 || 2520 tiling_mode == I915_TILING_NONE) 2521 return (size); 2522 2523 /* Previous chips need a power-of-two fence region when tiling */ 2524 if (INTEL_INFO(dev)->gen == 3) 2525 gtt_size = 1024*1024; 2526 else 2527 gtt_size = 512*1024; 2528 2529 while (gtt_size < size) 2530 gtt_size <<= 1; 2531 2532 return (gtt_size); 2533} 2534 2535/** 2536 * i915_gem_get_gtt_alignment - return required GTT alignment for an object 2537 * @obj: object to check 2538 * 2539 * Return the required GTT alignment for an object, taking into account 2540 * potential fence register mapping. 2541 */ 2542static uint32_t 2543i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size, 2544 int tiling_mode) 2545{ 2546 2547 /* 2548 * Minimum alignment is 4k (GTT page size), but might be greater 2549 * if a fence register is needed for the object. 2550 */ 2551 if (INTEL_INFO(dev)->gen >= 4 || 2552 tiling_mode == I915_TILING_NONE) 2553 return (4096); 2554 2555 /* 2556 * Previous chips need to be aligned to the size of the smallest 2557 * fence register that can contain the object. 2558 */ 2559 return (i915_gem_get_gtt_size(dev, size, tiling_mode)); 2560} 2561 2562uint32_t 2563i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev, uint32_t size, 2564 int tiling_mode) 2565{ 2566 2567 if (tiling_mode == I915_TILING_NONE) 2568 return (4096); 2569 2570 /* 2571 * Minimum alignment is 4k (GTT page size) for sane hw. 2572 */ 2573 if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev)) 2574 return (4096); 2575 2576 /* 2577 * Previous hardware however needs to be aligned to a power-of-two 2578 * tile height. The simplest method for determining this is to reuse 2579 * the power-of-tile object size. 2580 */ 2581 return (i915_gem_get_gtt_size(dev, size, tiling_mode)); 2582} 2583 2584static int 2585i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, 2586 unsigned alignment, bool map_and_fenceable) 2587{ 2588 struct drm_device *dev; 2589 struct drm_i915_private *dev_priv; 2590 struct drm_mm_node *free_space; 2591 uint32_t size, fence_size, fence_alignment, unfenced_alignment; 2592 bool mappable, fenceable; 2593 int ret; 2594 2595 dev = obj->base.dev; 2596 dev_priv = dev->dev_private; 2597 2598 if (obj->madv != I915_MADV_WILLNEED) { 2599 DRM_ERROR("Attempting to bind a purgeable object\n"); 2600 return (-EINVAL); 2601 } 2602 2603 fence_size = i915_gem_get_gtt_size(dev, obj->base.size, 2604 obj->tiling_mode); 2605 fence_alignment = i915_gem_get_gtt_alignment(dev, obj->base.size, 2606 obj->tiling_mode); 2607 unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(dev, 2608 obj->base.size, obj->tiling_mode); 2609 if (alignment == 0) 2610 alignment = map_and_fenceable ? fence_alignment : 2611 unfenced_alignment; 2612 if (map_and_fenceable && (alignment & (fence_alignment - 1)) != 0) { 2613 DRM_ERROR("Invalid object alignment requested %u\n", alignment); 2614 return (-EINVAL); 2615 } 2616 2617 size = map_and_fenceable ? fence_size : obj->base.size; 2618 2619 /* If the object is bigger than the entire aperture, reject it early 2620 * before evicting everything in a vain attempt to find space. 2621 */ 2622 if (obj->base.size > (map_and_fenceable ? 2623 dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) { 2624 DRM_ERROR( 2625"Attempting to bind an object larger than the aperture\n"); 2626 return (-E2BIG); 2627 } 2628 2629 search_free: 2630 if (map_and_fenceable) 2631 free_space = drm_mm_search_free_in_range( 2632 &dev_priv->mm.gtt_space, size, alignment, 0, 2633 dev_priv->mm.gtt_mappable_end, 0); 2634 else 2635 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space, 2636 size, alignment, 0); 2637 if (free_space != NULL) { 2638 if (map_and_fenceable) 2639 obj->gtt_space = drm_mm_get_block_range_generic( 2640 free_space, size, alignment, 0, 0, 2641 dev_priv->mm.gtt_mappable_end, 1); 2642 else 2643 obj->gtt_space = drm_mm_get_block_generic(free_space, 2644 size, alignment, 0, 1); 2645 } 2646 if (obj->gtt_space == NULL) { 2647 ret = i915_gem_evict_something(dev, size, alignment, 2648 map_and_fenceable); 2649 if (ret != 0) 2650 return (ret); 2651 goto search_free; 2652 } 2653 ret = i915_gem_object_get_pages_gtt(obj, 0); 2654 if (ret != 0) { 2655 drm_mm_put_block(obj->gtt_space); 2656 obj->gtt_space = NULL; 2657 /* 2658 * i915_gem_object_get_pages_gtt() cannot return 2659 * ENOMEM, since we use vm_page_grab(). 2660 */ 2661 return (ret); 2662 } 2663 2664 ret = i915_gem_gtt_prepare_object(obj); 2665 if (ret != 0) { 2666 i915_gem_object_put_pages_gtt(obj); 2667 drm_mm_put_block(obj->gtt_space); 2668 obj->gtt_space = NULL; 2669 if (i915_gem_evict_everything(dev, false)) 2670 return (ret); 2671 goto search_free; 2672 } 2673 2674 if (!dev_priv->mm.aliasing_ppgtt) 2675 i915_gem_gtt_bind_object(obj, obj->cache_level); 2676 2677 list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list); 2678 list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 2679 2680 KASSERT((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0, 2681 ("Object in gpu read domain")); 2682 KASSERT((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0, 2683 ("Object in gpu write domain")); 2684 2685 obj->gtt_offset = obj->gtt_space->start; 2686 2687 fenceable = 2688 obj->gtt_space->size == fence_size && 2689 (obj->gtt_space->start & (fence_alignment - 1)) == 0; 2690 2691 mappable = 2692 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end; 2693 obj->map_and_fenceable = mappable && fenceable; 2694 2695 CTR4(KTR_DRM, "object_bind %p %x %x %d", obj, obj->gtt_offset, 2696 obj->base.size, map_and_fenceable); 2697 return (0); 2698} 2699 2700int 2701i915_gem_object_sync(struct drm_i915_gem_object *obj, 2702 struct intel_ring_buffer *to) 2703{ 2704 struct intel_ring_buffer *from = obj->ring; 2705 u32 seqno; 2706 int ret, idx; 2707 2708 if (from == NULL || to == from) 2709 return 0; 2710 2711 if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev)) 2712 return i915_gem_object_wait_rendering(obj); 2713 2714 idx = intel_ring_sync_index(from, to); 2715 2716 seqno = obj->last_rendering_seqno; 2717 if (seqno <= from->sync_seqno[idx]) 2718 return 0; 2719 2720 if (seqno == from->outstanding_lazy_request) { 2721 struct drm_i915_gem_request *request; 2722 2723 request = malloc(sizeof(*request), DRM_I915_GEM, 2724 M_WAITOK | M_ZERO); 2725 ret = i915_add_request(from, NULL, request); 2726 if (ret) { 2727 free(request, DRM_I915_GEM); 2728 return ret; 2729 } 2730 seqno = request->seqno; 2731 } 2732 2733 2734 ret = to->sync_to(to, from, seqno); 2735 if (!ret) 2736 from->sync_seqno[idx] = seqno; 2737 2738 return ret; 2739} 2740 2741static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj) 2742{ 2743 u32 old_write_domain, old_read_domains; 2744 2745 /* Act a barrier for all accesses through the GTT */ 2746 mb(); 2747 2748 /* Force a pagefault for domain tracking on next user access */ 2749 i915_gem_release_mmap(obj); 2750 2751 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) 2752 return; 2753 2754 old_read_domains = obj->base.read_domains; 2755 old_write_domain = obj->base.write_domain; 2756 2757 obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT; 2758 obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT; 2759 2760 CTR3(KTR_DRM, "object_change_domain finish gtt %p %x %x", 2761 obj, old_read_domains, old_write_domain); 2762} 2763 2764int 2765i915_gem_object_unbind(struct drm_i915_gem_object *obj) 2766{ 2767 drm_i915_private_t *dev_priv; 2768 int ret; 2769 2770 dev_priv = obj->base.dev->dev_private; 2771 ret = 0; 2772 if (obj->gtt_space == NULL) 2773 return (0); 2774 if (obj->pin_count != 0) 2775 return (-EINVAL); 2776 2777 ret = i915_gem_object_finish_gpu(obj); 2778 if (ret == -ERESTARTSYS || ret == -EINTR) 2779 return (ret); 2780 2781 i915_gem_object_finish_gtt(obj); 2782 2783 if (ret == 0) 2784 ret = i915_gem_object_set_to_cpu_domain(obj, 1); 2785 if (ret == -ERESTARTSYS || ret == -EINTR) 2786 return (ret); 2787 if (ret != 0) { 2788 i915_gem_clflush_object(obj); 2789 obj->base.read_domains = obj->base.write_domain = 2790 I915_GEM_DOMAIN_CPU; 2791 } 2792 2793 ret = i915_gem_object_put_fence(obj); 2794 if (ret) 2795 return (ret); 2796 2797 if (obj->has_global_gtt_mapping) 2798 i915_gem_gtt_unbind_object(obj); 2799 if (obj->has_aliasing_ppgtt_mapping) { 2800 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj); 2801 obj->has_aliasing_ppgtt_mapping = 0; 2802 } 2803 i915_gem_gtt_finish_object(obj); 2804 2805 i915_gem_object_put_pages_gtt(obj); 2806 2807 list_del_init(&obj->gtt_list); 2808 list_del_init(&obj->mm_list); 2809 obj->map_and_fenceable = true; 2810 2811 drm_mm_put_block(obj->gtt_space); 2812 obj->gtt_space = NULL; 2813 obj->gtt_offset = 0; 2814 2815 if (i915_gem_object_is_purgeable(obj)) 2816 i915_gem_object_truncate(obj); 2817 CTR1(KTR_DRM, "object_unbind %p", obj); 2818 2819 return (ret); 2820} 2821 2822static void 2823i915_gem_object_put_pages_range_locked(struct drm_i915_gem_object *obj, 2824 vm_pindex_t si, vm_pindex_t ei) 2825{ 2826 vm_object_t vm_obj; 2827 vm_page_t m; 2828 vm_pindex_t i; 2829 2830 vm_obj = obj->base.vm_obj; 2831 VM_OBJECT_ASSERT_LOCKED(vm_obj); 2832 for (i = si, m = vm_page_lookup(vm_obj, i); i < ei; 2833 m = vm_page_next(m), i++) { 2834 KASSERT(m->pindex == i, ("pindex %jx %jx", 2835 (uintmax_t)m->pindex, (uintmax_t)i)); 2836 vm_page_lock(m); 2837 vm_page_unwire(m, 0); 2838 if (m->wire_count == 0) 2839 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 2840 vm_page_unlock(m); 2841 } 2842} 2843 2844static void 2845i915_gem_object_put_pages_range(struct drm_i915_gem_object *obj, 2846 off_t start, off_t end) 2847{ 2848 vm_object_t vm_obj; 2849 2850 vm_obj = obj->base.vm_obj; 2851 VM_OBJECT_WLOCK(vm_obj); 2852 i915_gem_object_put_pages_range_locked(obj, 2853 OFF_TO_IDX(trunc_page(start)), OFF_TO_IDX(round_page(end))); 2854 VM_OBJECT_WUNLOCK(vm_obj); 2855} 2856 2857static int 2858i915_gem_object_get_pages_range(struct drm_i915_gem_object *obj, 2859 off_t start, off_t end) 2860{ 2861 vm_object_t vm_obj; 2862 vm_page_t m; 2863 vm_pindex_t si, ei, i; 2864 bool need_swizzle, fresh; 2865 2866 need_swizzle = i915_gem_object_needs_bit17_swizzle(obj) != 0; 2867 vm_obj = obj->base.vm_obj; 2868 si = OFF_TO_IDX(trunc_page(start)); 2869 ei = OFF_TO_IDX(round_page(end)); 2870 VM_OBJECT_WLOCK(vm_obj); 2871 for (i = si; i < ei; i++) { 2872 m = i915_gem_wire_page(vm_obj, i, &fresh); 2873 if (m == NULL) 2874 goto failed; 2875 if (need_swizzle && fresh) 2876 i915_gem_object_do_bit_17_swizzle_page(obj, m); 2877 } 2878 VM_OBJECT_WUNLOCK(vm_obj); 2879 return (0); 2880failed: 2881 i915_gem_object_put_pages_range_locked(obj, si, i); 2882 VM_OBJECT_WUNLOCK(vm_obj); 2883 return (-EIO); 2884} 2885 2886static int 2887i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, 2888 int flags) 2889{ 2890 struct drm_device *dev; 2891 vm_object_t vm_obj; 2892 vm_page_t m; 2893 vm_pindex_t i, page_count; 2894 int res; 2895 2896 dev = obj->base.dev; 2897 KASSERT(obj->pages == NULL, ("Obj already has pages")); 2898 page_count = OFF_TO_IDX(obj->base.size); 2899 obj->pages = malloc(page_count * sizeof(vm_page_t), DRM_I915_GEM, 2900 M_WAITOK); 2901 res = i915_gem_object_get_pages_range(obj, 0, obj->base.size); 2902 if (res != 0) { 2903 free(obj->pages, DRM_I915_GEM); 2904 obj->pages = NULL; 2905 return (res); 2906 } 2907 vm_obj = obj->base.vm_obj; 2908 VM_OBJECT_WLOCK(vm_obj); 2909 for (i = 0, m = vm_page_lookup(vm_obj, 0); i < page_count; 2910 i++, m = vm_page_next(m)) { 2911 KASSERT(m->pindex == i, ("pindex %jx %jx", 2912 (uintmax_t)m->pindex, (uintmax_t)i)); 2913 obj->pages[i] = m; 2914 } 2915 VM_OBJECT_WUNLOCK(vm_obj); 2916 return (0); 2917} 2918 2919#define GEM_PARANOID_CHECK_GTT 0 2920#if GEM_PARANOID_CHECK_GTT 2921static void 2922i915_gem_assert_pages_not_mapped(struct drm_device *dev, vm_page_t *ma, 2923 int page_count) 2924{ 2925 struct drm_i915_private *dev_priv; 2926 vm_paddr_t pa; 2927 unsigned long start, end; 2928 u_int i; 2929 int j; 2930 2931 dev_priv = dev->dev_private; 2932 start = OFF_TO_IDX(dev_priv->mm.gtt_start); 2933 end = OFF_TO_IDX(dev_priv->mm.gtt_end); 2934 for (i = start; i < end; i++) { 2935 pa = intel_gtt_read_pte_paddr(i); 2936 for (j = 0; j < page_count; j++) { 2937 if (pa == VM_PAGE_TO_PHYS(ma[j])) { 2938 panic("Page %p in GTT pte index %d pte %x", 2939 ma[i], i, intel_gtt_read_pte(i)); 2940 } 2941 } 2942 } 2943} 2944#endif 2945 2946static void 2947i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) 2948{ 2949 vm_page_t m; 2950 int page_count, i; 2951 2952 KASSERT(obj->madv != I915_MADV_PURGED_INTERNAL, ("Purged object")); 2953 2954 if (obj->tiling_mode != I915_TILING_NONE) 2955 i915_gem_object_save_bit_17_swizzle(obj); 2956 if (obj->madv == I915_MADV_DONTNEED) 2957 obj->dirty = 0; 2958 page_count = obj->base.size / PAGE_SIZE; 2959 VM_OBJECT_WLOCK(obj->base.vm_obj); 2960#if GEM_PARANOID_CHECK_GTT 2961 i915_gem_assert_pages_not_mapped(obj->base.dev, obj->pages, page_count); 2962#endif 2963 for (i = 0; i < page_count; i++) { 2964 m = obj->pages[i]; 2965 if (obj->dirty) 2966 vm_page_dirty(m); 2967 if (obj->madv == I915_MADV_WILLNEED) 2968 vm_page_reference(m); 2969 vm_page_lock(m); 2970 vm_page_unwire(obj->pages[i], 1); 2971 vm_page_unlock(m); 2972 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 2973 } 2974 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 2975 obj->dirty = 0; 2976 free(obj->pages, DRM_I915_GEM); 2977 obj->pages = NULL; 2978} 2979 2980void 2981i915_gem_release_mmap(struct drm_i915_gem_object *obj) 2982{ 2983 vm_object_t devobj; 2984 vm_page_t m; 2985 int i, page_count; 2986 2987 if (!obj->fault_mappable) 2988 return; 2989 2990 CTR3(KTR_DRM, "release_mmap %p %x %x", obj, obj->gtt_offset, 2991 OFF_TO_IDX(obj->base.size)); 2992 devobj = cdev_pager_lookup(obj); 2993 if (devobj != NULL) { 2994 page_count = OFF_TO_IDX(obj->base.size); 2995 2996 VM_OBJECT_WLOCK(devobj); 2997retry: 2998 for (i = 0; i < page_count; i++) { 2999 m = vm_page_lookup(devobj, i); 3000 if (m == NULL) 3001 continue; 3002 if (vm_page_sleep_if_busy(m, "915unm")) 3003 goto retry; 3004 cdev_pager_free_page(devobj, m); 3005 } 3006 VM_OBJECT_WUNLOCK(devobj); 3007 vm_object_deallocate(devobj); 3008 } 3009 3010 obj->fault_mappable = false; 3011} 3012 3013int 3014i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj) 3015{ 3016 int ret; 3017 3018 KASSERT((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0, 3019 ("In GPU write domain")); 3020 3021 CTR5(KTR_DRM, "object_wait_rendering %p %s %x %d %d", obj, 3022 obj->ring != NULL ? obj->ring->name : "none", obj->gtt_offset, 3023 obj->active, obj->last_rendering_seqno); 3024 if (obj->active) { 3025 ret = i915_wait_request(obj->ring, obj->last_rendering_seqno); 3026 if (ret != 0) 3027 return (ret); 3028 i915_gem_retire_requests_ring(obj->ring); 3029 } 3030 return (0); 3031} 3032 3033void 3034i915_gem_object_move_to_active(struct drm_i915_gem_object *obj, 3035 struct intel_ring_buffer *ring, uint32_t seqno) 3036{ 3037 struct drm_device *dev = obj->base.dev; 3038 struct drm_i915_private *dev_priv = dev->dev_private; 3039 struct drm_i915_fence_reg *reg; 3040 3041 obj->ring = ring; 3042 KASSERT(ring != NULL, ("NULL ring")); 3043 3044 /* Add a reference if we're newly entering the active list. */ 3045 if (!obj->active) { 3046 drm_gem_object_reference(&obj->base); 3047 obj->active = 1; 3048 } 3049 3050 /* Move from whatever list we were on to the tail of execution. */ 3051 list_move_tail(&obj->mm_list, &dev_priv->mm.active_list); 3052 list_move_tail(&obj->ring_list, &ring->active_list); 3053 3054 obj->last_rendering_seqno = seqno; 3055 if (obj->fenced_gpu_access) { 3056 obj->last_fenced_seqno = seqno; 3057 3058 /* Bump MRU to take account of the delayed flush */ 3059 if (obj->fence_reg != I915_FENCE_REG_NONE) { 3060 reg = &dev_priv->fence_regs[obj->fence_reg]; 3061 list_move_tail(®->lru_list, 3062 &dev_priv->mm.fence_list); 3063 } 3064 } 3065} 3066 3067static void 3068i915_gem_object_move_off_active(struct drm_i915_gem_object *obj) 3069{ 3070 list_del_init(&obj->ring_list); 3071 obj->last_rendering_seqno = 0; 3072 obj->last_fenced_seqno = 0; 3073} 3074 3075static void 3076i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj) 3077{ 3078 struct drm_device *dev = obj->base.dev; 3079 drm_i915_private_t *dev_priv = dev->dev_private; 3080 3081 KASSERT(obj->active, ("Object not active")); 3082 list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list); 3083 3084 i915_gem_object_move_off_active(obj); 3085} 3086 3087static void 3088i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj) 3089{ 3090 struct drm_device *dev = obj->base.dev; 3091 struct drm_i915_private *dev_priv = dev->dev_private; 3092 3093 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 3094 3095 KASSERT(list_empty(&obj->gpu_write_list), ("On gpu_write_list")); 3096 KASSERT(obj->active, ("Object not active")); 3097 obj->ring = NULL; 3098 3099 i915_gem_object_move_off_active(obj); 3100 obj->fenced_gpu_access = false; 3101 3102 obj->active = 0; 3103 obj->pending_gpu_write = false; 3104 drm_gem_object_unreference(&obj->base); 3105 3106#if 1 3107 KIB_NOTYET(); 3108#else 3109 WARN_ON(i915_verify_lists(dev)); 3110#endif 3111} 3112 3113static void 3114i915_gem_object_truncate(struct drm_i915_gem_object *obj) 3115{ 3116 vm_object_t vm_obj; 3117 3118 vm_obj = obj->base.vm_obj; 3119 VM_OBJECT_WLOCK(vm_obj); 3120 vm_object_page_remove(vm_obj, 0, 0, false); 3121 VM_OBJECT_WUNLOCK(vm_obj); 3122 drm_gem_free_mmap_offset(&obj->base); 3123 obj->madv = I915_MADV_PURGED_INTERNAL; 3124} 3125 3126static inline int 3127i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj) 3128{ 3129 3130 return (obj->madv == I915_MADV_DONTNEED); 3131} 3132 3133static void 3134i915_gem_process_flushing_list(struct intel_ring_buffer *ring, 3135 uint32_t flush_domains) 3136{ 3137 struct drm_i915_gem_object *obj, *next; 3138 uint32_t old_write_domain; 3139 3140 list_for_each_entry_safe(obj, next, &ring->gpu_write_list, 3141 gpu_write_list) { 3142 if (obj->base.write_domain & flush_domains) { 3143 old_write_domain = obj->base.write_domain; 3144 obj->base.write_domain = 0; 3145 list_del_init(&obj->gpu_write_list); 3146 i915_gem_object_move_to_active(obj, ring, 3147 i915_gem_next_request_seqno(ring)); 3148 3149 CTR3(KTR_DRM, "object_change_domain process_flush %p %x %x", 3150 obj, obj->base.read_domains, old_write_domain); 3151 } 3152 } 3153} 3154 3155static int 3156i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj) 3157{ 3158 drm_i915_private_t *dev_priv; 3159 3160 dev_priv = obj->base.dev->dev_private; 3161 return (dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && 3162 obj->tiling_mode != I915_TILING_NONE); 3163} 3164 3165static vm_page_t 3166i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex, bool *fresh) 3167{ 3168 vm_page_t m; 3169 int rv; 3170 3171 VM_OBJECT_ASSERT_WLOCKED(object); 3172 m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL); 3173 if (m->valid != VM_PAGE_BITS_ALL) { 3174 if (vm_pager_has_page(object, pindex, NULL, NULL)) { 3175 rv = vm_pager_get_pages(object, &m, 1, 0); 3176 m = vm_page_lookup(object, pindex); 3177 if (m == NULL) 3178 return (NULL); 3179 if (rv != VM_PAGER_OK) { 3180 vm_page_lock(m); 3181 vm_page_free(m); 3182 vm_page_unlock(m); 3183 return (NULL); 3184 } 3185 if (fresh != NULL) 3186 *fresh = true; 3187 } else { 3188 pmap_zero_page(m); 3189 m->valid = VM_PAGE_BITS_ALL; 3190 m->dirty = 0; 3191 if (fresh != NULL) 3192 *fresh = false; 3193 } 3194 } else if (fresh != NULL) { 3195 *fresh = false; 3196 } 3197 vm_page_lock(m); 3198 vm_page_wire(m); 3199 vm_page_unlock(m); 3200 vm_page_xunbusy(m); 3201 atomic_add_long(&i915_gem_wired_pages_cnt, 1); 3202 return (m); 3203} 3204 3205int 3206i915_gem_flush_ring(struct intel_ring_buffer *ring, uint32_t invalidate_domains, 3207 uint32_t flush_domains) 3208{ 3209 int ret; 3210 3211 if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0) 3212 return 0; 3213 3214 CTR3(KTR_DRM, "ring_flush %s %x %x", ring->name, invalidate_domains, 3215 flush_domains); 3216 ret = ring->flush(ring, invalidate_domains, flush_domains); 3217 if (ret) 3218 return ret; 3219 3220 if (flush_domains & I915_GEM_GPU_DOMAINS) 3221 i915_gem_process_flushing_list(ring, flush_domains); 3222 return 0; 3223} 3224 3225static int 3226i915_ring_idle(struct intel_ring_buffer *ring) 3227{ 3228 int ret; 3229 3230 if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list)) 3231 return 0; 3232 3233 if (!list_empty(&ring->gpu_write_list)) { 3234 ret = i915_gem_flush_ring(ring, I915_GEM_GPU_DOMAINS, 3235 I915_GEM_GPU_DOMAINS); 3236 if (ret != 0) 3237 return ret; 3238 } 3239 3240 return (i915_wait_request(ring, i915_gem_next_request_seqno(ring))); 3241} 3242 3243int 3244i915_gpu_idle(struct drm_device *dev) 3245{ 3246 drm_i915_private_t *dev_priv = dev->dev_private; 3247 struct intel_ring_buffer *ring; 3248 int ret, i; 3249 3250 /* Flush everything onto the inactive list. */ 3251 for_each_ring(ring, dev_priv, i) { 3252 ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID); 3253 if (ret) 3254 return ret; 3255 3256 ret = i915_ring_idle(ring); 3257 if (ret) 3258 return ret; 3259 3260 /* Is the device fubar? */ 3261 if (!list_empty(&ring->gpu_write_list)) 3262 return -EBUSY; 3263 } 3264 3265 return 0; 3266} 3267 3268static int 3269i915_gem_check_wedge(struct drm_i915_private *dev_priv) 3270{ 3271 DRM_LOCK_ASSERT(dev_priv->dev); 3272 3273 if (atomic_load_acq_int(&dev_priv->mm.wedged) != 0) { 3274 bool recovery_complete; 3275 /* Give the error handler a chance to run. */ 3276 mtx_lock(&dev_priv->error_completion_lock); 3277 recovery_complete = (&dev_priv->error_completion) > 0; 3278 mtx_unlock(&dev_priv->error_completion_lock); 3279 return (recovery_complete ? -EIO : -EAGAIN); 3280 } 3281 3282 return 0; 3283} 3284 3285/* 3286 * Compare seqno against outstanding lazy request. Emit a request if they are 3287 * equal. 3288 */ 3289static int 3290i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno) 3291{ 3292 int ret = 0; 3293 3294 DRM_LOCK_ASSERT(ring->dev); 3295 3296 if (seqno == ring->outstanding_lazy_request) { 3297 struct drm_i915_gem_request *request; 3298 3299 request = malloc(sizeof(*request), DRM_I915_GEM, 3300 M_WAITOK | M_ZERO); 3301 3302 ret = i915_add_request(ring, NULL, request); 3303 if (ret != 0) { 3304 free(request, DRM_I915_GEM); 3305 return (ret); 3306 } 3307 3308 MPASS(seqno == request->seqno); 3309 } 3310 return ret; 3311} 3312 3313static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno, 3314 bool interruptible) 3315{ 3316 drm_i915_private_t *dev_priv = ring->dev->dev_private; 3317 int ret = 0, flags; 3318 3319 if (i915_seqno_passed(ring->get_seqno(ring), seqno)) 3320 return 0; 3321 3322 CTR2(KTR_DRM, "request_wait_begin %s %d", ring->name, seqno); 3323 3324 mtx_lock(&dev_priv->irq_lock); 3325 if (!ring->irq_get(ring)) { 3326 mtx_unlock(&dev_priv->irq_lock); 3327 return (-ENODEV); 3328 } 3329 3330 flags = interruptible ? PCATCH : 0; 3331 while (!i915_seqno_passed(ring->get_seqno(ring), seqno) 3332 && !atomic_load_acq_int(&dev_priv->mm.wedged) && 3333 ret == 0) { 3334 ret = -msleep(ring, &dev_priv->irq_lock, flags, "915gwr", 0); 3335 if (ret == -ERESTART) 3336 ret = -ERESTARTSYS; 3337 } 3338 ring->irq_put(ring); 3339 mtx_unlock(&dev_priv->irq_lock); 3340 3341 CTR3(KTR_DRM, "request_wait_end %s %d %d", ring->name, seqno, ret); 3342 3343 return ret; 3344} 3345 3346int 3347i915_wait_request(struct intel_ring_buffer *ring, uint32_t seqno) 3348{ 3349 drm_i915_private_t *dev_priv; 3350 int ret; 3351 3352 KASSERT(seqno != 0, ("Zero seqno")); 3353 3354 dev_priv = ring->dev->dev_private; 3355 ret = 0; 3356 3357 ret = i915_gem_check_wedge(dev_priv); 3358 if (ret) 3359 return ret; 3360 3361 ret = i915_gem_check_olr(ring, seqno); 3362 if (ret) 3363 return ret; 3364 3365 ret = __wait_seqno(ring, seqno, dev_priv->mm.interruptible); 3366 if (atomic_load_acq_int(&dev_priv->mm.wedged)) 3367 ret = -EAGAIN; 3368 3369 return (ret); 3370} 3371 3372static u32 3373i915_gem_get_seqno(struct drm_device *dev) 3374{ 3375 drm_i915_private_t *dev_priv = dev->dev_private; 3376 u32 seqno = dev_priv->next_seqno; 3377 3378 /* reserve 0 for non-seqno */ 3379 if (++dev_priv->next_seqno == 0) 3380 dev_priv->next_seqno = 1; 3381 3382 return seqno; 3383} 3384 3385u32 3386i915_gem_next_request_seqno(struct intel_ring_buffer *ring) 3387{ 3388 if (ring->outstanding_lazy_request == 0) 3389 ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev); 3390 3391 return ring->outstanding_lazy_request; 3392} 3393 3394int 3395i915_add_request(struct intel_ring_buffer *ring, struct drm_file *file, 3396 struct drm_i915_gem_request *request) 3397{ 3398 drm_i915_private_t *dev_priv; 3399 struct drm_i915_file_private *file_priv; 3400 uint32_t seqno; 3401 u32 request_ring_position; 3402 int was_empty; 3403 int ret; 3404 3405 KASSERT(request != NULL, ("NULL request in add")); 3406 DRM_LOCK_ASSERT(ring->dev); 3407 dev_priv = ring->dev->dev_private; 3408 3409 seqno = i915_gem_next_request_seqno(ring); 3410 request_ring_position = intel_ring_get_tail(ring); 3411 3412 ret = ring->add_request(ring, &seqno); 3413 if (ret != 0) 3414 return ret; 3415 3416 CTR2(KTR_DRM, "request_add %s %d", ring->name, seqno); 3417 3418 request->seqno = seqno; 3419 request->ring = ring; 3420 request->tail = request_ring_position; 3421 request->emitted_jiffies = ticks; 3422 was_empty = list_empty(&ring->request_list); 3423 list_add_tail(&request->list, &ring->request_list); 3424 3425 if (file != NULL) { 3426 file_priv = file->driver_priv; 3427 3428 mtx_lock(&file_priv->mm.lck); 3429 request->file_priv = file_priv; 3430 list_add_tail(&request->client_list, 3431 &file_priv->mm.request_list); 3432 mtx_unlock(&file_priv->mm.lck); 3433 } 3434 3435 ring->outstanding_lazy_request = 0; 3436 3437 if (!dev_priv->mm.suspended) { 3438 if (i915_enable_hangcheck) { 3439 callout_schedule(&dev_priv->hangcheck_timer, 3440 DRM_I915_HANGCHECK_PERIOD); 3441 } 3442 if (was_empty) 3443 taskqueue_enqueue_timeout(dev_priv->tq, 3444 &dev_priv->mm.retire_task, hz); 3445 } 3446 return (0); 3447} 3448 3449static inline void 3450i915_gem_request_remove_from_client(struct drm_i915_gem_request *request) 3451{ 3452 struct drm_i915_file_private *file_priv = request->file_priv; 3453 3454 if (!file_priv) 3455 return; 3456 3457 DRM_LOCK_ASSERT(request->ring->dev); 3458 3459 mtx_lock(&file_priv->mm.lck); 3460 if (request->file_priv != NULL) { 3461 list_del(&request->client_list); 3462 request->file_priv = NULL; 3463 } 3464 mtx_unlock(&file_priv->mm.lck); 3465} 3466 3467void 3468i915_gem_release(struct drm_device *dev, struct drm_file *file) 3469{ 3470 struct drm_i915_file_private *file_priv; 3471 struct drm_i915_gem_request *request; 3472 3473 file_priv = file->driver_priv; 3474 3475 /* Clean up our request list when the client is going away, so that 3476 * later retire_requests won't dereference our soon-to-be-gone 3477 * file_priv. 3478 */ 3479 mtx_lock(&file_priv->mm.lck); 3480 while (!list_empty(&file_priv->mm.request_list)) { 3481 request = list_first_entry(&file_priv->mm.request_list, 3482 struct drm_i915_gem_request, 3483 client_list); 3484 list_del(&request->client_list); 3485 request->file_priv = NULL; 3486 } 3487 mtx_unlock(&file_priv->mm.lck); 3488} 3489 3490static void 3491i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv, 3492 struct intel_ring_buffer *ring) 3493{ 3494 3495 if (ring->dev != NULL) 3496 DRM_LOCK_ASSERT(ring->dev); 3497 3498 while (!list_empty(&ring->request_list)) { 3499 struct drm_i915_gem_request *request; 3500 3501 request = list_first_entry(&ring->request_list, 3502 struct drm_i915_gem_request, list); 3503 3504 list_del(&request->list); 3505 i915_gem_request_remove_from_client(request); 3506 free(request, DRM_I915_GEM); 3507 } 3508 3509 while (!list_empty(&ring->active_list)) { 3510 struct drm_i915_gem_object *obj; 3511 3512 obj = list_first_entry(&ring->active_list, 3513 struct drm_i915_gem_object, ring_list); 3514 3515 obj->base.write_domain = 0; 3516 list_del_init(&obj->gpu_write_list); 3517 i915_gem_object_move_to_inactive(obj); 3518 } 3519} 3520 3521static void 3522i915_gem_reset_fences(struct drm_device *dev) 3523{ 3524 struct drm_i915_private *dev_priv = dev->dev_private; 3525 int i; 3526 3527 for (i = 0; i < dev_priv->num_fence_regs; i++) { 3528 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; 3529 3530 i915_gem_write_fence(dev, i, NULL); 3531 3532 if (reg->obj) 3533 i915_gem_object_fence_lost(reg->obj); 3534 3535 reg->pin_count = 0; 3536 reg->obj = NULL; 3537 INIT_LIST_HEAD(®->lru_list); 3538 } 3539 3540 INIT_LIST_HEAD(&dev_priv->mm.fence_list); 3541} 3542 3543void 3544i915_gem_reset(struct drm_device *dev) 3545{ 3546 struct drm_i915_private *dev_priv = dev->dev_private; 3547 struct drm_i915_gem_object *obj; 3548 struct intel_ring_buffer *ring; 3549 int i; 3550 3551 for_each_ring(ring, dev_priv, i) 3552 i915_gem_reset_ring_lists(dev_priv, ring); 3553 3554 /* Remove anything from the flushing lists. The GPU cache is likely 3555 * to be lost on reset along with the data, so simply move the 3556 * lost bo to the inactive list. 3557 */ 3558 while (!list_empty(&dev_priv->mm.flushing_list)) { 3559 obj = list_first_entry(&dev_priv->mm.flushing_list, 3560 struct drm_i915_gem_object, 3561 mm_list); 3562 3563 obj->base.write_domain = 0; 3564 list_del_init(&obj->gpu_write_list); 3565 i915_gem_object_move_to_inactive(obj); 3566 } 3567 3568 /* Move everything out of the GPU domains to ensure we do any 3569 * necessary invalidation upon reuse. 3570 */ 3571 list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) { 3572 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 3573 } 3574 3575 /* The fence registers are invalidated so clear them out */ 3576 i915_gem_reset_fences(dev); 3577} 3578 3579/** 3580 * This function clears the request list as sequence numbers are passed. 3581 */ 3582void 3583i915_gem_retire_requests_ring(struct intel_ring_buffer *ring) 3584{ 3585 uint32_t seqno; 3586 int i; 3587 3588 if (list_empty(&ring->request_list)) 3589 return; 3590 3591 seqno = ring->get_seqno(ring); 3592 CTR2(KTR_DRM, "retire_request_ring %s %d", ring->name, seqno); 3593 3594 for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++) 3595 if (seqno >= ring->sync_seqno[i]) 3596 ring->sync_seqno[i] = 0; 3597 3598 while (!list_empty(&ring->request_list)) { 3599 struct drm_i915_gem_request *request; 3600 3601 request = list_first_entry(&ring->request_list, 3602 struct drm_i915_gem_request, 3603 list); 3604 3605 if (!i915_seqno_passed(seqno, request->seqno)) 3606 break; 3607 3608 CTR2(KTR_DRM, "retire_request_seqno_passed %s %d", 3609 ring->name, seqno); 3610 ring->last_retired_head = request->tail; 3611 3612 list_del(&request->list); 3613 i915_gem_request_remove_from_client(request); 3614 free(request, DRM_I915_GEM); 3615 } 3616 3617 /* Move any buffers on the active list that are no longer referenced 3618 * by the ringbuffer to the flushing/inactive lists as appropriate. 3619 */ 3620 while (!list_empty(&ring->active_list)) { 3621 struct drm_i915_gem_object *obj; 3622 3623 obj = list_first_entry(&ring->active_list, 3624 struct drm_i915_gem_object, 3625 ring_list); 3626 3627 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno)) 3628 break; 3629 3630 if (obj->base.write_domain != 0) 3631 i915_gem_object_move_to_flushing(obj); 3632 else 3633 i915_gem_object_move_to_inactive(obj); 3634 } 3635 3636 if (ring->trace_irq_seqno && 3637 i915_seqno_passed(seqno, ring->trace_irq_seqno)) { 3638 struct drm_i915_private *dev_priv = ring->dev->dev_private; 3639 mtx_lock(&dev_priv->irq_lock); 3640 ring->irq_put(ring); 3641 mtx_unlock(&dev_priv->irq_lock); 3642 ring->trace_irq_seqno = 0; 3643 } 3644} 3645 3646void 3647i915_gem_retire_requests(struct drm_device *dev) 3648{ 3649 drm_i915_private_t *dev_priv = dev->dev_private; 3650 struct intel_ring_buffer *ring; 3651 int i; 3652 3653 for_each_ring(ring, dev_priv, i) 3654 i915_gem_retire_requests_ring(ring); 3655} 3656 3657static void sandybridge_write_fence_reg(struct drm_device *dev, int reg, 3658 struct drm_i915_gem_object *obj) 3659{ 3660 drm_i915_private_t *dev_priv = dev->dev_private; 3661 uint64_t val; 3662 3663 if (obj) { 3664 u32 size = obj->gtt_space->size; 3665 3666 val = (uint64_t)((obj->gtt_offset + size - 4096) & 3667 0xfffff000) << 32; 3668 val |= obj->gtt_offset & 0xfffff000; 3669 val |= (uint64_t)((obj->stride / 128) - 1) << 3670 SANDYBRIDGE_FENCE_PITCH_SHIFT; 3671 3672 if (obj->tiling_mode == I915_TILING_Y) 3673 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 3674 val |= I965_FENCE_REG_VALID; 3675 } else 3676 val = 0; 3677 3678 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val); 3679 POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8); 3680} 3681 3682static void i965_write_fence_reg(struct drm_device *dev, int reg, 3683 struct drm_i915_gem_object *obj) 3684{ 3685 drm_i915_private_t *dev_priv = dev->dev_private; 3686 uint64_t val; 3687 3688 if (obj) { 3689 u32 size = obj->gtt_space->size; 3690 3691 val = (uint64_t)((obj->gtt_offset + size - 4096) & 3692 0xfffff000) << 32; 3693 val |= obj->gtt_offset & 0xfffff000; 3694 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT; 3695 if (obj->tiling_mode == I915_TILING_Y) 3696 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 3697 val |= I965_FENCE_REG_VALID; 3698 } else 3699 val = 0; 3700 3701 I915_WRITE64(FENCE_REG_965_0 + reg * 8, val); 3702 POSTING_READ(FENCE_REG_965_0 + reg * 8); 3703} 3704 3705static void i915_write_fence_reg(struct drm_device *dev, int reg, 3706 struct drm_i915_gem_object *obj) 3707{ 3708 drm_i915_private_t *dev_priv = dev->dev_private; 3709 u32 val; 3710 3711 if (obj) { 3712 u32 size = obj->gtt_space->size; 3713 int pitch_val; 3714 int tile_width; 3715 3716 if ((obj->gtt_offset & ~I915_FENCE_START_MASK) || 3717 (size & -size) != size || 3718 (obj->gtt_offset & (size - 1))) 3719 printf( 3720 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n", 3721 obj->gtt_offset, obj->map_and_fenceable, size); 3722 3723 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)) 3724 tile_width = 128; 3725 else 3726 tile_width = 512; 3727 3728 /* Note: pitch better be a power of two tile widths */ 3729 pitch_val = obj->stride / tile_width; 3730 pitch_val = ffs(pitch_val) - 1; 3731 3732 val = obj->gtt_offset; 3733 if (obj->tiling_mode == I915_TILING_Y) 3734 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 3735 val |= I915_FENCE_SIZE_BITS(size); 3736 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 3737 val |= I830_FENCE_REG_VALID; 3738 } else 3739 val = 0; 3740 3741 if (reg < 8) 3742 reg = FENCE_REG_830_0 + reg * 4; 3743 else 3744 reg = FENCE_REG_945_8 + (reg - 8) * 4; 3745 3746 I915_WRITE(reg, val); 3747 POSTING_READ(reg); 3748} 3749 3750static void i830_write_fence_reg(struct drm_device *dev, int reg, 3751 struct drm_i915_gem_object *obj) 3752{ 3753 drm_i915_private_t *dev_priv = dev->dev_private; 3754 uint32_t val; 3755 3756 if (obj) { 3757 u32 size = obj->gtt_space->size; 3758 uint32_t pitch_val; 3759 3760 if ((obj->gtt_offset & ~I830_FENCE_START_MASK) || 3761 (size & -size) != size || 3762 (obj->gtt_offset & (size - 1))) 3763 printf( 3764 "object 0x%08x not 512K or pot-size 0x%08x aligned\n", 3765 obj->gtt_offset, size); 3766 3767 pitch_val = obj->stride / 128; 3768 pitch_val = ffs(pitch_val) - 1; 3769 3770 val = obj->gtt_offset; 3771 if (obj->tiling_mode == I915_TILING_Y) 3772 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 3773 val |= I830_FENCE_SIZE_BITS(size); 3774 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 3775 val |= I830_FENCE_REG_VALID; 3776 } else 3777 val = 0; 3778 3779 I915_WRITE(FENCE_REG_830_0 + reg * 4, val); 3780 POSTING_READ(FENCE_REG_830_0 + reg * 4); 3781} 3782 3783static void i915_gem_write_fence(struct drm_device *dev, int reg, 3784 struct drm_i915_gem_object *obj) 3785{ 3786 switch (INTEL_INFO(dev)->gen) { 3787 case 7: 3788 case 6: sandybridge_write_fence_reg(dev, reg, obj); break; 3789 case 5: 3790 case 4: i965_write_fence_reg(dev, reg, obj); break; 3791 case 3: i915_write_fence_reg(dev, reg, obj); break; 3792 case 2: i830_write_fence_reg(dev, reg, obj); break; 3793 default: break; 3794 } 3795} 3796 3797static inline int fence_number(struct drm_i915_private *dev_priv, 3798 struct drm_i915_fence_reg *fence) 3799{ 3800 return fence - dev_priv->fence_regs; 3801} 3802 3803static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, 3804 struct drm_i915_fence_reg *fence, 3805 bool enable) 3806{ 3807 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 3808 int reg = fence_number(dev_priv, fence); 3809 3810 i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL); 3811 3812 if (enable) { 3813 obj->fence_reg = reg; 3814 fence->obj = obj; 3815 list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list); 3816 } else { 3817 obj->fence_reg = I915_FENCE_REG_NONE; 3818 fence->obj = NULL; 3819 list_del_init(&fence->lru_list); 3820 } 3821} 3822 3823static int 3824i915_gem_object_flush_fence(struct drm_i915_gem_object *obj) 3825{ 3826 int ret; 3827 3828 if (obj->fenced_gpu_access) { 3829 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 3830 ret = i915_gem_flush_ring(obj->ring, 3831 0, obj->base.write_domain); 3832 if (ret) 3833 return ret; 3834 } 3835 3836 obj->fenced_gpu_access = false; 3837 } 3838 3839 if (obj->last_fenced_seqno) { 3840 ret = i915_wait_request(obj->ring, 3841 obj->last_fenced_seqno); 3842 if (ret) 3843 return ret; 3844 3845 obj->last_fenced_seqno = 0; 3846 } 3847 3848 /* Ensure that all CPU reads are completed before installing a fence 3849 * and all writes before removing the fence. 3850 */ 3851 if (obj->base.read_domains & I915_GEM_DOMAIN_GTT) 3852 mb(); 3853 3854 return 0; 3855} 3856 3857int 3858i915_gem_object_put_fence(struct drm_i915_gem_object *obj) 3859{ 3860 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 3861 int ret; 3862 3863 ret = i915_gem_object_flush_fence(obj); 3864 if (ret) 3865 return ret; 3866 3867 if (obj->fence_reg == I915_FENCE_REG_NONE) 3868 return 0; 3869 3870 i915_gem_object_update_fence(obj, 3871 &dev_priv->fence_regs[obj->fence_reg], 3872 false); 3873 i915_gem_object_fence_lost(obj); 3874 3875 return 0; 3876} 3877 3878static struct drm_i915_fence_reg * 3879i915_find_fence_reg(struct drm_device *dev) 3880{ 3881 struct drm_i915_private *dev_priv = dev->dev_private; 3882 struct drm_i915_fence_reg *reg, *avail; 3883 int i; 3884 3885 /* First try to find a free reg */ 3886 avail = NULL; 3887 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) { 3888 reg = &dev_priv->fence_regs[i]; 3889 if (!reg->obj) 3890 return reg; 3891 3892 if (!reg->pin_count) 3893 avail = reg; 3894 } 3895 3896 if (avail == NULL) 3897 return NULL; 3898 3899 /* None available, try to steal one or wait for a user to finish */ 3900 list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) { 3901 if (reg->pin_count) 3902 continue; 3903 3904 return reg; 3905 } 3906 3907 return NULL; 3908} 3909 3910int 3911i915_gem_object_get_fence(struct drm_i915_gem_object *obj) 3912{ 3913 struct drm_device *dev = obj->base.dev; 3914 struct drm_i915_private *dev_priv = dev->dev_private; 3915 bool enable = obj->tiling_mode != I915_TILING_NONE; 3916 struct drm_i915_fence_reg *reg; 3917 int ret; 3918 3919 /* Have we updated the tiling parameters upon the object and so 3920 * will need to serialise the write to the associated fence register? 3921 */ 3922 if (obj->fence_dirty) { 3923 ret = i915_gem_object_flush_fence(obj); 3924 if (ret) 3925 return ret; 3926 } 3927 3928 ret = 0; 3929 3930 if (obj->fence_reg != I915_FENCE_REG_NONE) { 3931 reg = &dev_priv->fence_regs[obj->fence_reg]; 3932 if (!obj->fence_dirty) { 3933 list_move_tail(®->lru_list, 3934 &dev_priv->mm.fence_list); 3935 return 0; 3936 } 3937 } else if (enable) { 3938 reg = i915_find_fence_reg(dev); 3939 if (reg == NULL) 3940 return -EDEADLK; 3941 3942 if (reg->obj) { 3943 struct drm_i915_gem_object *old = reg->obj; 3944 3945 ret = i915_gem_object_flush_fence(old); 3946 if (ret) 3947 return ret; 3948 3949 i915_gem_object_fence_lost(old); 3950 } 3951 } else 3952 return 0; 3953 3954 i915_gem_object_update_fence(obj, reg, enable); 3955 obj->fence_dirty = false; 3956 3957 return 0; 3958} 3959 3960int 3961i915_gem_init_object(struct drm_gem_object *obj) 3962{ 3963 3964 printf("i915_gem_init_object called\n"); 3965 return (0); 3966} 3967 3968static bool 3969i915_gem_object_is_inactive(struct drm_i915_gem_object *obj) 3970{ 3971 3972 return !obj->active; 3973} 3974 3975static void 3976i915_gem_retire_task_handler(void *arg, int pending) 3977{ 3978 drm_i915_private_t *dev_priv; 3979 struct drm_device *dev; 3980 struct intel_ring_buffer *ring; 3981 bool idle; 3982 int i; 3983 3984 dev_priv = arg; 3985 dev = dev_priv->dev; 3986 3987 /* Come back later if the device is busy... */ 3988 if (!sx_try_xlock(&dev->dev_struct_lock)) { 3989 taskqueue_enqueue_timeout(dev_priv->tq, 3990 &dev_priv->mm.retire_task, hz); 3991 return; 3992 } 3993 3994 CTR0(KTR_DRM, "retire_task"); 3995 3996 i915_gem_retire_requests(dev); 3997 3998 /* Send a periodic flush down the ring so we don't hold onto GEM 3999 * objects indefinitely. 4000 */ 4001 idle = true; 4002 for_each_ring(ring, dev_priv, i) { 4003 struct intel_ring_buffer *ring = &dev_priv->rings[i]; 4004 4005 if (!list_empty(&ring->gpu_write_list)) { 4006 struct drm_i915_gem_request *request; 4007 int ret; 4008 4009 ret = i915_gem_flush_ring(ring, 4010 0, I915_GEM_GPU_DOMAINS); 4011 request = malloc(sizeof(*request), DRM_I915_GEM, 4012 M_WAITOK | M_ZERO); 4013 if (ret || request == NULL || 4014 i915_add_request(ring, NULL, request)) 4015 free(request, DRM_I915_GEM); 4016 } 4017 4018 idle &= list_empty(&ring->request_list); 4019 } 4020 4021 if (!dev_priv->mm.suspended && !idle) 4022 taskqueue_enqueue_timeout(dev_priv->tq, 4023 &dev_priv->mm.retire_task, hz); 4024 4025 DRM_UNLOCK(dev); 4026} 4027 4028void 4029i915_gem_lastclose(struct drm_device *dev) 4030{ 4031 int ret; 4032 4033 if (drm_core_check_feature(dev, DRIVER_MODESET)) 4034 return; 4035 4036 ret = i915_gem_idle(dev); 4037 if (ret != 0) 4038 DRM_ERROR("failed to idle hardware: %d\n", ret); 4039} 4040 4041static int 4042i915_gem_init_phys_object(struct drm_device *dev, int id, int size, int align) 4043{ 4044 drm_i915_private_t *dev_priv; 4045 struct drm_i915_gem_phys_object *phys_obj; 4046 int ret; 4047 4048 dev_priv = dev->dev_private; 4049 if (dev_priv->mm.phys_objs[id - 1] != NULL || size == 0) 4050 return (0); 4051 4052 phys_obj = malloc(sizeof(struct drm_i915_gem_phys_object), DRM_I915_GEM, 4053 M_WAITOK | M_ZERO); 4054 4055 phys_obj->id = id; 4056 4057 phys_obj->handle = drm_pci_alloc(dev, size, align, BUS_SPACE_MAXADDR); 4058 if (phys_obj->handle == NULL) { 4059 ret = -ENOMEM; 4060 goto free_obj; 4061 } 4062 pmap_change_attr((vm_offset_t)phys_obj->handle->vaddr, 4063 size / PAGE_SIZE, PAT_WRITE_COMBINING); 4064 4065 dev_priv->mm.phys_objs[id - 1] = phys_obj; 4066 4067 return (0); 4068 4069free_obj: 4070 free(phys_obj, DRM_I915_GEM); 4071 return (ret); 4072} 4073 4074static void 4075i915_gem_free_phys_object(struct drm_device *dev, int id) 4076{ 4077 drm_i915_private_t *dev_priv; 4078 struct drm_i915_gem_phys_object *phys_obj; 4079 4080 dev_priv = dev->dev_private; 4081 if (dev_priv->mm.phys_objs[id - 1] == NULL) 4082 return; 4083 4084 phys_obj = dev_priv->mm.phys_objs[id - 1]; 4085 if (phys_obj->cur_obj != NULL) 4086 i915_gem_detach_phys_object(dev, phys_obj->cur_obj); 4087 4088 drm_pci_free(dev, phys_obj->handle); 4089 free(phys_obj, DRM_I915_GEM); 4090 dev_priv->mm.phys_objs[id - 1] = NULL; 4091} 4092 4093void 4094i915_gem_free_all_phys_object(struct drm_device *dev) 4095{ 4096 int i; 4097 4098 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++) 4099 i915_gem_free_phys_object(dev, i); 4100} 4101 4102void 4103i915_gem_detach_phys_object(struct drm_device *dev, 4104 struct drm_i915_gem_object *obj) 4105{ 4106 vm_page_t m; 4107 struct sf_buf *sf; 4108 char *vaddr, *dst; 4109 int i, page_count; 4110 4111 if (obj->phys_obj == NULL) 4112 return; 4113 vaddr = obj->phys_obj->handle->vaddr; 4114 4115 page_count = obj->base.size / PAGE_SIZE; 4116 VM_OBJECT_WLOCK(obj->base.vm_obj); 4117 for (i = 0; i < page_count; i++) { 4118 m = i915_gem_wire_page(obj->base.vm_obj, i, NULL); 4119 if (m == NULL) 4120 continue; /* XXX */ 4121 4122 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 4123 sf = sf_buf_alloc(m, 0); 4124 if (sf != NULL) { 4125 dst = (char *)sf_buf_kva(sf); 4126 memcpy(dst, vaddr + IDX_TO_OFF(i), PAGE_SIZE); 4127 sf_buf_free(sf); 4128 } 4129 drm_clflush_pages(&m, 1); 4130 4131 VM_OBJECT_WLOCK(obj->base.vm_obj); 4132 vm_page_reference(m); 4133 vm_page_lock(m); 4134 vm_page_dirty(m); 4135 vm_page_unwire(m, 0); 4136 vm_page_unlock(m); 4137 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 4138 } 4139 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 4140 intel_gtt_chipset_flush(); 4141 4142 obj->phys_obj->cur_obj = NULL; 4143 obj->phys_obj = NULL; 4144} 4145 4146int 4147i915_gem_attach_phys_object(struct drm_device *dev, 4148 struct drm_i915_gem_object *obj, int id, int align) 4149{ 4150 drm_i915_private_t *dev_priv; 4151 vm_page_t m; 4152 struct sf_buf *sf; 4153 char *dst, *src; 4154 int i, page_count, ret; 4155 4156 if (id > I915_MAX_PHYS_OBJECT) 4157 return (-EINVAL); 4158 4159 if (obj->phys_obj != NULL) { 4160 if (obj->phys_obj->id == id) 4161 return (0); 4162 i915_gem_detach_phys_object(dev, obj); 4163 } 4164 4165 dev_priv = dev->dev_private; 4166 if (dev_priv->mm.phys_objs[id - 1] == NULL) { 4167 ret = i915_gem_init_phys_object(dev, id, obj->base.size, align); 4168 if (ret != 0) { 4169 DRM_ERROR("failed to init phys object %d size: %zu\n", 4170 id, obj->base.size); 4171 return (ret); 4172 } 4173 } 4174 4175 /* bind to the object */ 4176 obj->phys_obj = dev_priv->mm.phys_objs[id - 1]; 4177 obj->phys_obj->cur_obj = obj; 4178 4179 page_count = obj->base.size / PAGE_SIZE; 4180 4181 VM_OBJECT_WLOCK(obj->base.vm_obj); 4182 ret = 0; 4183 for (i = 0; i < page_count; i++) { 4184 m = i915_gem_wire_page(obj->base.vm_obj, i, NULL); 4185 if (m == NULL) { 4186 ret = -EIO; 4187 break; 4188 } 4189 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 4190 sf = sf_buf_alloc(m, 0); 4191 src = (char *)sf_buf_kva(sf); 4192 dst = (char *)obj->phys_obj->handle->vaddr + IDX_TO_OFF(i); 4193 memcpy(dst, src, PAGE_SIZE); 4194 sf_buf_free(sf); 4195 4196 VM_OBJECT_WLOCK(obj->base.vm_obj); 4197 4198 vm_page_reference(m); 4199 vm_page_lock(m); 4200 vm_page_unwire(m, 0); 4201 vm_page_unlock(m); 4202 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 4203 } 4204 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 4205 4206 return (ret); 4207} 4208 4209static int 4210i915_gpu_is_active(struct drm_device *dev) 4211{ 4212 drm_i915_private_t *dev_priv; 4213 4214 dev_priv = dev->dev_private; 4215 return (!list_empty(&dev_priv->mm.flushing_list) || 4216 !list_empty(&dev_priv->mm.active_list)); 4217} 4218 4219static void 4220i915_gem_lowmem(void *arg) 4221{ 4222 struct drm_device *dev; 4223 struct drm_i915_private *dev_priv; 4224 struct drm_i915_gem_object *obj, *next; 4225 int cnt, cnt_fail, cnt_total; 4226 4227 dev = arg; 4228 dev_priv = dev->dev_private; 4229 4230 if (!sx_try_xlock(&dev->dev_struct_lock)) 4231 return; 4232 4233 CTR0(KTR_DRM, "gem_lowmem"); 4234 4235rescan: 4236 /* first scan for clean buffers */ 4237 i915_gem_retire_requests(dev); 4238 4239 cnt_total = cnt_fail = cnt = 0; 4240 4241 list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list, 4242 mm_list) { 4243 if (i915_gem_object_is_purgeable(obj)) { 4244 if (i915_gem_object_unbind(obj) != 0) 4245 cnt_total++; 4246 } else 4247 cnt_total++; 4248 } 4249 4250 /* second pass, evict/count anything still on the inactive list */ 4251 list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list, 4252 mm_list) { 4253 if (i915_gem_object_unbind(obj) == 0) 4254 cnt++; 4255 else 4256 cnt_fail++; 4257 } 4258 4259 if (cnt_fail > cnt_total / 100 && i915_gpu_is_active(dev)) { 4260 /* 4261 * We are desperate for pages, so as a last resort, wait 4262 * for the GPU to finish and discard whatever we can. 4263 * This has a dramatic impact to reduce the number of 4264 * OOM-killer events whilst running the GPU aggressively. 4265 */ 4266 if (i915_gpu_idle(dev) == 0) 4267 goto rescan; 4268 } 4269 DRM_UNLOCK(dev); 4270} 4271 4272void 4273i915_gem_unload(struct drm_device *dev) 4274{ 4275 struct drm_i915_private *dev_priv; 4276 4277 dev_priv = dev->dev_private; 4278 EVENTHANDLER_DEREGISTER(vm_lowmem, dev_priv->mm.i915_lowmem); 4279} 4280