1254885Sdumbbell/* 2254885Sdumbbell * Copyright 2009 Jerome Glisse. 3254885Sdumbbell * All Rights Reserved. 4254885Sdumbbell * 5254885Sdumbbell * Permission is hereby granted, free of charge, to any person obtaining a 6254885Sdumbbell * copy of this software and associated documentation files (the 7254885Sdumbbell * "Software"), to deal in the Software without restriction, including 8254885Sdumbbell * without limitation the rights to use, copy, modify, merge, publish, 9254885Sdumbbell * distribute, sub license, and/or sell copies of the Software, and to 10254885Sdumbbell * permit persons to whom the Software is furnished to do so, subject to 11254885Sdumbbell * the following conditions: 12254885Sdumbbell * 13254885Sdumbbell * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14254885Sdumbbell * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15254885Sdumbbell * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16254885Sdumbbell * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17254885Sdumbbell * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18254885Sdumbbell * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19254885Sdumbbell * USE OR OTHER DEALINGS IN THE SOFTWARE. 20254885Sdumbbell * 21254885Sdumbbell * The above copyright notice and this permission notice (including the 22254885Sdumbbell * next paragraph) shall be included in all copies or substantial portions 23254885Sdumbbell * of the Software. 24254885Sdumbbell * 25254885Sdumbbell */ 26254885Sdumbbell/* 27254885Sdumbbell * Authors: 28254885Sdumbbell * Jerome Glisse <glisse@freedesktop.org> 29254885Sdumbbell * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> 30254885Sdumbbell * Dave Airlie 31254885Sdumbbell */ 32254885Sdumbbell 33254885Sdumbbell#include <sys/cdefs.h> 34254885Sdumbbell__FBSDID("$FreeBSD$"); 35254885Sdumbbell 36254885Sdumbbell#include <dev/drm2/drmP.h> 37254885Sdumbbell#include <dev/drm2/radeon/radeon_drm.h> 38254885Sdumbbell#include "radeon.h" 39254885Sdumbbell#ifdef DUMBBELL_WIP 40254885Sdumbbell#include "radeon_trace.h" 41254885Sdumbbell#endif /* DUMBBELL_WIP */ 42254885Sdumbbell 43254885Sdumbbell 44254885Sdumbbellstatic void radeon_bo_clear_surface_reg(struct radeon_bo *bo); 45254885Sdumbbell 46254885Sdumbbell/* 47254885Sdumbbell * To exclude mutual BO access we rely on bo_reserve exclusion, as all 48254885Sdumbbell * function are calling it. 49254885Sdumbbell */ 50254885Sdumbbell 51254885Sdumbbellstatic void radeon_bo_clear_va(struct radeon_bo *bo) 52254885Sdumbbell{ 53254885Sdumbbell struct radeon_bo_va *bo_va, *tmp; 54254885Sdumbbell 55254885Sdumbbell list_for_each_entry_safe(bo_va, tmp, &bo->va, bo_list) { 56254885Sdumbbell /* remove from all vm address space */ 57254885Sdumbbell radeon_vm_bo_rmv(bo->rdev, bo_va); 58254885Sdumbbell } 59254885Sdumbbell} 60254885Sdumbbell 61254885Sdumbbellstatic void radeon_ttm_bo_destroy(struct ttm_buffer_object *tbo) 62254885Sdumbbell{ 63254885Sdumbbell struct radeon_bo *bo; 64254885Sdumbbell 65254885Sdumbbell bo = container_of(tbo, struct radeon_bo, tbo); 66254885Sdumbbell sx_xlock(&bo->rdev->gem.mutex); 67254885Sdumbbell list_del_init(&bo->list); 68254885Sdumbbell sx_xunlock(&bo->rdev->gem.mutex); 69254885Sdumbbell radeon_bo_clear_surface_reg(bo); 70254885Sdumbbell radeon_bo_clear_va(bo); 71254885Sdumbbell drm_gem_object_release(&bo->gem_base); 72254885Sdumbbell free(bo, DRM_MEM_DRIVER); 73254885Sdumbbell} 74254885Sdumbbell 75254885Sdumbbellbool radeon_ttm_bo_is_radeon_bo(struct ttm_buffer_object *bo) 76254885Sdumbbell{ 77254885Sdumbbell if (bo->destroy == &radeon_ttm_bo_destroy) 78254885Sdumbbell return true; 79254885Sdumbbell return false; 80254885Sdumbbell} 81254885Sdumbbell 82254885Sdumbbellvoid radeon_ttm_placement_from_domain(struct radeon_bo *rbo, u32 domain) 83254885Sdumbbell{ 84254885Sdumbbell u32 c = 0; 85254885Sdumbbell 86254885Sdumbbell rbo->placement.fpfn = 0; 87254885Sdumbbell rbo->placement.lpfn = 0; 88254885Sdumbbell rbo->placement.placement = rbo->placements; 89254885Sdumbbell rbo->placement.busy_placement = rbo->placements; 90254885Sdumbbell if (domain & RADEON_GEM_DOMAIN_VRAM) 91254885Sdumbbell rbo->placements[c++] = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED | 92254885Sdumbbell TTM_PL_FLAG_VRAM; 93254885Sdumbbell if (domain & RADEON_GEM_DOMAIN_GTT) { 94254885Sdumbbell if (rbo->rdev->flags & RADEON_IS_AGP) { 95254885Sdumbbell rbo->placements[c++] = TTM_PL_FLAG_WC | TTM_PL_FLAG_TT; 96254885Sdumbbell } else { 97254885Sdumbbell rbo->placements[c++] = TTM_PL_FLAG_CACHED | TTM_PL_FLAG_TT; 98254885Sdumbbell } 99254885Sdumbbell } 100254885Sdumbbell if (domain & RADEON_GEM_DOMAIN_CPU) { 101254885Sdumbbell if (rbo->rdev->flags & RADEON_IS_AGP) { 102254885Sdumbbell rbo->placements[c++] = TTM_PL_FLAG_WC | TTM_PL_FLAG_SYSTEM; 103254885Sdumbbell } else { 104254885Sdumbbell rbo->placements[c++] = TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM; 105254885Sdumbbell } 106254885Sdumbbell } 107254885Sdumbbell if (!c) 108254885Sdumbbell rbo->placements[c++] = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM; 109254885Sdumbbell rbo->placement.num_placement = c; 110254885Sdumbbell rbo->placement.num_busy_placement = c; 111254885Sdumbbell} 112254885Sdumbbell 113254885Sdumbbellint radeon_bo_create(struct radeon_device *rdev, 114254885Sdumbbell unsigned long size, int byte_align, bool kernel, u32 domain, 115254885Sdumbbell struct sg_table *sg, struct radeon_bo **bo_ptr) 116254885Sdumbbell{ 117254885Sdumbbell struct radeon_bo *bo; 118254885Sdumbbell enum ttm_bo_type type; 119254885Sdumbbell unsigned long page_align = roundup2(byte_align, PAGE_SIZE) >> PAGE_SHIFT; 120254885Sdumbbell size_t acc_size; 121254885Sdumbbell int r; 122254885Sdumbbell 123254885Sdumbbell size = roundup2(size, PAGE_SIZE); 124254885Sdumbbell 125254885Sdumbbell if (kernel) { 126254885Sdumbbell type = ttm_bo_type_kernel; 127254885Sdumbbell } else if (sg) { 128254885Sdumbbell type = ttm_bo_type_sg; 129254885Sdumbbell } else { 130254885Sdumbbell type = ttm_bo_type_device; 131254885Sdumbbell } 132254885Sdumbbell *bo_ptr = NULL; 133254885Sdumbbell 134254885Sdumbbell acc_size = ttm_bo_dma_acc_size(&rdev->mman.bdev, size, 135254885Sdumbbell sizeof(struct radeon_bo)); 136254885Sdumbbell 137254885Sdumbbell bo = malloc(sizeof(struct radeon_bo), 138254885Sdumbbell DRM_MEM_DRIVER, M_ZERO | M_WAITOK); 139254885Sdumbbell if (bo == NULL) 140254885Sdumbbell return -ENOMEM; 141254885Sdumbbell r = drm_gem_object_init(rdev->ddev, &bo->gem_base, size); 142254885Sdumbbell if (unlikely(r)) { 143254885Sdumbbell free(bo, DRM_MEM_DRIVER); 144254885Sdumbbell return r; 145254885Sdumbbell } 146254885Sdumbbell bo->rdev = rdev; 147254885Sdumbbell bo->gem_base.driver_private = NULL; 148254885Sdumbbell bo->surface_reg = -1; 149254885Sdumbbell INIT_LIST_HEAD(&bo->list); 150254885Sdumbbell INIT_LIST_HEAD(&bo->va); 151254885Sdumbbell radeon_ttm_placement_from_domain(bo, domain); 152254885Sdumbbell /* Kernel allocation are uninterruptible */ 153254885Sdumbbell sx_slock(&rdev->pm.mclk_lock); 154254885Sdumbbell r = ttm_bo_init(&rdev->mman.bdev, &bo->tbo, size, type, 155254885Sdumbbell &bo->placement, page_align, !kernel, NULL, 156254885Sdumbbell acc_size, sg, &radeon_ttm_bo_destroy); 157254885Sdumbbell sx_sunlock(&rdev->pm.mclk_lock); 158254885Sdumbbell if (unlikely(r != 0)) { 159254885Sdumbbell return r; 160254885Sdumbbell } 161254885Sdumbbell *bo_ptr = bo; 162254885Sdumbbell 163254885Sdumbbell#ifdef DUMBBELL_WIP 164254885Sdumbbell trace_radeon_bo_create(bo); 165254885Sdumbbell#endif /* DUMBBELL_WIP */ 166254885Sdumbbell 167254885Sdumbbell return 0; 168254885Sdumbbell} 169254885Sdumbbell 170254885Sdumbbellint radeon_bo_kmap(struct radeon_bo *bo, void **ptr) 171254885Sdumbbell{ 172254885Sdumbbell bool is_iomem; 173254885Sdumbbell int r; 174254885Sdumbbell 175254885Sdumbbell if (bo->kptr) { 176254885Sdumbbell if (ptr) { 177254885Sdumbbell *ptr = bo->kptr; 178254885Sdumbbell } 179254885Sdumbbell return 0; 180254885Sdumbbell } 181254885Sdumbbell r = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, &bo->kmap); 182254885Sdumbbell if (r) { 183254885Sdumbbell return r; 184254885Sdumbbell } 185254885Sdumbbell bo->kptr = ttm_kmap_obj_virtual(&bo->kmap, &is_iomem); 186254885Sdumbbell if (ptr) { 187254885Sdumbbell *ptr = bo->kptr; 188254885Sdumbbell } 189254885Sdumbbell radeon_bo_check_tiling(bo, 0, 0); 190254885Sdumbbell return 0; 191254885Sdumbbell} 192254885Sdumbbell 193254885Sdumbbellvoid radeon_bo_kunmap(struct radeon_bo *bo) 194254885Sdumbbell{ 195254885Sdumbbell if (bo->kptr == NULL) 196254885Sdumbbell return; 197254885Sdumbbell bo->kptr = NULL; 198254885Sdumbbell radeon_bo_check_tiling(bo, 0, 0); 199254885Sdumbbell ttm_bo_kunmap(&bo->kmap); 200254885Sdumbbell} 201254885Sdumbbell 202254885Sdumbbellvoid radeon_bo_unref(struct radeon_bo **bo) 203254885Sdumbbell{ 204254885Sdumbbell struct ttm_buffer_object *tbo; 205254885Sdumbbell struct radeon_device *rdev; 206254885Sdumbbell 207254885Sdumbbell if ((*bo) == NULL) 208254885Sdumbbell return; 209254885Sdumbbell rdev = (*bo)->rdev; 210254885Sdumbbell tbo = &((*bo)->tbo); 211254885Sdumbbell sx_slock(&rdev->pm.mclk_lock); 212254885Sdumbbell ttm_bo_unref(&tbo); 213254885Sdumbbell sx_sunlock(&rdev->pm.mclk_lock); 214254885Sdumbbell if (tbo == NULL) 215254885Sdumbbell *bo = NULL; 216254885Sdumbbell} 217254885Sdumbbell 218254885Sdumbbellint radeon_bo_pin_restricted(struct radeon_bo *bo, u32 domain, u64 max_offset, 219254885Sdumbbell u64 *gpu_addr) 220254885Sdumbbell{ 221254885Sdumbbell int r, i; 222254885Sdumbbell 223254885Sdumbbell if (bo->pin_count) { 224254885Sdumbbell bo->pin_count++; 225254885Sdumbbell if (gpu_addr) 226254885Sdumbbell *gpu_addr = radeon_bo_gpu_offset(bo); 227254885Sdumbbell 228254885Sdumbbell if (max_offset != 0) { 229254885Sdumbbell u64 domain_start; 230254885Sdumbbell 231254885Sdumbbell if (domain == RADEON_GEM_DOMAIN_VRAM) 232254885Sdumbbell domain_start = bo->rdev->mc.vram_start; 233254885Sdumbbell else 234254885Sdumbbell domain_start = bo->rdev->mc.gtt_start; 235254885Sdumbbell if (max_offset < (radeon_bo_gpu_offset(bo) - domain_start)) { 236254885Sdumbbell DRM_ERROR("radeon_bo_pin_restricted: " 237254885Sdumbbell "max_offset(%ju) < " 238254885Sdumbbell "(radeon_bo_gpu_offset(%ju) - " 239254885Sdumbbell "domain_start(%ju)", 240254885Sdumbbell (uintmax_t)max_offset, (uintmax_t)radeon_bo_gpu_offset(bo), 241254885Sdumbbell (uintmax_t)domain_start); 242254885Sdumbbell } 243254885Sdumbbell } 244254885Sdumbbell 245254885Sdumbbell return 0; 246254885Sdumbbell } 247254885Sdumbbell radeon_ttm_placement_from_domain(bo, domain); 248254885Sdumbbell if (domain == RADEON_GEM_DOMAIN_VRAM) { 249254885Sdumbbell /* force to pin into visible video ram */ 250254885Sdumbbell bo->placement.lpfn = bo->rdev->mc.visible_vram_size >> PAGE_SHIFT; 251254885Sdumbbell } 252254885Sdumbbell if (max_offset) { 253254885Sdumbbell u64 lpfn = max_offset >> PAGE_SHIFT; 254254885Sdumbbell 255254885Sdumbbell if (!bo->placement.lpfn) 256254885Sdumbbell bo->placement.lpfn = bo->rdev->mc.gtt_size >> PAGE_SHIFT; 257254885Sdumbbell 258254885Sdumbbell if (lpfn < bo->placement.lpfn) 259254885Sdumbbell bo->placement.lpfn = lpfn; 260254885Sdumbbell } 261254885Sdumbbell for (i = 0; i < bo->placement.num_placement; i++) 262254885Sdumbbell bo->placements[i] |= TTM_PL_FLAG_NO_EVICT; 263254885Sdumbbell r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false); 264254885Sdumbbell if (likely(r == 0)) { 265254885Sdumbbell bo->pin_count = 1; 266254885Sdumbbell if (gpu_addr != NULL) 267254885Sdumbbell *gpu_addr = radeon_bo_gpu_offset(bo); 268254885Sdumbbell } 269254885Sdumbbell if (unlikely(r != 0)) 270254885Sdumbbell dev_err(bo->rdev->dev, "%p pin failed\n", bo); 271254885Sdumbbell return r; 272254885Sdumbbell} 273254885Sdumbbell 274254885Sdumbbellint radeon_bo_pin(struct radeon_bo *bo, u32 domain, u64 *gpu_addr) 275254885Sdumbbell{ 276254885Sdumbbell return radeon_bo_pin_restricted(bo, domain, 0, gpu_addr); 277254885Sdumbbell} 278254885Sdumbbell 279254885Sdumbbellint radeon_bo_unpin(struct radeon_bo *bo) 280254885Sdumbbell{ 281254885Sdumbbell int r, i; 282254885Sdumbbell 283254885Sdumbbell if (!bo->pin_count) { 284254885Sdumbbell dev_warn(bo->rdev->dev, "%p unpin not necessary\n", bo); 285254885Sdumbbell return 0; 286254885Sdumbbell } 287254885Sdumbbell bo->pin_count--; 288254885Sdumbbell if (bo->pin_count) 289254885Sdumbbell return 0; 290254885Sdumbbell for (i = 0; i < bo->placement.num_placement; i++) 291254885Sdumbbell bo->placements[i] &= ~TTM_PL_FLAG_NO_EVICT; 292254885Sdumbbell r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false); 293254885Sdumbbell if (unlikely(r != 0)) 294254885Sdumbbell dev_err(bo->rdev->dev, "%p validate failed for unpin\n", bo); 295254885Sdumbbell return r; 296254885Sdumbbell} 297254885Sdumbbell 298254885Sdumbbellint radeon_bo_evict_vram(struct radeon_device *rdev) 299254885Sdumbbell{ 300254885Sdumbbell /* late 2.6.33 fix IGP hibernate - we need pm ops to do this correct */ 301254885Sdumbbell if (0 && (rdev->flags & RADEON_IS_IGP)) { 302254885Sdumbbell if (rdev->mc.igp_sideport_enabled == false) 303254885Sdumbbell /* Useless to evict on IGP chips */ 304254885Sdumbbell return 0; 305254885Sdumbbell } 306254885Sdumbbell return ttm_bo_evict_mm(&rdev->mman.bdev, TTM_PL_VRAM); 307254885Sdumbbell} 308254885Sdumbbell 309254885Sdumbbellvoid radeon_bo_force_delete(struct radeon_device *rdev) 310254885Sdumbbell{ 311254885Sdumbbell struct radeon_bo *bo, *n; 312254885Sdumbbell 313254885Sdumbbell if (list_empty(&rdev->gem.objects)) { 314254885Sdumbbell return; 315254885Sdumbbell } 316254885Sdumbbell dev_err(rdev->dev, "Userspace still has active objects !\n"); 317254885Sdumbbell list_for_each_entry_safe(bo, n, &rdev->gem.objects, list) { 318254885Sdumbbell dev_err(rdev->dev, "%p %p %lu %lu force free\n", 319254885Sdumbbell &bo->gem_base, bo, (unsigned long)bo->gem_base.size, 320254885Sdumbbell *((unsigned long *)&bo->gem_base.refcount)); 321254885Sdumbbell sx_xlock(&bo->rdev->gem.mutex); 322254885Sdumbbell list_del_init(&bo->list); 323254885Sdumbbell sx_xunlock(&bo->rdev->gem.mutex); 324254885Sdumbbell /* this should unref the ttm bo */ 325254885Sdumbbell drm_gem_object_unreference(&bo->gem_base); 326254885Sdumbbell } 327254885Sdumbbell} 328254885Sdumbbell 329254885Sdumbbellint radeon_bo_init(struct radeon_device *rdev) 330254885Sdumbbell{ 331254885Sdumbbell /* Add an MTRR for the VRAM */ 332254885Sdumbbell rdev->mc.vram_mtrr = drm_mtrr_add(rdev->mc.aper_base, rdev->mc.aper_size, 333254885Sdumbbell DRM_MTRR_WC); 334254885Sdumbbell DRM_INFO("Detected VRAM RAM=%juM, BAR=%juM\n", 335254885Sdumbbell (uintmax_t)rdev->mc.mc_vram_size >> 20, 336254885Sdumbbell (uintmax_t)rdev->mc.aper_size >> 20); 337254885Sdumbbell DRM_INFO("RAM width %dbits %cDR\n", 338254885Sdumbbell rdev->mc.vram_width, rdev->mc.vram_is_ddr ? 'D' : 'S'); 339254885Sdumbbell return radeon_ttm_init(rdev); 340254885Sdumbbell} 341254885Sdumbbell 342254885Sdumbbellvoid radeon_bo_fini(struct radeon_device *rdev) 343254885Sdumbbell{ 344254885Sdumbbell radeon_ttm_fini(rdev); 345254885Sdumbbell} 346254885Sdumbbell 347254885Sdumbbellvoid radeon_bo_list_add_object(struct radeon_bo_list *lobj, 348254885Sdumbbell struct list_head *head) 349254885Sdumbbell{ 350254885Sdumbbell if (lobj->wdomain) { 351254885Sdumbbell list_add(&lobj->tv.head, head); 352254885Sdumbbell } else { 353254885Sdumbbell list_add_tail(&lobj->tv.head, head); 354254885Sdumbbell } 355254885Sdumbbell} 356254885Sdumbbell 357254885Sdumbbellint radeon_bo_list_validate(struct list_head *head) 358254885Sdumbbell{ 359254885Sdumbbell struct radeon_bo_list *lobj; 360254885Sdumbbell struct radeon_bo *bo; 361254885Sdumbbell u32 domain; 362254885Sdumbbell int r; 363254885Sdumbbell 364254885Sdumbbell r = ttm_eu_reserve_buffers(head); 365254885Sdumbbell if (unlikely(r != 0)) { 366254885Sdumbbell return r; 367254885Sdumbbell } 368254885Sdumbbell list_for_each_entry(lobj, head, tv.head) { 369254885Sdumbbell bo = lobj->bo; 370254885Sdumbbell if (!bo->pin_count) { 371254885Sdumbbell domain = lobj->wdomain ? lobj->wdomain : lobj->rdomain; 372254885Sdumbbell 373254885Sdumbbell retry: 374254885Sdumbbell radeon_ttm_placement_from_domain(bo, domain); 375254885Sdumbbell r = ttm_bo_validate(&bo->tbo, &bo->placement, 376254885Sdumbbell true, false); 377254885Sdumbbell if (unlikely(r)) { 378254885Sdumbbell if (r != -ERESTARTSYS && domain == RADEON_GEM_DOMAIN_VRAM) { 379254885Sdumbbell domain |= RADEON_GEM_DOMAIN_GTT; 380254885Sdumbbell goto retry; 381254885Sdumbbell } 382254885Sdumbbell return r; 383254885Sdumbbell } 384254885Sdumbbell } 385254885Sdumbbell lobj->gpu_offset = radeon_bo_gpu_offset(bo); 386254885Sdumbbell lobj->tiling_flags = bo->tiling_flags; 387254885Sdumbbell } 388254885Sdumbbell return 0; 389254885Sdumbbell} 390254885Sdumbbell 391254885Sdumbbell#ifdef DUMBBELL_WIP 392254885Sdumbbellint radeon_bo_fbdev_mmap(struct radeon_bo *bo, 393254885Sdumbbell struct vm_area_struct *vma) 394254885Sdumbbell{ 395254885Sdumbbell return ttm_fbdev_mmap(vma, &bo->tbo); 396254885Sdumbbell} 397254885Sdumbbell#endif /* DUMBBELL_WIP */ 398254885Sdumbbell 399254885Sdumbbellint radeon_bo_get_surface_reg(struct radeon_bo *bo) 400254885Sdumbbell{ 401254885Sdumbbell struct radeon_device *rdev = bo->rdev; 402254885Sdumbbell struct radeon_surface_reg *reg; 403254885Sdumbbell struct radeon_bo *old_object; 404254885Sdumbbell int steal; 405254885Sdumbbell int i; 406254885Sdumbbell 407254885Sdumbbell KASSERT(radeon_bo_is_reserved(bo), 408254885Sdumbbell ("radeon_bo_get_surface_reg: radeon_bo is not reserved")); 409254885Sdumbbell 410254885Sdumbbell if (!bo->tiling_flags) 411254885Sdumbbell return 0; 412254885Sdumbbell 413254885Sdumbbell if (bo->surface_reg >= 0) { 414254885Sdumbbell reg = &rdev->surface_regs[bo->surface_reg]; 415254885Sdumbbell i = bo->surface_reg; 416254885Sdumbbell goto out; 417254885Sdumbbell } 418254885Sdumbbell 419254885Sdumbbell steal = -1; 420254885Sdumbbell for (i = 0; i < RADEON_GEM_MAX_SURFACES; i++) { 421254885Sdumbbell 422254885Sdumbbell reg = &rdev->surface_regs[i]; 423254885Sdumbbell if (!reg->bo) 424254885Sdumbbell break; 425254885Sdumbbell 426254885Sdumbbell old_object = reg->bo; 427254885Sdumbbell if (old_object->pin_count == 0) 428254885Sdumbbell steal = i; 429254885Sdumbbell } 430254885Sdumbbell 431254885Sdumbbell /* if we are all out */ 432254885Sdumbbell if (i == RADEON_GEM_MAX_SURFACES) { 433254885Sdumbbell if (steal == -1) 434254885Sdumbbell return -ENOMEM; 435254885Sdumbbell /* find someone with a surface reg and nuke their BO */ 436254885Sdumbbell reg = &rdev->surface_regs[steal]; 437254885Sdumbbell old_object = reg->bo; 438254885Sdumbbell /* blow away the mapping */ 439254885Sdumbbell DRM_DEBUG("stealing surface reg %d from %p\n", steal, old_object); 440254885Sdumbbell ttm_bo_unmap_virtual(&old_object->tbo); 441254885Sdumbbell old_object->surface_reg = -1; 442254885Sdumbbell i = steal; 443254885Sdumbbell } 444254885Sdumbbell 445254885Sdumbbell bo->surface_reg = i; 446254885Sdumbbell reg->bo = bo; 447254885Sdumbbell 448254885Sdumbbellout: 449254885Sdumbbell radeon_set_surface_reg(rdev, i, bo->tiling_flags, bo->pitch, 450254885Sdumbbell bo->tbo.mem.start << PAGE_SHIFT, 451254885Sdumbbell bo->tbo.num_pages << PAGE_SHIFT); 452254885Sdumbbell return 0; 453254885Sdumbbell} 454254885Sdumbbell 455254885Sdumbbellstatic void radeon_bo_clear_surface_reg(struct radeon_bo *bo) 456254885Sdumbbell{ 457254885Sdumbbell struct radeon_device *rdev = bo->rdev; 458254885Sdumbbell struct radeon_surface_reg *reg; 459254885Sdumbbell 460254885Sdumbbell if (bo->surface_reg == -1) 461254885Sdumbbell return; 462254885Sdumbbell 463254885Sdumbbell reg = &rdev->surface_regs[bo->surface_reg]; 464254885Sdumbbell radeon_clear_surface_reg(rdev, bo->surface_reg); 465254885Sdumbbell 466254885Sdumbbell reg->bo = NULL; 467254885Sdumbbell bo->surface_reg = -1; 468254885Sdumbbell} 469254885Sdumbbell 470254885Sdumbbellint radeon_bo_set_tiling_flags(struct radeon_bo *bo, 471254885Sdumbbell uint32_t tiling_flags, uint32_t pitch) 472254885Sdumbbell{ 473254885Sdumbbell struct radeon_device *rdev = bo->rdev; 474254885Sdumbbell int r; 475254885Sdumbbell 476254885Sdumbbell if (rdev->family >= CHIP_CEDAR) { 477254885Sdumbbell unsigned bankw, bankh, mtaspect, tilesplit, stilesplit; 478254885Sdumbbell 479254885Sdumbbell bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK; 480254885Sdumbbell bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK; 481254885Sdumbbell mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK; 482254885Sdumbbell tilesplit = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK; 483254885Sdumbbell stilesplit = (tiling_flags >> RADEON_TILING_EG_STENCIL_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_STENCIL_TILE_SPLIT_MASK; 484254885Sdumbbell switch (bankw) { 485254885Sdumbbell case 0: 486254885Sdumbbell case 1: 487254885Sdumbbell case 2: 488254885Sdumbbell case 4: 489254885Sdumbbell case 8: 490254885Sdumbbell break; 491254885Sdumbbell default: 492254885Sdumbbell return -EINVAL; 493254885Sdumbbell } 494254885Sdumbbell switch (bankh) { 495254885Sdumbbell case 0: 496254885Sdumbbell case 1: 497254885Sdumbbell case 2: 498254885Sdumbbell case 4: 499254885Sdumbbell case 8: 500254885Sdumbbell break; 501254885Sdumbbell default: 502254885Sdumbbell return -EINVAL; 503254885Sdumbbell } 504254885Sdumbbell switch (mtaspect) { 505254885Sdumbbell case 0: 506254885Sdumbbell case 1: 507254885Sdumbbell case 2: 508254885Sdumbbell case 4: 509254885Sdumbbell case 8: 510254885Sdumbbell break; 511254885Sdumbbell default: 512254885Sdumbbell return -EINVAL; 513254885Sdumbbell } 514254885Sdumbbell if (tilesplit > 6) { 515254885Sdumbbell return -EINVAL; 516254885Sdumbbell } 517254885Sdumbbell if (stilesplit > 6) { 518254885Sdumbbell return -EINVAL; 519254885Sdumbbell } 520254885Sdumbbell } 521254885Sdumbbell r = radeon_bo_reserve(bo, false); 522254885Sdumbbell if (unlikely(r != 0)) 523254885Sdumbbell return r; 524254885Sdumbbell bo->tiling_flags = tiling_flags; 525254885Sdumbbell bo->pitch = pitch; 526254885Sdumbbell radeon_bo_unreserve(bo); 527254885Sdumbbell return 0; 528254885Sdumbbell} 529254885Sdumbbell 530254885Sdumbbellvoid radeon_bo_get_tiling_flags(struct radeon_bo *bo, 531254885Sdumbbell uint32_t *tiling_flags, 532254885Sdumbbell uint32_t *pitch) 533254885Sdumbbell{ 534254885Sdumbbell KASSERT(radeon_bo_is_reserved(bo), 535254885Sdumbbell ("radeon_bo_get_tiling_flags: radeon_bo is not reserved")); 536254885Sdumbbell if (tiling_flags) 537254885Sdumbbell *tiling_flags = bo->tiling_flags; 538254885Sdumbbell if (pitch) 539254885Sdumbbell *pitch = bo->pitch; 540254885Sdumbbell} 541254885Sdumbbell 542254885Sdumbbellint radeon_bo_check_tiling(struct radeon_bo *bo, bool has_moved, 543254885Sdumbbell bool force_drop) 544254885Sdumbbell{ 545254885Sdumbbell KASSERT((radeon_bo_is_reserved(bo) || force_drop), 546254885Sdumbbell ("radeon_bo_check_tiling: radeon_bo is not reserved && !force_drop")); 547254885Sdumbbell 548254885Sdumbbell if (!(bo->tiling_flags & RADEON_TILING_SURFACE)) 549254885Sdumbbell return 0; 550254885Sdumbbell 551254885Sdumbbell if (force_drop) { 552254885Sdumbbell radeon_bo_clear_surface_reg(bo); 553254885Sdumbbell return 0; 554254885Sdumbbell } 555254885Sdumbbell 556254885Sdumbbell if (bo->tbo.mem.mem_type != TTM_PL_VRAM) { 557254885Sdumbbell if (!has_moved) 558254885Sdumbbell return 0; 559254885Sdumbbell 560254885Sdumbbell if (bo->surface_reg >= 0) 561254885Sdumbbell radeon_bo_clear_surface_reg(bo); 562254885Sdumbbell return 0; 563254885Sdumbbell } 564254885Sdumbbell 565254885Sdumbbell if ((bo->surface_reg >= 0) && !has_moved) 566254885Sdumbbell return 0; 567254885Sdumbbell 568254885Sdumbbell return radeon_bo_get_surface_reg(bo); 569254885Sdumbbell} 570254885Sdumbbell 571254885Sdumbbellvoid radeon_bo_move_notify(struct ttm_buffer_object *bo, 572254885Sdumbbell struct ttm_mem_reg *mem) 573254885Sdumbbell{ 574254885Sdumbbell struct radeon_bo *rbo; 575254885Sdumbbell if (!radeon_ttm_bo_is_radeon_bo(bo)) 576254885Sdumbbell return; 577254885Sdumbbell rbo = container_of(bo, struct radeon_bo, tbo); 578254885Sdumbbell radeon_bo_check_tiling(rbo, 0, 1); 579254885Sdumbbell radeon_vm_bo_invalidate(rbo->rdev, rbo); 580254885Sdumbbell} 581254885Sdumbbell 582254885Sdumbbellint radeon_bo_fault_reserve_notify(struct ttm_buffer_object *bo) 583254885Sdumbbell{ 584254885Sdumbbell struct radeon_device *rdev; 585254885Sdumbbell struct radeon_bo *rbo; 586254885Sdumbbell unsigned long offset, size; 587254885Sdumbbell int r; 588254885Sdumbbell 589254885Sdumbbell if (!radeon_ttm_bo_is_radeon_bo(bo)) 590254885Sdumbbell return 0; 591254885Sdumbbell rbo = container_of(bo, struct radeon_bo, tbo); 592254885Sdumbbell radeon_bo_check_tiling(rbo, 0, 0); 593254885Sdumbbell rdev = rbo->rdev; 594254885Sdumbbell if (bo->mem.mem_type == TTM_PL_VRAM) { 595254885Sdumbbell size = bo->mem.num_pages << PAGE_SHIFT; 596254885Sdumbbell offset = bo->mem.start << PAGE_SHIFT; 597254885Sdumbbell if ((offset + size) > rdev->mc.visible_vram_size) { 598254885Sdumbbell /* hurrah the memory is not visible ! */ 599254885Sdumbbell radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_VRAM); 600254885Sdumbbell rbo->placement.lpfn = rdev->mc.visible_vram_size >> PAGE_SHIFT; 601254885Sdumbbell r = ttm_bo_validate(bo, &rbo->placement, false, false); 602254885Sdumbbell if (unlikely(r != 0)) 603254885Sdumbbell return r; 604254885Sdumbbell offset = bo->mem.start << PAGE_SHIFT; 605254885Sdumbbell /* this should not happen */ 606254885Sdumbbell if ((offset + size) > rdev->mc.visible_vram_size) 607254885Sdumbbell return -EINVAL; 608254885Sdumbbell } 609254885Sdumbbell } 610254885Sdumbbell return 0; 611254885Sdumbbell} 612254885Sdumbbell 613254885Sdumbbellint radeon_bo_wait(struct radeon_bo *bo, u32 *mem_type, bool no_wait) 614254885Sdumbbell{ 615254885Sdumbbell int r; 616254885Sdumbbell 617254885Sdumbbell r = ttm_bo_reserve(&bo->tbo, true, no_wait, false, 0); 618254885Sdumbbell if (unlikely(r != 0)) 619254885Sdumbbell return r; 620254885Sdumbbell mtx_lock(&bo->tbo.bdev->fence_lock); 621254885Sdumbbell if (mem_type) 622254885Sdumbbell *mem_type = bo->tbo.mem.mem_type; 623254885Sdumbbell if (bo->tbo.sync_obj) 624254885Sdumbbell r = ttm_bo_wait(&bo->tbo, true, true, no_wait); 625254885Sdumbbell mtx_unlock(&bo->tbo.bdev->fence_lock); 626254885Sdumbbell ttm_bo_unreserve(&bo->tbo); 627254885Sdumbbell return r; 628254885Sdumbbell} 629254885Sdumbbell 630254885Sdumbbell 631254885Sdumbbell/** 632254885Sdumbbell * radeon_bo_reserve - reserve bo 633254885Sdumbbell * @bo: bo structure 634254885Sdumbbell * @no_intr: don't return -ERESTARTSYS on pending signal 635254885Sdumbbell * 636254885Sdumbbell * Returns: 637254885Sdumbbell * -ERESTARTSYS: A wait for the buffer to become unreserved was interrupted by 638254885Sdumbbell * a signal. Release all buffer reservations and return to user-space. 639254885Sdumbbell */ 640254885Sdumbbellint radeon_bo_reserve(struct radeon_bo *bo, bool no_intr) 641254885Sdumbbell{ 642254885Sdumbbell int r; 643254885Sdumbbell 644254885Sdumbbell r = ttm_bo_reserve(&bo->tbo, !no_intr, false, false, 0); 645254885Sdumbbell if (unlikely(r != 0)) { 646254885Sdumbbell if (r != -ERESTARTSYS) 647254885Sdumbbell dev_err(bo->rdev->dev, "%p reserve failed\n", bo); 648254885Sdumbbell return r; 649254885Sdumbbell } 650254885Sdumbbell return 0; 651254885Sdumbbell} 652