1254885Sdumbbell/* 2254885Sdumbbell * Copyright 2009 Jerome Glisse. 3254885Sdumbbell * All Rights Reserved. 4254885Sdumbbell * 5254885Sdumbbell * Permission is hereby granted, free of charge, to any person obtaining a 6254885Sdumbbell * copy of this software and associated documentation files (the 7254885Sdumbbell * "Software"), to deal in the Software without restriction, including 8254885Sdumbbell * without limitation the rights to use, copy, modify, merge, publish, 9254885Sdumbbell * distribute, sub license, and/or sell copies of the Software, and to 10254885Sdumbbell * permit persons to whom the Software is furnished to do so, subject to 11254885Sdumbbell * the following conditions: 12254885Sdumbbell * 13254885Sdumbbell * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14254885Sdumbbell * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15254885Sdumbbell * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16254885Sdumbbell * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17254885Sdumbbell * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18254885Sdumbbell * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19254885Sdumbbell * USE OR OTHER DEALINGS IN THE SOFTWARE. 20254885Sdumbbell * 21254885Sdumbbell * The above copyright notice and this permission notice (including the 22254885Sdumbbell * next paragraph) shall be included in all copies or substantial portions 23254885Sdumbbell * of the Software. 24254885Sdumbbell * 25254885Sdumbbell */ 26254885Sdumbbell/* 27254885Sdumbbell * Authors: 28254885Sdumbbell * Jerome Glisse <glisse@freedesktop.org> 29254885Sdumbbell * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> 30254885Sdumbbell * Dave Airlie 31254885Sdumbbell */ 32254885Sdumbbell 33254885Sdumbbell#include <sys/cdefs.h> 34254885Sdumbbell__FBSDID("$FreeBSD$"); 35254885Sdumbbell 36254885Sdumbbell#include <dev/drm2/ttm/ttm_bo_api.h> 37254885Sdumbbell#include <dev/drm2/ttm/ttm_bo_driver.h> 38254885Sdumbbell#include <dev/drm2/ttm/ttm_placement.h> 39254885Sdumbbell#include <dev/drm2/ttm/ttm_module.h> 40254885Sdumbbell#include <dev/drm2/ttm/ttm_page_alloc.h> 41254885Sdumbbell#include <dev/drm2/drmP.h> 42254885Sdumbbell#include <dev/drm2/radeon/radeon_drm.h> 43254885Sdumbbell#include "radeon_reg.h" 44254885Sdumbbell#include "radeon.h" 45254885Sdumbbell 46254885Sdumbbell#define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT) 47254885Sdumbbell 48254885Sdumbbellstatic int radeon_ttm_debugfs_init(struct radeon_device *rdev); 49254885Sdumbbell 50254885Sdumbbellstatic struct radeon_device *radeon_get_rdev(struct ttm_bo_device *bdev) 51254885Sdumbbell{ 52254885Sdumbbell struct radeon_mman *mman; 53254885Sdumbbell struct radeon_device *rdev; 54254885Sdumbbell 55254885Sdumbbell mman = container_of(bdev, struct radeon_mman, bdev); 56254885Sdumbbell rdev = container_of(mman, struct radeon_device, mman); 57254885Sdumbbell return rdev; 58254885Sdumbbell} 59254885Sdumbbell 60254885Sdumbbell 61254885Sdumbbell/* 62254885Sdumbbell * Global memory. 63254885Sdumbbell */ 64254885Sdumbbellstatic int radeon_ttm_mem_global_init(struct drm_global_reference *ref) 65254885Sdumbbell{ 66254885Sdumbbell return ttm_mem_global_init(ref->object); 67254885Sdumbbell} 68254885Sdumbbell 69254885Sdumbbellstatic void radeon_ttm_mem_global_release(struct drm_global_reference *ref) 70254885Sdumbbell{ 71254885Sdumbbell ttm_mem_global_release(ref->object); 72254885Sdumbbell} 73254885Sdumbbell 74254885Sdumbbellstatic int radeon_ttm_global_init(struct radeon_device *rdev) 75254885Sdumbbell{ 76254885Sdumbbell struct drm_global_reference *global_ref; 77254885Sdumbbell int r; 78254885Sdumbbell 79254885Sdumbbell rdev->mman.mem_global_referenced = false; 80254885Sdumbbell global_ref = &rdev->mman.mem_global_ref; 81254885Sdumbbell global_ref->global_type = DRM_GLOBAL_TTM_MEM; 82254885Sdumbbell global_ref->size = sizeof(struct ttm_mem_global); 83254885Sdumbbell global_ref->init = &radeon_ttm_mem_global_init; 84254885Sdumbbell global_ref->release = &radeon_ttm_mem_global_release; 85254885Sdumbbell r = drm_global_item_ref(global_ref); 86254885Sdumbbell if (r != 0) { 87254885Sdumbbell DRM_ERROR("Failed setting up TTM memory accounting " 88254885Sdumbbell "subsystem.\n"); 89254885Sdumbbell return r; 90254885Sdumbbell } 91254885Sdumbbell 92254885Sdumbbell rdev->mman.bo_global_ref.mem_glob = 93254885Sdumbbell rdev->mman.mem_global_ref.object; 94254885Sdumbbell global_ref = &rdev->mman.bo_global_ref.ref; 95254885Sdumbbell global_ref->global_type = DRM_GLOBAL_TTM_BO; 96254885Sdumbbell global_ref->size = sizeof(struct ttm_bo_global); 97254885Sdumbbell global_ref->init = &ttm_bo_global_init; 98254885Sdumbbell global_ref->release = &ttm_bo_global_release; 99254885Sdumbbell r = drm_global_item_ref(global_ref); 100254885Sdumbbell if (r != 0) { 101254885Sdumbbell DRM_ERROR("Failed setting up TTM BO subsystem.\n"); 102254885Sdumbbell drm_global_item_unref(&rdev->mman.mem_global_ref); 103254885Sdumbbell return r; 104254885Sdumbbell } 105254885Sdumbbell 106254885Sdumbbell rdev->mman.mem_global_referenced = true; 107254885Sdumbbell return 0; 108254885Sdumbbell} 109254885Sdumbbell 110254885Sdumbbellstatic void radeon_ttm_global_fini(struct radeon_device *rdev) 111254885Sdumbbell{ 112254885Sdumbbell if (rdev->mman.mem_global_referenced) { 113254885Sdumbbell drm_global_item_unref(&rdev->mman.bo_global_ref.ref); 114254885Sdumbbell drm_global_item_unref(&rdev->mman.mem_global_ref); 115254885Sdumbbell rdev->mman.mem_global_referenced = false; 116254885Sdumbbell } 117254885Sdumbbell} 118254885Sdumbbell 119254885Sdumbbellstatic int radeon_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags) 120254885Sdumbbell{ 121254885Sdumbbell return 0; 122254885Sdumbbell} 123254885Sdumbbell 124254885Sdumbbellstatic int radeon_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, 125254885Sdumbbell struct ttm_mem_type_manager *man) 126254885Sdumbbell{ 127254885Sdumbbell struct radeon_device *rdev; 128254885Sdumbbell 129254885Sdumbbell rdev = radeon_get_rdev(bdev); 130254885Sdumbbell 131254885Sdumbbell switch (type) { 132254885Sdumbbell case TTM_PL_SYSTEM: 133254885Sdumbbell /* System memory */ 134254885Sdumbbell man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; 135254885Sdumbbell man->available_caching = TTM_PL_MASK_CACHING; 136254885Sdumbbell man->default_caching = TTM_PL_FLAG_CACHED; 137254885Sdumbbell break; 138254885Sdumbbell case TTM_PL_TT: 139254885Sdumbbell man->func = &ttm_bo_manager_func; 140254885Sdumbbell man->gpu_offset = rdev->mc.gtt_start; 141254885Sdumbbell man->available_caching = TTM_PL_MASK_CACHING; 142254885Sdumbbell man->default_caching = TTM_PL_FLAG_CACHED; 143254885Sdumbbell man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA; 144254885Sdumbbell#if __OS_HAS_AGP 145254885Sdumbbell if (rdev->flags & RADEON_IS_AGP) { 146254885Sdumbbell if (!(drm_core_has_AGP(rdev->ddev) && rdev->ddev->agp)) { 147254885Sdumbbell DRM_ERROR("AGP is not enabled for memory type %u\n", 148254885Sdumbbell (unsigned)type); 149254885Sdumbbell return -EINVAL; 150254885Sdumbbell } 151254885Sdumbbell if (!rdev->ddev->agp->cant_use_aperture) 152254885Sdumbbell man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; 153254885Sdumbbell man->available_caching = TTM_PL_FLAG_UNCACHED | 154254885Sdumbbell TTM_PL_FLAG_WC; 155254885Sdumbbell man->default_caching = TTM_PL_FLAG_WC; 156254885Sdumbbell } 157254885Sdumbbell#endif 158254885Sdumbbell break; 159254885Sdumbbell case TTM_PL_VRAM: 160254885Sdumbbell /* "On-card" video ram */ 161254885Sdumbbell man->func = &ttm_bo_manager_func; 162254885Sdumbbell man->gpu_offset = rdev->mc.vram_start; 163254885Sdumbbell man->flags = TTM_MEMTYPE_FLAG_FIXED | 164254885Sdumbbell TTM_MEMTYPE_FLAG_MAPPABLE; 165254885Sdumbbell man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC; 166254885Sdumbbell man->default_caching = TTM_PL_FLAG_WC; 167254885Sdumbbell break; 168254885Sdumbbell default: 169254885Sdumbbell DRM_ERROR("Unsupported memory type %u\n", (unsigned)type); 170254885Sdumbbell return -EINVAL; 171254885Sdumbbell } 172254885Sdumbbell return 0; 173254885Sdumbbell} 174254885Sdumbbell 175254885Sdumbbellstatic void radeon_evict_flags(struct ttm_buffer_object *bo, 176254885Sdumbbell struct ttm_placement *placement) 177254885Sdumbbell{ 178254885Sdumbbell struct radeon_bo *rbo; 179254885Sdumbbell static u32 placements = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM; 180254885Sdumbbell 181254885Sdumbbell if (!radeon_ttm_bo_is_radeon_bo(bo)) { 182254885Sdumbbell placement->fpfn = 0; 183254885Sdumbbell placement->lpfn = 0; 184254885Sdumbbell placement->placement = &placements; 185254885Sdumbbell placement->busy_placement = &placements; 186254885Sdumbbell placement->num_placement = 1; 187254885Sdumbbell placement->num_busy_placement = 1; 188254885Sdumbbell return; 189254885Sdumbbell } 190254885Sdumbbell rbo = container_of(bo, struct radeon_bo, tbo); 191254885Sdumbbell switch (bo->mem.mem_type) { 192254885Sdumbbell case TTM_PL_VRAM: 193254885Sdumbbell if (rbo->rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready == false) 194254885Sdumbbell radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_CPU); 195254885Sdumbbell else 196254885Sdumbbell radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_GTT); 197254885Sdumbbell break; 198254885Sdumbbell case TTM_PL_TT: 199254885Sdumbbell default: 200254885Sdumbbell radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_CPU); 201254885Sdumbbell } 202254885Sdumbbell *placement = rbo->placement; 203254885Sdumbbell} 204254885Sdumbbell 205254885Sdumbbellstatic int radeon_verify_access(struct ttm_buffer_object *bo) 206254885Sdumbbell{ 207254885Sdumbbell return 0; 208254885Sdumbbell} 209254885Sdumbbell 210254885Sdumbbellstatic void radeon_move_null(struct ttm_buffer_object *bo, 211254885Sdumbbell struct ttm_mem_reg *new_mem) 212254885Sdumbbell{ 213254885Sdumbbell struct ttm_mem_reg *old_mem = &bo->mem; 214254885Sdumbbell 215254885Sdumbbell KASSERT(old_mem->mm_node == NULL, ("old_mem->mm_node != NULL")); 216254885Sdumbbell *old_mem = *new_mem; 217254885Sdumbbell new_mem->mm_node = NULL; 218254885Sdumbbell} 219254885Sdumbbell 220254885Sdumbbellstatic int radeon_move_blit(struct ttm_buffer_object *bo, 221254885Sdumbbell bool evict, bool no_wait_gpu, 222254885Sdumbbell struct ttm_mem_reg *new_mem, 223254885Sdumbbell struct ttm_mem_reg *old_mem) 224254885Sdumbbell{ 225254885Sdumbbell struct radeon_device *rdev; 226254885Sdumbbell uint64_t old_start, new_start; 227254885Sdumbbell struct radeon_fence *fence; 228254885Sdumbbell int r, ridx; 229254885Sdumbbell 230254885Sdumbbell rdev = radeon_get_rdev(bo->bdev); 231254885Sdumbbell ridx = radeon_copy_ring_index(rdev); 232254885Sdumbbell old_start = old_mem->start << PAGE_SHIFT; 233254885Sdumbbell new_start = new_mem->start << PAGE_SHIFT; 234254885Sdumbbell 235254885Sdumbbell switch (old_mem->mem_type) { 236254885Sdumbbell case TTM_PL_VRAM: 237254885Sdumbbell old_start += rdev->mc.vram_start; 238254885Sdumbbell break; 239254885Sdumbbell case TTM_PL_TT: 240254885Sdumbbell old_start += rdev->mc.gtt_start; 241254885Sdumbbell break; 242254885Sdumbbell default: 243254885Sdumbbell DRM_ERROR("Unknown placement %d\n", old_mem->mem_type); 244254885Sdumbbell return -EINVAL; 245254885Sdumbbell } 246254885Sdumbbell switch (new_mem->mem_type) { 247254885Sdumbbell case TTM_PL_VRAM: 248254885Sdumbbell new_start += rdev->mc.vram_start; 249254885Sdumbbell break; 250254885Sdumbbell case TTM_PL_TT: 251254885Sdumbbell new_start += rdev->mc.gtt_start; 252254885Sdumbbell break; 253254885Sdumbbell default: 254254885Sdumbbell DRM_ERROR("Unknown placement %d\n", old_mem->mem_type); 255254885Sdumbbell return -EINVAL; 256254885Sdumbbell } 257254885Sdumbbell if (!rdev->ring[ridx].ready) { 258254885Sdumbbell DRM_ERROR("Trying to move memory with ring turned off.\n"); 259254885Sdumbbell return -EINVAL; 260254885Sdumbbell } 261254885Sdumbbell 262254885Sdumbbell CTASSERT((PAGE_SIZE % RADEON_GPU_PAGE_SIZE) == 0); 263254885Sdumbbell 264254885Sdumbbell /* sync other rings */ 265254885Sdumbbell fence = bo->sync_obj; 266254885Sdumbbell r = radeon_copy(rdev, old_start, new_start, 267254885Sdumbbell new_mem->num_pages * (PAGE_SIZE / RADEON_GPU_PAGE_SIZE), /* GPU pages */ 268254885Sdumbbell &fence); 269254885Sdumbbell /* FIXME: handle copy error */ 270254885Sdumbbell r = ttm_bo_move_accel_cleanup(bo, (void *)fence, 271254885Sdumbbell evict, no_wait_gpu, new_mem); 272254885Sdumbbell radeon_fence_unref(&fence); 273254885Sdumbbell return r; 274254885Sdumbbell} 275254885Sdumbbell 276254885Sdumbbellstatic int radeon_move_vram_ram(struct ttm_buffer_object *bo, 277254885Sdumbbell bool evict, bool interruptible, 278254885Sdumbbell bool no_wait_gpu, 279254885Sdumbbell struct ttm_mem_reg *new_mem) 280254885Sdumbbell{ 281254885Sdumbbell struct radeon_device *rdev; 282254885Sdumbbell struct ttm_mem_reg *old_mem = &bo->mem; 283254885Sdumbbell struct ttm_mem_reg tmp_mem; 284254885Sdumbbell u32 placements; 285254885Sdumbbell struct ttm_placement placement; 286254885Sdumbbell int r; 287254885Sdumbbell 288254885Sdumbbell rdev = radeon_get_rdev(bo->bdev); 289254885Sdumbbell tmp_mem = *new_mem; 290254885Sdumbbell tmp_mem.mm_node = NULL; 291254885Sdumbbell placement.fpfn = 0; 292254885Sdumbbell placement.lpfn = 0; 293254885Sdumbbell placement.num_placement = 1; 294254885Sdumbbell placement.placement = &placements; 295254885Sdumbbell placement.num_busy_placement = 1; 296254885Sdumbbell placement.busy_placement = &placements; 297254885Sdumbbell placements = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; 298254885Sdumbbell r = ttm_bo_mem_space(bo, &placement, &tmp_mem, 299254885Sdumbbell interruptible, no_wait_gpu); 300254885Sdumbbell if (unlikely(r)) { 301254885Sdumbbell return r; 302254885Sdumbbell } 303254885Sdumbbell 304254885Sdumbbell r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement); 305254885Sdumbbell if (unlikely(r)) { 306254885Sdumbbell goto out_cleanup; 307254885Sdumbbell } 308254885Sdumbbell 309254885Sdumbbell r = ttm_tt_bind(bo->ttm, &tmp_mem); 310254885Sdumbbell if (unlikely(r)) { 311254885Sdumbbell goto out_cleanup; 312254885Sdumbbell } 313254885Sdumbbell r = radeon_move_blit(bo, true, no_wait_gpu, &tmp_mem, old_mem); 314254885Sdumbbell if (unlikely(r)) { 315254885Sdumbbell goto out_cleanup; 316254885Sdumbbell } 317254885Sdumbbell r = ttm_bo_move_ttm(bo, true, no_wait_gpu, new_mem); 318254885Sdumbbellout_cleanup: 319254885Sdumbbell ttm_bo_mem_put(bo, &tmp_mem); 320254885Sdumbbell return r; 321254885Sdumbbell} 322254885Sdumbbell 323254885Sdumbbellstatic int radeon_move_ram_vram(struct ttm_buffer_object *bo, 324254885Sdumbbell bool evict, bool interruptible, 325254885Sdumbbell bool no_wait_gpu, 326254885Sdumbbell struct ttm_mem_reg *new_mem) 327254885Sdumbbell{ 328254885Sdumbbell struct radeon_device *rdev; 329254885Sdumbbell struct ttm_mem_reg *old_mem = &bo->mem; 330254885Sdumbbell struct ttm_mem_reg tmp_mem; 331254885Sdumbbell struct ttm_placement placement; 332254885Sdumbbell u32 placements; 333254885Sdumbbell int r; 334254885Sdumbbell 335254885Sdumbbell rdev = radeon_get_rdev(bo->bdev); 336254885Sdumbbell tmp_mem = *new_mem; 337254885Sdumbbell tmp_mem.mm_node = NULL; 338254885Sdumbbell placement.fpfn = 0; 339254885Sdumbbell placement.lpfn = 0; 340254885Sdumbbell placement.num_placement = 1; 341254885Sdumbbell placement.placement = &placements; 342254885Sdumbbell placement.num_busy_placement = 1; 343254885Sdumbbell placement.busy_placement = &placements; 344254885Sdumbbell placements = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; 345254885Sdumbbell r = ttm_bo_mem_space(bo, &placement, &tmp_mem, 346254885Sdumbbell interruptible, no_wait_gpu); 347254885Sdumbbell if (unlikely(r)) { 348254885Sdumbbell return r; 349254885Sdumbbell } 350254885Sdumbbell r = ttm_bo_move_ttm(bo, true, no_wait_gpu, &tmp_mem); 351254885Sdumbbell if (unlikely(r)) { 352254885Sdumbbell goto out_cleanup; 353254885Sdumbbell } 354254885Sdumbbell r = radeon_move_blit(bo, true, no_wait_gpu, new_mem, old_mem); 355254885Sdumbbell if (unlikely(r)) { 356254885Sdumbbell goto out_cleanup; 357254885Sdumbbell } 358254885Sdumbbellout_cleanup: 359254885Sdumbbell ttm_bo_mem_put(bo, &tmp_mem); 360254885Sdumbbell return r; 361254885Sdumbbell} 362254885Sdumbbell 363254885Sdumbbellstatic int radeon_bo_move(struct ttm_buffer_object *bo, 364254885Sdumbbell bool evict, bool interruptible, 365254885Sdumbbell bool no_wait_gpu, 366254885Sdumbbell struct ttm_mem_reg *new_mem) 367254885Sdumbbell{ 368254885Sdumbbell struct radeon_device *rdev; 369254885Sdumbbell struct ttm_mem_reg *old_mem = &bo->mem; 370254885Sdumbbell int r; 371254885Sdumbbell 372254885Sdumbbell rdev = radeon_get_rdev(bo->bdev); 373254885Sdumbbell if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { 374254885Sdumbbell radeon_move_null(bo, new_mem); 375254885Sdumbbell return 0; 376254885Sdumbbell } 377254885Sdumbbell if ((old_mem->mem_type == TTM_PL_TT && 378254885Sdumbbell new_mem->mem_type == TTM_PL_SYSTEM) || 379254885Sdumbbell (old_mem->mem_type == TTM_PL_SYSTEM && 380254885Sdumbbell new_mem->mem_type == TTM_PL_TT)) { 381254885Sdumbbell /* bind is enough */ 382254885Sdumbbell radeon_move_null(bo, new_mem); 383254885Sdumbbell return 0; 384254885Sdumbbell } 385254885Sdumbbell if (!rdev->ring[radeon_copy_ring_index(rdev)].ready || 386254885Sdumbbell rdev->asic->copy.copy == NULL) { 387254885Sdumbbell /* use memcpy */ 388254885Sdumbbell goto memcpy; 389254885Sdumbbell } 390254885Sdumbbell 391254885Sdumbbell if (old_mem->mem_type == TTM_PL_VRAM && 392254885Sdumbbell new_mem->mem_type == TTM_PL_SYSTEM) { 393254885Sdumbbell r = radeon_move_vram_ram(bo, evict, interruptible, 394254885Sdumbbell no_wait_gpu, new_mem); 395254885Sdumbbell } else if (old_mem->mem_type == TTM_PL_SYSTEM && 396254885Sdumbbell new_mem->mem_type == TTM_PL_VRAM) { 397254885Sdumbbell r = radeon_move_ram_vram(bo, evict, interruptible, 398254885Sdumbbell no_wait_gpu, new_mem); 399254885Sdumbbell } else { 400254885Sdumbbell r = radeon_move_blit(bo, evict, no_wait_gpu, new_mem, old_mem); 401254885Sdumbbell } 402254885Sdumbbell 403254885Sdumbbell if (r) { 404254885Sdumbbellmemcpy: 405254885Sdumbbell r = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem); 406254885Sdumbbell } 407254885Sdumbbell return r; 408254885Sdumbbell} 409254885Sdumbbell 410254885Sdumbbellstatic int radeon_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 411254885Sdumbbell{ 412254885Sdumbbell struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 413254885Sdumbbell struct radeon_device *rdev = radeon_get_rdev(bdev); 414254885Sdumbbell 415254885Sdumbbell mem->bus.addr = NULL; 416254885Sdumbbell mem->bus.offset = 0; 417254885Sdumbbell mem->bus.size = mem->num_pages << PAGE_SHIFT; 418254885Sdumbbell mem->bus.base = 0; 419254885Sdumbbell mem->bus.is_iomem = false; 420254885Sdumbbell if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE)) 421254885Sdumbbell return -EINVAL; 422254885Sdumbbell switch (mem->mem_type) { 423254885Sdumbbell case TTM_PL_SYSTEM: 424254885Sdumbbell /* system memory */ 425254885Sdumbbell return 0; 426254885Sdumbbell case TTM_PL_TT: 427254885Sdumbbell#if __OS_HAS_AGP 428254885Sdumbbell if (rdev->flags & RADEON_IS_AGP) { 429254885Sdumbbell /* RADEON_IS_AGP is set only if AGP is active */ 430254885Sdumbbell mem->bus.offset = mem->start << PAGE_SHIFT; 431254885Sdumbbell mem->bus.base = rdev->mc.agp_base; 432254885Sdumbbell mem->bus.is_iomem = !rdev->ddev->agp->cant_use_aperture; 433254885Sdumbbell } 434254885Sdumbbell#endif 435254885Sdumbbell break; 436254885Sdumbbell case TTM_PL_VRAM: 437254885Sdumbbell mem->bus.offset = mem->start << PAGE_SHIFT; 438254885Sdumbbell /* check if it's visible */ 439254885Sdumbbell if ((mem->bus.offset + mem->bus.size) > rdev->mc.visible_vram_size) 440254885Sdumbbell return -EINVAL; 441254885Sdumbbell mem->bus.base = rdev->mc.aper_base; 442254885Sdumbbell mem->bus.is_iomem = true; 443254885Sdumbbell#ifdef __alpha__ 444254885Sdumbbell /* 445254885Sdumbbell * Alpha: use bus.addr to hold the ioremap() return, 446254885Sdumbbell * so we can modify bus.base below. 447254885Sdumbbell */ 448254885Sdumbbell if (mem->placement & TTM_PL_FLAG_WC) 449254885Sdumbbell mem->bus.addr = 450254885Sdumbbell ioremap_wc(mem->bus.base + mem->bus.offset, 451254885Sdumbbell mem->bus.size); 452254885Sdumbbell else 453254885Sdumbbell mem->bus.addr = 454254885Sdumbbell ioremap_nocache(mem->bus.base + mem->bus.offset, 455254885Sdumbbell mem->bus.size); 456254885Sdumbbell 457254885Sdumbbell /* 458254885Sdumbbell * Alpha: Use just the bus offset plus 459254885Sdumbbell * the hose/domain memory base for bus.base. 460254885Sdumbbell * It then can be used to build PTEs for VRAM 461254885Sdumbbell * access, as done in ttm_bo_vm_fault(). 462254885Sdumbbell */ 463254885Sdumbbell mem->bus.base = (mem->bus.base & 0x0ffffffffUL) + 464254885Sdumbbell rdev->ddev->hose->dense_mem_base; 465254885Sdumbbell#endif 466254885Sdumbbell break; 467254885Sdumbbell default: 468254885Sdumbbell return -EINVAL; 469254885Sdumbbell } 470254885Sdumbbell return 0; 471254885Sdumbbell} 472254885Sdumbbell 473254885Sdumbbellstatic void radeon_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 474254885Sdumbbell{ 475254885Sdumbbell} 476254885Sdumbbell 477254885Sdumbbellstatic int radeon_sync_obj_wait(void *sync_obj, bool lazy, bool interruptible) 478254885Sdumbbell{ 479254885Sdumbbell return radeon_fence_wait((struct radeon_fence *)sync_obj, interruptible); 480254885Sdumbbell} 481254885Sdumbbell 482254885Sdumbbellstatic int radeon_sync_obj_flush(void *sync_obj) 483254885Sdumbbell{ 484254885Sdumbbell return 0; 485254885Sdumbbell} 486254885Sdumbbell 487254885Sdumbbellstatic void radeon_sync_obj_unref(void **sync_obj) 488254885Sdumbbell{ 489254885Sdumbbell radeon_fence_unref((struct radeon_fence **)sync_obj); 490254885Sdumbbell} 491254885Sdumbbell 492254885Sdumbbellstatic void *radeon_sync_obj_ref(void *sync_obj) 493254885Sdumbbell{ 494254885Sdumbbell return radeon_fence_ref((struct radeon_fence *)sync_obj); 495254885Sdumbbell} 496254885Sdumbbell 497254885Sdumbbellstatic bool radeon_sync_obj_signaled(void *sync_obj) 498254885Sdumbbell{ 499254885Sdumbbell return radeon_fence_signaled((struct radeon_fence *)sync_obj); 500254885Sdumbbell} 501254885Sdumbbell 502254885Sdumbbell/* 503254885Sdumbbell * TTM backend functions. 504254885Sdumbbell */ 505254885Sdumbbellstruct radeon_ttm_tt { 506254885Sdumbbell struct ttm_dma_tt ttm; 507254885Sdumbbell struct radeon_device *rdev; 508254885Sdumbbell u64 offset; 509254885Sdumbbell}; 510254885Sdumbbell 511254885Sdumbbellstatic int radeon_ttm_backend_bind(struct ttm_tt *ttm, 512254885Sdumbbell struct ttm_mem_reg *bo_mem) 513254885Sdumbbell{ 514254885Sdumbbell struct radeon_ttm_tt *gtt = (void*)ttm; 515254885Sdumbbell int r; 516254885Sdumbbell 517254885Sdumbbell gtt->offset = (unsigned long)(bo_mem->start << PAGE_SHIFT); 518254885Sdumbbell if (!ttm->num_pages) { 519254885Sdumbbell DRM_ERROR("nothing to bind %lu pages for mreg %p back %p!\n", 520254885Sdumbbell ttm->num_pages, bo_mem, ttm); 521254885Sdumbbell } 522254885Sdumbbell r = radeon_gart_bind(gtt->rdev, gtt->offset, 523254885Sdumbbell ttm->num_pages, ttm->pages, gtt->ttm.dma_address); 524254885Sdumbbell if (r) { 525254885Sdumbbell DRM_ERROR("failed to bind %lu pages at 0x%08X\n", 526254885Sdumbbell ttm->num_pages, (unsigned)gtt->offset); 527254885Sdumbbell return r; 528254885Sdumbbell } 529254885Sdumbbell return 0; 530254885Sdumbbell} 531254885Sdumbbell 532254885Sdumbbellstatic int radeon_ttm_backend_unbind(struct ttm_tt *ttm) 533254885Sdumbbell{ 534254885Sdumbbell struct radeon_ttm_tt *gtt = (void *)ttm; 535254885Sdumbbell 536254885Sdumbbell radeon_gart_unbind(gtt->rdev, gtt->offset, ttm->num_pages); 537254885Sdumbbell return 0; 538254885Sdumbbell} 539254885Sdumbbell 540254885Sdumbbellstatic void radeon_ttm_backend_destroy(struct ttm_tt *ttm) 541254885Sdumbbell{ 542254885Sdumbbell struct radeon_ttm_tt *gtt = (void *)ttm; 543254885Sdumbbell 544254885Sdumbbell ttm_dma_tt_fini(>t->ttm); 545254885Sdumbbell free(gtt, DRM_MEM_DRIVER); 546254885Sdumbbell} 547254885Sdumbbell 548254885Sdumbbellstatic struct ttm_backend_func radeon_backend_func = { 549254885Sdumbbell .bind = &radeon_ttm_backend_bind, 550254885Sdumbbell .unbind = &radeon_ttm_backend_unbind, 551254885Sdumbbell .destroy = &radeon_ttm_backend_destroy, 552254885Sdumbbell}; 553254885Sdumbbell 554254885Sdumbbellstatic struct ttm_tt *radeon_ttm_tt_create(struct ttm_bo_device *bdev, 555254885Sdumbbell unsigned long size, uint32_t page_flags, 556254885Sdumbbell vm_page_t dummy_read_page) 557254885Sdumbbell{ 558254885Sdumbbell struct radeon_device *rdev; 559254885Sdumbbell struct radeon_ttm_tt *gtt; 560254885Sdumbbell 561254885Sdumbbell rdev = radeon_get_rdev(bdev); 562254885Sdumbbell#if __OS_HAS_AGP 563254885Sdumbbell#ifdef DUMBBELL_WIP 564254885Sdumbbell if (rdev->flags & RADEON_IS_AGP) { 565254885Sdumbbell return ttm_agp_tt_create(bdev, rdev->ddev->agp->agpdev, 566254885Sdumbbell size, page_flags, dummy_read_page); 567254885Sdumbbell } 568254885Sdumbbell#endif /* DUMBBELL_WIP */ 569254885Sdumbbell#endif 570254885Sdumbbell 571254885Sdumbbell gtt = malloc(sizeof(struct radeon_ttm_tt), 572254885Sdumbbell DRM_MEM_DRIVER, M_WAITOK | M_ZERO); 573254885Sdumbbell if (gtt == NULL) { 574254885Sdumbbell return NULL; 575254885Sdumbbell } 576254885Sdumbbell gtt->ttm.ttm.func = &radeon_backend_func; 577254885Sdumbbell gtt->rdev = rdev; 578254885Sdumbbell if (ttm_dma_tt_init(>t->ttm, bdev, size, page_flags, dummy_read_page)) { 579254885Sdumbbell free(gtt, DRM_MEM_DRIVER); 580254885Sdumbbell return NULL; 581254885Sdumbbell } 582254885Sdumbbell return >t->ttm.ttm; 583254885Sdumbbell} 584254885Sdumbbell 585254885Sdumbbellstatic int radeon_ttm_tt_populate(struct ttm_tt *ttm) 586254885Sdumbbell{ 587254885Sdumbbell struct radeon_device *rdev; 588254885Sdumbbell struct radeon_ttm_tt *gtt = (void *)ttm; 589254885Sdumbbell unsigned i; 590254885Sdumbbell int r; 591254885Sdumbbell#ifdef DUMBBELL_WIP 592254885Sdumbbell bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); 593254885Sdumbbell#endif /* DUMBBELL_WIP */ 594254885Sdumbbell 595254885Sdumbbell if (ttm->state != tt_unpopulated) 596254885Sdumbbell return 0; 597254885Sdumbbell 598254885Sdumbbell#ifdef DUMBBELL_WIP 599254885Sdumbbell /* 600254885Sdumbbell * Maybe unneeded on FreeBSD. 601254885Sdumbbell * -- dumbbell@ 602254885Sdumbbell */ 603254885Sdumbbell if (slave && ttm->sg) { 604254885Sdumbbell drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages, 605254885Sdumbbell gtt->ttm.dma_address, ttm->num_pages); 606254885Sdumbbell ttm->state = tt_unbound; 607254885Sdumbbell return 0; 608254885Sdumbbell } 609254885Sdumbbell#endif /* DUMBBELL_WIP */ 610254885Sdumbbell 611254885Sdumbbell rdev = radeon_get_rdev(ttm->bdev); 612254885Sdumbbell#if __OS_HAS_AGP 613254885Sdumbbell#ifdef DUMBBELL_WIP 614254885Sdumbbell if (rdev->flags & RADEON_IS_AGP) { 615254885Sdumbbell return ttm_agp_tt_populate(ttm); 616254885Sdumbbell } 617254885Sdumbbell#endif /* DUMBBELL_WIP */ 618254885Sdumbbell#endif 619254885Sdumbbell 620254885Sdumbbell#ifdef CONFIG_SWIOTLB 621254885Sdumbbell if (swiotlb_nr_tbl()) { 622254885Sdumbbell return ttm_dma_populate(>t->ttm, rdev->dev); 623254885Sdumbbell } 624254885Sdumbbell#endif 625254885Sdumbbell 626254885Sdumbbell r = ttm_pool_populate(ttm); 627254885Sdumbbell if (r) { 628254885Sdumbbell return r; 629254885Sdumbbell } 630254885Sdumbbell 631254885Sdumbbell for (i = 0; i < ttm->num_pages; i++) { 632254885Sdumbbell gtt->ttm.dma_address[i] = VM_PAGE_TO_PHYS(ttm->pages[i]); 633254885Sdumbbell#ifdef DUMBBELL_WIP 634254885Sdumbbell gtt->ttm.dma_address[i] = pci_map_page(rdev->pdev, ttm->pages[i], 635254885Sdumbbell 0, PAGE_SIZE, 636254885Sdumbbell PCI_DMA_BIDIRECTIONAL); 637254885Sdumbbell if (pci_dma_mapping_error(rdev->pdev, gtt->ttm.dma_address[i])) { 638254885Sdumbbell while (--i) { 639254885Sdumbbell pci_unmap_page(rdev->pdev, gtt->ttm.dma_address[i], 640254885Sdumbbell PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 641254885Sdumbbell gtt->ttm.dma_address[i] = 0; 642254885Sdumbbell } 643254885Sdumbbell ttm_pool_unpopulate(ttm); 644254885Sdumbbell return -EFAULT; 645254885Sdumbbell } 646254885Sdumbbell#endif /* DUMBBELL_WIP */ 647254885Sdumbbell } 648254885Sdumbbell return 0; 649254885Sdumbbell} 650254885Sdumbbell 651254885Sdumbbellstatic void radeon_ttm_tt_unpopulate(struct ttm_tt *ttm) 652254885Sdumbbell{ 653254885Sdumbbell struct radeon_device *rdev; 654254885Sdumbbell struct radeon_ttm_tt *gtt = (void *)ttm; 655254885Sdumbbell unsigned i; 656254885Sdumbbell bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); 657254885Sdumbbell 658254885Sdumbbell if (slave) 659254885Sdumbbell return; 660254885Sdumbbell 661254885Sdumbbell rdev = radeon_get_rdev(ttm->bdev); 662254885Sdumbbell#if __OS_HAS_AGP 663254885Sdumbbell#ifdef DUMBBELL_WIP 664254885Sdumbbell if (rdev->flags & RADEON_IS_AGP) { 665254885Sdumbbell ttm_agp_tt_unpopulate(ttm); 666254885Sdumbbell return; 667254885Sdumbbell } 668254885Sdumbbell#endif /* DUMBBELL_WIP */ 669254885Sdumbbell#endif 670254885Sdumbbell 671254885Sdumbbell#ifdef CONFIG_SWIOTLB 672254885Sdumbbell if (swiotlb_nr_tbl()) { 673254885Sdumbbell ttm_dma_unpopulate(>t->ttm, rdev->dev); 674254885Sdumbbell return; 675254885Sdumbbell } 676254885Sdumbbell#endif 677254885Sdumbbell 678254885Sdumbbell for (i = 0; i < ttm->num_pages; i++) { 679254885Sdumbbell if (gtt->ttm.dma_address[i]) { 680254885Sdumbbell gtt->ttm.dma_address[i] = 0; 681254885Sdumbbell#ifdef DUMBBELL_WIP 682254885Sdumbbell pci_unmap_page(rdev->pdev, gtt->ttm.dma_address[i], 683254885Sdumbbell PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 684254885Sdumbbell#endif /* DUMBBELL_WIP */ 685254885Sdumbbell } 686254885Sdumbbell } 687254885Sdumbbell 688254885Sdumbbell ttm_pool_unpopulate(ttm); 689254885Sdumbbell} 690254885Sdumbbell 691254885Sdumbbellstatic struct ttm_bo_driver radeon_bo_driver = { 692254885Sdumbbell .ttm_tt_create = &radeon_ttm_tt_create, 693254885Sdumbbell .ttm_tt_populate = &radeon_ttm_tt_populate, 694254885Sdumbbell .ttm_tt_unpopulate = &radeon_ttm_tt_unpopulate, 695254885Sdumbbell .invalidate_caches = &radeon_invalidate_caches, 696254885Sdumbbell .init_mem_type = &radeon_init_mem_type, 697254885Sdumbbell .evict_flags = &radeon_evict_flags, 698254885Sdumbbell .move = &radeon_bo_move, 699254885Sdumbbell .verify_access = &radeon_verify_access, 700254885Sdumbbell .sync_obj_signaled = &radeon_sync_obj_signaled, 701254885Sdumbbell .sync_obj_wait = &radeon_sync_obj_wait, 702254885Sdumbbell .sync_obj_flush = &radeon_sync_obj_flush, 703254885Sdumbbell .sync_obj_unref = &radeon_sync_obj_unref, 704254885Sdumbbell .sync_obj_ref = &radeon_sync_obj_ref, 705254885Sdumbbell .move_notify = &radeon_bo_move_notify, 706254885Sdumbbell .fault_reserve_notify = &radeon_bo_fault_reserve_notify, 707254885Sdumbbell .io_mem_reserve = &radeon_ttm_io_mem_reserve, 708254885Sdumbbell .io_mem_free = &radeon_ttm_io_mem_free, 709254885Sdumbbell}; 710254885Sdumbbell 711254885Sdumbbellint radeon_ttm_init(struct radeon_device *rdev) 712254885Sdumbbell{ 713254885Sdumbbell int r, r2; 714254885Sdumbbell 715254885Sdumbbell r = radeon_ttm_global_init(rdev); 716254885Sdumbbell if (r) { 717254885Sdumbbell return r; 718254885Sdumbbell } 719254885Sdumbbell /* No others user of address space so set it to 0 */ 720254885Sdumbbell r = ttm_bo_device_init(&rdev->mman.bdev, 721254885Sdumbbell rdev->mman.bo_global_ref.ref.object, 722254885Sdumbbell &radeon_bo_driver, DRM_FILE_PAGE_OFFSET, 723254885Sdumbbell rdev->need_dma32); 724254885Sdumbbell if (r) { 725254885Sdumbbell DRM_ERROR("failed initializing buffer object driver(%d).\n", r); 726254885Sdumbbell return r; 727254885Sdumbbell } 728254885Sdumbbell rdev->mman.initialized = true; 729254885Sdumbbell rdev->ddev->drm_ttm_bdev = &rdev->mman.bdev; 730254885Sdumbbell r = ttm_bo_init_mm(&rdev->mman.bdev, TTM_PL_VRAM, 731254885Sdumbbell rdev->mc.real_vram_size >> PAGE_SHIFT); 732254885Sdumbbell if (r) { 733254885Sdumbbell DRM_ERROR("Failed initializing VRAM heap.\n"); 734254885Sdumbbell return r; 735254885Sdumbbell } 736254885Sdumbbell r = radeon_bo_create(rdev, 256 * 1024, PAGE_SIZE, true, 737254885Sdumbbell RADEON_GEM_DOMAIN_VRAM, 738254885Sdumbbell NULL, &rdev->stollen_vga_memory); 739254885Sdumbbell if (r) { 740254885Sdumbbell return r; 741254885Sdumbbell } 742254885Sdumbbell r = radeon_bo_reserve(rdev->stollen_vga_memory, false); 743254885Sdumbbell if (r) { 744254885Sdumbbell radeon_bo_unref(&rdev->stollen_vga_memory); 745254885Sdumbbell return r; 746254885Sdumbbell } 747254885Sdumbbell r = radeon_bo_pin(rdev->stollen_vga_memory, RADEON_GEM_DOMAIN_VRAM, NULL); 748254885Sdumbbell radeon_bo_unreserve(rdev->stollen_vga_memory); 749254885Sdumbbell if (r) { 750254885Sdumbbell radeon_bo_unref(&rdev->stollen_vga_memory); 751254885Sdumbbell return r; 752254885Sdumbbell } 753254885Sdumbbell DRM_INFO("radeon: %uM of VRAM memory ready\n", 754254885Sdumbbell (unsigned)rdev->mc.real_vram_size / (1024 * 1024)); 755254885Sdumbbell r = ttm_bo_init_mm(&rdev->mman.bdev, TTM_PL_TT, 756254885Sdumbbell rdev->mc.gtt_size >> PAGE_SHIFT); 757254885Sdumbbell if (r) { 758254885Sdumbbell DRM_ERROR("Failed initializing GTT heap.\n"); 759254885Sdumbbell r2 = radeon_bo_reserve(rdev->stollen_vga_memory, false); 760254885Sdumbbell if (likely(r2 == 0)) { 761254885Sdumbbell radeon_bo_unpin(rdev->stollen_vga_memory); 762254885Sdumbbell radeon_bo_unreserve(rdev->stollen_vga_memory); 763254885Sdumbbell } 764254885Sdumbbell radeon_bo_unref(&rdev->stollen_vga_memory); 765254885Sdumbbell return r; 766254885Sdumbbell } 767254885Sdumbbell DRM_INFO("radeon: %uM of GTT memory ready.\n", 768254885Sdumbbell (unsigned)(rdev->mc.gtt_size / (1024 * 1024))); 769254885Sdumbbell 770254885Sdumbbell r = radeon_ttm_debugfs_init(rdev); 771254885Sdumbbell if (r) { 772254885Sdumbbell DRM_ERROR("Failed to init debugfs\n"); 773254885Sdumbbell r2 = radeon_bo_reserve(rdev->stollen_vga_memory, false); 774254885Sdumbbell if (likely(r2 == 0)) { 775254885Sdumbbell radeon_bo_unpin(rdev->stollen_vga_memory); 776254885Sdumbbell radeon_bo_unreserve(rdev->stollen_vga_memory); 777254885Sdumbbell } 778254885Sdumbbell radeon_bo_unref(&rdev->stollen_vga_memory); 779254885Sdumbbell return r; 780254885Sdumbbell } 781254885Sdumbbell return 0; 782254885Sdumbbell} 783254885Sdumbbell 784254885Sdumbbellvoid radeon_ttm_fini(struct radeon_device *rdev) 785254885Sdumbbell{ 786254885Sdumbbell int r; 787254885Sdumbbell 788254885Sdumbbell if (!rdev->mman.initialized) 789254885Sdumbbell return; 790254885Sdumbbell if (rdev->stollen_vga_memory) { 791254885Sdumbbell r = radeon_bo_reserve(rdev->stollen_vga_memory, false); 792254885Sdumbbell if (r == 0) { 793254885Sdumbbell radeon_bo_unpin(rdev->stollen_vga_memory); 794254885Sdumbbell radeon_bo_unreserve(rdev->stollen_vga_memory); 795254885Sdumbbell } 796254885Sdumbbell radeon_bo_unref(&rdev->stollen_vga_memory); 797254885Sdumbbell } 798254885Sdumbbell ttm_bo_clean_mm(&rdev->mman.bdev, TTM_PL_VRAM); 799254885Sdumbbell ttm_bo_clean_mm(&rdev->mman.bdev, TTM_PL_TT); 800254885Sdumbbell ttm_bo_device_release(&rdev->mman.bdev); 801254885Sdumbbell radeon_gart_fini(rdev); 802254885Sdumbbell radeon_ttm_global_fini(rdev); 803254885Sdumbbell rdev->mman.initialized = false; 804254885Sdumbbell DRM_INFO("radeon: ttm finalized\n"); 805254885Sdumbbell} 806254885Sdumbbell 807254885Sdumbbell/* this should only be called at bootup or when userspace 808254885Sdumbbell * isn't running */ 809254885Sdumbbellvoid radeon_ttm_set_active_vram_size(struct radeon_device *rdev, u64 size) 810254885Sdumbbell{ 811254885Sdumbbell struct ttm_mem_type_manager *man; 812254885Sdumbbell 813254885Sdumbbell if (!rdev->mman.initialized) 814254885Sdumbbell return; 815254885Sdumbbell 816254885Sdumbbell man = &rdev->mman.bdev.man[TTM_PL_VRAM]; 817254885Sdumbbell /* this just adjusts TTM size idea, which sets lpfn to the correct value */ 818254885Sdumbbell man->size = size >> PAGE_SHIFT; 819254885Sdumbbell} 820254885Sdumbbell 821254885Sdumbbell#ifdef DUMBBELL_WIP 822254885Sdumbbellstatic struct vm_operations_struct radeon_ttm_vm_ops; 823254885Sdumbbellstatic const struct vm_operations_struct *ttm_vm_ops = NULL; 824254885Sdumbbell 825254885Sdumbbellstatic int radeon_ttm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 826254885Sdumbbell{ 827254885Sdumbbell struct ttm_buffer_object *bo; 828254885Sdumbbell struct radeon_device *rdev; 829254885Sdumbbell int r; 830254885Sdumbbell 831254885Sdumbbell bo = (struct ttm_buffer_object *)vma->vm_private_data; 832254885Sdumbbell if (bo == NULL) { 833254885Sdumbbell return VM_FAULT_NOPAGE; 834254885Sdumbbell } 835254885Sdumbbell rdev = radeon_get_rdev(bo->bdev); 836254885Sdumbbell sx_slock(&rdev->pm.mclk_lock); 837254885Sdumbbell r = ttm_vm_ops->fault(vma, vmf); 838254885Sdumbbell sx_sunlock(&rdev->pm.mclk_lock); 839254885Sdumbbell return r; 840254885Sdumbbell} 841254885Sdumbbell 842254885Sdumbbellint radeon_mmap(struct file *filp, struct vm_area_struct *vma) 843254885Sdumbbell{ 844254885Sdumbbell struct drm_file *file_priv; 845254885Sdumbbell struct radeon_device *rdev; 846254885Sdumbbell int r; 847254885Sdumbbell 848254885Sdumbbell if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET)) { 849254885Sdumbbell return drm_mmap(filp, vma); 850254885Sdumbbell } 851254885Sdumbbell 852254885Sdumbbell file_priv = filp->private_data; 853254885Sdumbbell rdev = file_priv->minor->dev->dev_private; 854254885Sdumbbell if (rdev == NULL) { 855254885Sdumbbell return -EINVAL; 856254885Sdumbbell } 857254885Sdumbbell r = ttm_bo_mmap(filp, vma, &rdev->mman.bdev); 858254885Sdumbbell if (unlikely(r != 0)) { 859254885Sdumbbell return r; 860254885Sdumbbell } 861254885Sdumbbell if (unlikely(ttm_vm_ops == NULL)) { 862254885Sdumbbell ttm_vm_ops = vma->vm_ops; 863254885Sdumbbell radeon_ttm_vm_ops = *ttm_vm_ops; 864254885Sdumbbell radeon_ttm_vm_ops.fault = &radeon_ttm_fault; 865254885Sdumbbell } 866254885Sdumbbell vma->vm_ops = &radeon_ttm_vm_ops; 867254885Sdumbbell return 0; 868254885Sdumbbell} 869254885Sdumbbell#endif /* DUMBBELL_WIP */ 870254885Sdumbbell 871254885Sdumbbell 872254885Sdumbbell#define RADEON_DEBUGFS_MEM_TYPES 2 873254885Sdumbbell 874254885Sdumbbell#if defined(CONFIG_DEBUG_FS) 875254885Sdumbbellstatic int radeon_mm_dump_table(struct seq_file *m, void *data) 876254885Sdumbbell{ 877254885Sdumbbell struct drm_info_node *node = (struct drm_info_node *)m->private; 878254885Sdumbbell struct drm_mm *mm = (struct drm_mm *)node->info_ent->data; 879254885Sdumbbell struct drm_device *dev = node->minor->dev; 880254885Sdumbbell struct radeon_device *rdev = dev->dev_private; 881254885Sdumbbell int ret; 882254885Sdumbbell struct ttm_bo_global *glob = rdev->mman.bdev.glob; 883254885Sdumbbell 884254885Sdumbbell spin_lock(&glob->lru_lock); 885254885Sdumbbell ret = drm_mm_dump_table(m, mm); 886254885Sdumbbell spin_unlock(&glob->lru_lock); 887254885Sdumbbell return ret; 888254885Sdumbbell} 889254885Sdumbbell#endif 890254885Sdumbbell 891254885Sdumbbellstatic int radeon_ttm_debugfs_init(struct radeon_device *rdev) 892254885Sdumbbell{ 893254885Sdumbbell#if defined(CONFIG_DEBUG_FS) 894254885Sdumbbell static struct drm_info_list radeon_mem_types_list[RADEON_DEBUGFS_MEM_TYPES+2]; 895254885Sdumbbell static char radeon_mem_types_names[RADEON_DEBUGFS_MEM_TYPES+2][32]; 896254885Sdumbbell unsigned i; 897254885Sdumbbell 898254885Sdumbbell for (i = 0; i < RADEON_DEBUGFS_MEM_TYPES; i++) { 899254885Sdumbbell if (i == 0) 900254885Sdumbbell sprintf(radeon_mem_types_names[i], "radeon_vram_mm"); 901254885Sdumbbell else 902254885Sdumbbell sprintf(radeon_mem_types_names[i], "radeon_gtt_mm"); 903254885Sdumbbell radeon_mem_types_list[i].name = radeon_mem_types_names[i]; 904254885Sdumbbell radeon_mem_types_list[i].show = &radeon_mm_dump_table; 905254885Sdumbbell radeon_mem_types_list[i].driver_features = 0; 906254885Sdumbbell if (i == 0) 907254885Sdumbbell radeon_mem_types_list[i].data = rdev->mman.bdev.man[TTM_PL_VRAM].priv; 908254885Sdumbbell else 909254885Sdumbbell radeon_mem_types_list[i].data = rdev->mman.bdev.man[TTM_PL_TT].priv; 910254885Sdumbbell 911254885Sdumbbell } 912254885Sdumbbell /* Add ttm page pool to debugfs */ 913254885Sdumbbell sprintf(radeon_mem_types_names[i], "ttm_page_pool"); 914254885Sdumbbell radeon_mem_types_list[i].name = radeon_mem_types_names[i]; 915254885Sdumbbell radeon_mem_types_list[i].show = &ttm_page_alloc_debugfs; 916254885Sdumbbell radeon_mem_types_list[i].driver_features = 0; 917254885Sdumbbell radeon_mem_types_list[i++].data = NULL; 918254885Sdumbbell#ifdef CONFIG_SWIOTLB 919254885Sdumbbell if (swiotlb_nr_tbl()) { 920254885Sdumbbell sprintf(radeon_mem_types_names[i], "ttm_dma_page_pool"); 921254885Sdumbbell radeon_mem_types_list[i].name = radeon_mem_types_names[i]; 922254885Sdumbbell radeon_mem_types_list[i].show = &ttm_dma_page_alloc_debugfs; 923254885Sdumbbell radeon_mem_types_list[i].driver_features = 0; 924254885Sdumbbell radeon_mem_types_list[i++].data = NULL; 925254885Sdumbbell } 926254885Sdumbbell#endif 927254885Sdumbbell return radeon_debugfs_add_files(rdev, radeon_mem_types_list, i); 928254885Sdumbbell 929254885Sdumbbell#endif 930254885Sdumbbell return 0; 931254885Sdumbbell} 932