1247835Skib/************************************************************************** 2247835Skib * 3247835Skib * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 4247835Skib * All Rights Reserved. 5247835Skib * 6247835Skib * Permission is hereby granted, free of charge, to any person obtaining a 7247835Skib * copy of this software and associated documentation files (the 8247835Skib * "Software"), to deal in the Software without restriction, including 9247835Skib * without limitation the rights to use, copy, modify, merge, publish, 10247835Skib * distribute, sub license, and/or sell copies of the Software, and to 11247835Skib * permit persons to whom the Software is furnished to do so, subject to 12247835Skib * the following conditions: 13247835Skib * 14247835Skib * The above copyright notice and this permission notice (including the 15247835Skib * next paragraph) shall be included in all copies or substantial portions 16247835Skib * of the Software. 17247835Skib * 18247835Skib * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19247835Skib * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20247835Skib * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21247835Skib * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22247835Skib * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23247835Skib * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24247835Skib * USE OR OTHER DEALINGS IN THE SOFTWARE. 25247835Skib * 26247835Skib **************************************************************************/ 27247835Skib/* 28247835Skib * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 29247835Skib */ 30247835Skib/* 31247835Skib * Copyright (c) 2013 The FreeBSD Foundation 32247835Skib * All rights reserved. 33247835Skib * 34247835Skib * Portions of this software were developed by Konstantin Belousov 35247835Skib * <kib@FreeBSD.org> under sponsorship from the FreeBSD Foundation. 36247835Skib */ 37247835Skib 38247835Skib#include <sys/cdefs.h> 39247835Skib__FBSDID("$FreeBSD$"); 40247835Skib 41247835Skib#include "opt_vm.h" 42247835Skib 43247835Skib#include <dev/drm2/drmP.h> 44247835Skib#include <dev/drm2/ttm/ttm_module.h> 45247835Skib#include <dev/drm2/ttm/ttm_bo_driver.h> 46247835Skib#include <dev/drm2/ttm/ttm_placement.h> 47247835Skib 48247835Skib#include <vm/vm.h> 49247835Skib#include <vm/vm_page.h> 50254141Sattilio#include <vm/vm_pageout.h> 51247835Skib 52247835Skib#define TTM_BO_VM_NUM_PREFAULT 16 53247835Skib 54247835SkibRB_GENERATE(ttm_bo_device_buffer_objects, ttm_buffer_object, vm_rb, 55247835Skib ttm_bo_cmp_rb_tree_items); 56247835Skib 57247835Skibint 58247835Skibttm_bo_cmp_rb_tree_items(struct ttm_buffer_object *a, 59247835Skib struct ttm_buffer_object *b) 60247835Skib{ 61247835Skib 62247835Skib if (a->vm_node->start < b->vm_node->start) { 63247835Skib return (-1); 64247835Skib } else if (a->vm_node->start > b->vm_node->start) { 65247835Skib return (1); 66247835Skib } else { 67247835Skib return (0); 68247835Skib } 69247835Skib} 70247835Skib 71247835Skibstatic struct ttm_buffer_object *ttm_bo_vm_lookup_rb(struct ttm_bo_device *bdev, 72247835Skib unsigned long page_start, 73247835Skib unsigned long num_pages) 74247835Skib{ 75247835Skib unsigned long cur_offset; 76247835Skib struct ttm_buffer_object *bo; 77247835Skib struct ttm_buffer_object *best_bo = NULL; 78247835Skib 79260779Savg bo = RB_ROOT(&bdev->addr_space_rb); 80260779Savg while (bo != NULL) { 81247835Skib cur_offset = bo->vm_node->start; 82247835Skib if (page_start >= cur_offset) { 83247835Skib best_bo = bo; 84247835Skib if (page_start == cur_offset) 85247835Skib break; 86260779Savg bo = RB_RIGHT(bo, vm_rb); 87260779Savg } else 88260779Savg bo = RB_LEFT(bo, vm_rb); 89247835Skib } 90247835Skib 91247835Skib if (unlikely(best_bo == NULL)) 92247835Skib return NULL; 93247835Skib 94247835Skib if (unlikely((best_bo->vm_node->start + best_bo->num_pages) < 95247835Skib (page_start + num_pages))) 96247835Skib return NULL; 97247835Skib 98247835Skib return best_bo; 99247835Skib} 100247835Skib 101247835Skibstatic int 102247835Skibttm_bo_vm_fault(vm_object_t vm_obj, vm_ooffset_t offset, 103247835Skib int prot, vm_page_t *mres) 104247835Skib{ 105247835Skib 106247835Skib struct ttm_buffer_object *bo = vm_obj->handle; 107247835Skib struct ttm_bo_device *bdev = bo->bdev; 108247835Skib struct ttm_tt *ttm = NULL; 109253710Skib vm_page_t m, m1, oldm; 110247835Skib int ret; 111247835Skib int retval = VM_PAGER_OK; 112247835Skib struct ttm_mem_type_manager *man = 113247835Skib &bdev->man[bo->mem.mem_type]; 114247835Skib 115247835Skib vm_object_pip_add(vm_obj, 1); 116247835Skib oldm = *mres; 117247835Skib if (oldm != NULL) { 118247835Skib vm_page_lock(oldm); 119247835Skib vm_page_remove(oldm); 120247835Skib vm_page_unlock(oldm); 121247835Skib *mres = NULL; 122247835Skib } else 123247835Skib oldm = NULL; 124247835Skibretry: 125248084Sattilio VM_OBJECT_WUNLOCK(vm_obj); 126247835Skib m = NULL; 127247835Skib 128247835Skibreserve: 129248657Sdumbbell ret = ttm_bo_reserve(bo, false, false, false, 0); 130247835Skib if (unlikely(ret != 0)) { 131247835Skib if (ret == -EBUSY) { 132247835Skib kern_yield(0); 133247835Skib goto reserve; 134247835Skib } 135247835Skib } 136247835Skib 137247835Skib if (bdev->driver->fault_reserve_notify) { 138247835Skib ret = bdev->driver->fault_reserve_notify(bo); 139247835Skib switch (ret) { 140247835Skib case 0: 141247835Skib break; 142247835Skib case -EBUSY: 143247835Skib case -ERESTART: 144247835Skib case -EINTR: 145247835Skib kern_yield(0); 146247835Skib goto reserve; 147247835Skib default: 148247835Skib retval = VM_PAGER_ERROR; 149247835Skib goto out_unlock; 150247835Skib } 151247835Skib } 152247835Skib 153247835Skib /* 154247835Skib * Wait for buffer data in transit, due to a pipelined 155247835Skib * move. 156247835Skib */ 157247835Skib 158247835Skib mtx_lock(&bdev->fence_lock); 159255044Sjkim if (test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags)) { 160254870Sdumbbell /* 161254870Sdumbbell * Here, the behavior differs between Linux and FreeBSD. 162254870Sdumbbell * 163254870Sdumbbell * On Linux, the wait is interruptible (3rd argument to 164254870Sdumbbell * ttm_bo_wait). There must be some mechanism to resume 165254870Sdumbbell * page fault handling, once the signal is processed. 166254870Sdumbbell * 167254870Sdumbbell * On FreeBSD, the wait is uninteruptible. This is not a 168254870Sdumbbell * problem as we can't end up with an unkillable process 169254870Sdumbbell * here, because the wait will eventually time out. 170254870Sdumbbell * 171254870Sdumbbell * An example of this situation is the Xorg process 172254870Sdumbbell * which uses SIGALRM internally. The signal could 173254870Sdumbbell * interrupt the wait, causing the page fault to fail 174254870Sdumbbell * and the process to receive SIGSEGV. 175254870Sdumbbell */ 176254870Sdumbbell ret = ttm_bo_wait(bo, false, false, false); 177247835Skib mtx_unlock(&bdev->fence_lock); 178247835Skib if (unlikely(ret != 0)) { 179247835Skib retval = VM_PAGER_ERROR; 180247835Skib goto out_unlock; 181247835Skib } 182247835Skib } else 183247835Skib mtx_unlock(&bdev->fence_lock); 184247835Skib 185247835Skib ret = ttm_mem_io_lock(man, true); 186247835Skib if (unlikely(ret != 0)) { 187247835Skib retval = VM_PAGER_ERROR; 188247835Skib goto out_unlock; 189247835Skib } 190247835Skib ret = ttm_mem_io_reserve_vm(bo); 191247835Skib if (unlikely(ret != 0)) { 192247835Skib retval = VM_PAGER_ERROR; 193247835Skib goto out_io_unlock; 194247835Skib } 195247835Skib 196247835Skib /* 197247835Skib * Strictly, we're not allowed to modify vma->vm_page_prot here, 198247835Skib * since the mmap_sem is only held in read mode. However, we 199247835Skib * modify only the caching bits of vma->vm_page_prot and 200247835Skib * consider those bits protected by 201247835Skib * the bo->mutex, as we should be the only writers. 202247835Skib * There shouldn't really be any readers of these bits except 203247835Skib * within vm_insert_mixed()? fork? 204247835Skib * 205247835Skib * TODO: Add a list of vmas to the bo, and change the 206247835Skib * vma->vm_page_prot when the object changes caching policy, with 207247835Skib * the correct locks held. 208247835Skib */ 209247835Skib if (!bo->mem.bus.is_iomem) { 210247835Skib /* Allocate all page at once, most common usage */ 211247835Skib ttm = bo->ttm; 212247835Skib if (ttm->bdev->driver->ttm_tt_populate(ttm)) { 213247835Skib retval = VM_PAGER_ERROR; 214247835Skib goto out_io_unlock; 215247835Skib } 216247835Skib } 217247835Skib 218247835Skib if (bo->mem.bus.is_iomem) { 219247835Skib m = vm_phys_fictitious_to_vm_page(bo->mem.bus.base + 220247835Skib bo->mem.bus.offset + offset); 221247835Skib pmap_page_set_memattr(m, ttm_io_prot(bo->mem.placement)); 222247835Skib } else { 223247835Skib ttm = bo->ttm; 224247835Skib m = ttm->pages[OFF_TO_IDX(offset)]; 225247835Skib if (unlikely(!m)) { 226247835Skib retval = VM_PAGER_ERROR; 227247835Skib goto out_io_unlock; 228247835Skib } 229247835Skib pmap_page_set_memattr(m, 230247835Skib (bo->mem.placement & TTM_PL_FLAG_CACHED) ? 231247835Skib VM_MEMATTR_WRITE_BACK : ttm_io_prot(bo->mem.placement)); 232247835Skib } 233247835Skib 234248084Sattilio VM_OBJECT_WLOCK(vm_obj); 235254138Sattilio if (vm_page_busied(m)) { 236254138Sattilio vm_page_lock(m); 237254138Sattilio VM_OBJECT_WUNLOCK(vm_obj); 238254138Sattilio vm_page_busy_sleep(m, "ttmpbs"); 239254138Sattilio VM_OBJECT_WLOCK(vm_obj); 240247835Skib ttm_mem_io_unlock(man); 241247835Skib ttm_bo_unreserve(bo); 242247835Skib goto retry; 243247835Skib } 244253710Skib m1 = vm_page_lookup(vm_obj, OFF_TO_IDX(offset)); 245253710Skib if (m1 == NULL) { 246254141Sattilio if (vm_page_insert(m, vm_obj, OFF_TO_IDX(offset))) { 247254141Sattilio VM_OBJECT_WUNLOCK(vm_obj); 248254141Sattilio VM_WAIT; 249254141Sattilio VM_OBJECT_WLOCK(vm_obj); 250254141Sattilio ttm_mem_io_unlock(man); 251254141Sattilio ttm_bo_unreserve(bo); 252254141Sattilio goto retry; 253254141Sattilio } 254253710Skib } else { 255253710Skib KASSERT(m == m1, 256253710Skib ("inconsistent insert bo %p m %p m1 %p offset %jx", 257253710Skib bo, m, m1, (uintmax_t)offset)); 258253710Skib } 259254141Sattilio m->valid = VM_PAGE_BITS_ALL; 260254141Sattilio *mres = m; 261254138Sattilio vm_page_xbusy(m); 262247835Skib 263247835Skib if (oldm != NULL) { 264247835Skib vm_page_lock(oldm); 265247835Skib vm_page_free(oldm); 266247835Skib vm_page_unlock(oldm); 267247835Skib } 268247835Skib 269247835Skibout_io_unlock1: 270247835Skib ttm_mem_io_unlock(man); 271247835Skibout_unlock1: 272247835Skib ttm_bo_unreserve(bo); 273247835Skib vm_object_pip_wakeup(vm_obj); 274247835Skib return (retval); 275247835Skib 276247835Skibout_io_unlock: 277248084Sattilio VM_OBJECT_WLOCK(vm_obj); 278247835Skib goto out_io_unlock1; 279247835Skib 280247835Skibout_unlock: 281248084Sattilio VM_OBJECT_WLOCK(vm_obj); 282247835Skib goto out_unlock1; 283247835Skib} 284247835Skib 285247835Skibstatic int 286247835Skibttm_bo_vm_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 287247835Skib vm_ooffset_t foff, struct ucred *cred, u_short *color) 288247835Skib{ 289247835Skib 290248661Sdumbbell /* 291254874Sdumbbell * On Linux, a reference to the buffer object is acquired here. 292254874Sdumbbell * The reason is that this function is not called when the 293254874Sdumbbell * mmap() is initialized, but only when a process forks for 294254874Sdumbbell * instance. Therefore on Linux, the reference on the bo is 295254874Sdumbbell * acquired either in ttm_bo_mmap() or ttm_bo_vm_open(). It's 296254874Sdumbbell * then released in ttm_bo_vm_close(). 297254874Sdumbbell * 298254874Sdumbbell * Here, this function is called during mmap() intialization. 299254874Sdumbbell * Thus, the reference acquired in ttm_bo_mmap_single() is 300254874Sdumbbell * sufficient. 301248661Sdumbbell */ 302248661Sdumbbell 303247835Skib *color = 0; 304247835Skib return (0); 305247835Skib} 306247835Skib 307247835Skibstatic void 308247835Skibttm_bo_vm_dtor(void *handle) 309247835Skib{ 310247835Skib struct ttm_buffer_object *bo = handle; 311247835Skib 312247835Skib ttm_bo_unref(&bo); 313247835Skib} 314247835Skib 315247835Skibstatic struct cdev_pager_ops ttm_pager_ops = { 316247835Skib .cdev_pg_fault = ttm_bo_vm_fault, 317247835Skib .cdev_pg_ctor = ttm_bo_vm_ctor, 318247835Skib .cdev_pg_dtor = ttm_bo_vm_dtor 319247835Skib}; 320247835Skib 321247835Skibint 322247835Skibttm_bo_mmap_single(struct ttm_bo_device *bdev, vm_ooffset_t *offset, vm_size_t size, 323247835Skib struct vm_object **obj_res, int nprot) 324247835Skib{ 325247835Skib struct ttm_bo_driver *driver; 326247835Skib struct ttm_buffer_object *bo; 327247835Skib struct vm_object *vm_obj; 328247835Skib int ret; 329247835Skib 330247835Skib rw_wlock(&bdev->vm_lock); 331247835Skib bo = ttm_bo_vm_lookup_rb(bdev, OFF_TO_IDX(*offset), OFF_TO_IDX(size)); 332247835Skib if (likely(bo != NULL)) 333247835Skib refcount_acquire(&bo->kref); 334247835Skib rw_wunlock(&bdev->vm_lock); 335247835Skib 336247835Skib if (unlikely(bo == NULL)) { 337247835Skib printf("[TTM] Could not find buffer object to map\n"); 338247835Skib return (EINVAL); 339247835Skib } 340247835Skib 341247835Skib driver = bo->bdev->driver; 342247835Skib if (unlikely(!driver->verify_access)) { 343247835Skib ret = EPERM; 344247835Skib goto out_unref; 345247835Skib } 346247835Skib ret = -driver->verify_access(bo); 347247835Skib if (unlikely(ret != 0)) 348247835Skib goto out_unref; 349247835Skib 350247835Skib vm_obj = cdev_pager_allocate(bo, OBJT_MGTDEVICE, &ttm_pager_ops, 351247835Skib size, nprot, 0, curthread->td_ucred); 352247835Skib if (vm_obj == NULL) { 353247835Skib ret = EINVAL; 354247835Skib goto out_unref; 355247835Skib } 356247835Skib /* 357247835Skib * Note: We're transferring the bo reference to vm_obj->handle here. 358247835Skib */ 359247835Skib *offset = 0; 360247835Skib *obj_res = vm_obj; 361247835Skib return 0; 362247835Skibout_unref: 363247835Skib ttm_bo_unref(&bo); 364247835Skib return ret; 365247835Skib} 366247835Skib 367254876Sdumbbellvoid 368254876Sdumbbellttm_bo_release_mmap(struct ttm_buffer_object *bo) 369254876Sdumbbell{ 370254876Sdumbbell vm_object_t vm_obj; 371254876Sdumbbell vm_page_t m; 372254876Sdumbbell int i; 373254876Sdumbbell 374254876Sdumbbell vm_obj = cdev_pager_lookup(bo); 375254877Sdumbbell if (vm_obj == NULL) 376254876Sdumbbell return; 377254876Sdumbbell 378254876Sdumbbell VM_OBJECT_WLOCK(vm_obj); 379254876Sdumbbellretry: 380254876Sdumbbell for (i = 0; i < bo->num_pages; i++) { 381254876Sdumbbell m = vm_page_lookup(vm_obj, i); 382254876Sdumbbell if (m == NULL) 383254876Sdumbbell continue; 384254876Sdumbbell if (vm_page_sleep_if_busy(m, "ttm_unm")) 385254876Sdumbbell goto retry; 386254876Sdumbbell cdev_pager_free_page(vm_obj, m); 387254876Sdumbbell } 388254876Sdumbbell VM_OBJECT_WUNLOCK(vm_obj); 389254876Sdumbbell 390254876Sdumbbell vm_object_deallocate(vm_obj); 391254876Sdumbbell} 392254876Sdumbbell 393247835Skib#if 0 394247835Skibint ttm_fbdev_mmap(struct vm_area_struct *vma, struct ttm_buffer_object *bo) 395247835Skib{ 396247835Skib if (vma->vm_pgoff != 0) 397247835Skib return -EACCES; 398247835Skib 399247835Skib vma->vm_ops = &ttm_bo_vm_ops; 400247835Skib vma->vm_private_data = ttm_bo_reference(bo); 401247835Skib vma->vm_flags |= VM_IO | VM_MIXEDMAP | VM_DONTEXPAND; 402247835Skib return 0; 403247835Skib} 404247835Skib 405247835Skibssize_t ttm_bo_io(struct ttm_bo_device *bdev, struct file *filp, 406247835Skib const char __user *wbuf, char __user *rbuf, size_t count, 407247835Skib loff_t *f_pos, bool write) 408247835Skib{ 409247835Skib struct ttm_buffer_object *bo; 410247835Skib struct ttm_bo_driver *driver; 411247835Skib struct ttm_bo_kmap_obj map; 412247835Skib unsigned long dev_offset = (*f_pos >> PAGE_SHIFT); 413247835Skib unsigned long kmap_offset; 414247835Skib unsigned long kmap_end; 415247835Skib unsigned long kmap_num; 416247835Skib size_t io_size; 417247835Skib unsigned int page_offset; 418247835Skib char *virtual; 419247835Skib int ret; 420247835Skib bool no_wait = false; 421247835Skib bool dummy; 422247835Skib 423247835Skib read_lock(&bdev->vm_lock); 424247835Skib bo = ttm_bo_vm_lookup_rb(bdev, dev_offset, 1); 425247835Skib if (likely(bo != NULL)) 426247835Skib ttm_bo_reference(bo); 427247835Skib read_unlock(&bdev->vm_lock); 428247835Skib 429247835Skib if (unlikely(bo == NULL)) 430247835Skib return -EFAULT; 431247835Skib 432247835Skib driver = bo->bdev->driver; 433247835Skib if (unlikely(!driver->verify_access)) { 434247835Skib ret = -EPERM; 435247835Skib goto out_unref; 436247835Skib } 437247835Skib 438247835Skib ret = driver->verify_access(bo, filp); 439247835Skib if (unlikely(ret != 0)) 440247835Skib goto out_unref; 441247835Skib 442247835Skib kmap_offset = dev_offset - bo->vm_node->start; 443247835Skib if (unlikely(kmap_offset >= bo->num_pages)) { 444247835Skib ret = -EFBIG; 445247835Skib goto out_unref; 446247835Skib } 447247835Skib 448247835Skib page_offset = *f_pos & ~PAGE_MASK; 449247835Skib io_size = bo->num_pages - kmap_offset; 450247835Skib io_size = (io_size << PAGE_SHIFT) - page_offset; 451247835Skib if (count < io_size) 452247835Skib io_size = count; 453247835Skib 454247835Skib kmap_end = (*f_pos + count - 1) >> PAGE_SHIFT; 455247835Skib kmap_num = kmap_end - kmap_offset + 1; 456247835Skib 457247835Skib ret = ttm_bo_reserve(bo, true, no_wait, false, 0); 458247835Skib 459247835Skib switch (ret) { 460247835Skib case 0: 461247835Skib break; 462247835Skib case -EBUSY: 463247835Skib ret = -EAGAIN; 464247835Skib goto out_unref; 465247835Skib default: 466247835Skib goto out_unref; 467247835Skib } 468247835Skib 469247835Skib ret = ttm_bo_kmap(bo, kmap_offset, kmap_num, &map); 470247835Skib if (unlikely(ret != 0)) { 471247835Skib ttm_bo_unreserve(bo); 472247835Skib goto out_unref; 473247835Skib } 474247835Skib 475247835Skib virtual = ttm_kmap_obj_virtual(&map, &dummy); 476247835Skib virtual += page_offset; 477247835Skib 478247835Skib if (write) 479247835Skib ret = copy_from_user(virtual, wbuf, io_size); 480247835Skib else 481247835Skib ret = copy_to_user(rbuf, virtual, io_size); 482247835Skib 483247835Skib ttm_bo_kunmap(&map); 484247835Skib ttm_bo_unreserve(bo); 485247835Skib ttm_bo_unref(&bo); 486247835Skib 487247835Skib if (unlikely(ret != 0)) 488247835Skib return -EFBIG; 489247835Skib 490247835Skib *f_pos += io_size; 491247835Skib 492247835Skib return io_size; 493247835Skibout_unref: 494247835Skib ttm_bo_unref(&bo); 495247835Skib return ret; 496247835Skib} 497247835Skib 498247835Skibssize_t ttm_bo_fbdev_io(struct ttm_buffer_object *bo, const char __user *wbuf, 499247835Skib char __user *rbuf, size_t count, loff_t *f_pos, 500247835Skib bool write) 501247835Skib{ 502247835Skib struct ttm_bo_kmap_obj map; 503247835Skib unsigned long kmap_offset; 504247835Skib unsigned long kmap_end; 505247835Skib unsigned long kmap_num; 506247835Skib size_t io_size; 507247835Skib unsigned int page_offset; 508247835Skib char *virtual; 509247835Skib int ret; 510247835Skib bool no_wait = false; 511247835Skib bool dummy; 512247835Skib 513247835Skib kmap_offset = (*f_pos >> PAGE_SHIFT); 514247835Skib if (unlikely(kmap_offset >= bo->num_pages)) 515247835Skib return -EFBIG; 516247835Skib 517247835Skib page_offset = *f_pos & ~PAGE_MASK; 518247835Skib io_size = bo->num_pages - kmap_offset; 519247835Skib io_size = (io_size << PAGE_SHIFT) - page_offset; 520247835Skib if (count < io_size) 521247835Skib io_size = count; 522247835Skib 523247835Skib kmap_end = (*f_pos + count - 1) >> PAGE_SHIFT; 524247835Skib kmap_num = kmap_end - kmap_offset + 1; 525247835Skib 526247835Skib ret = ttm_bo_reserve(bo, true, no_wait, false, 0); 527247835Skib 528247835Skib switch (ret) { 529247835Skib case 0: 530247835Skib break; 531247835Skib case -EBUSY: 532247835Skib return -EAGAIN; 533247835Skib default: 534247835Skib return ret; 535247835Skib } 536247835Skib 537247835Skib ret = ttm_bo_kmap(bo, kmap_offset, kmap_num, &map); 538247835Skib if (unlikely(ret != 0)) { 539247835Skib ttm_bo_unreserve(bo); 540247835Skib return ret; 541247835Skib } 542247835Skib 543247835Skib virtual = ttm_kmap_obj_virtual(&map, &dummy); 544247835Skib virtual += page_offset; 545247835Skib 546247835Skib if (write) 547247835Skib ret = copy_from_user(virtual, wbuf, io_size); 548247835Skib else 549247835Skib ret = copy_to_user(rbuf, virtual, io_size); 550247835Skib 551247835Skib ttm_bo_kunmap(&map); 552247835Skib ttm_bo_unreserve(bo); 553247835Skib ttm_bo_unref(&bo); 554247835Skib 555247835Skib if (unlikely(ret != 0)) 556247835Skib return ret; 557247835Skib 558247835Skib *f_pos += io_size; 559247835Skib 560247835Skib return io_size; 561247835Skib} 562247835Skib#endif 563