1235783Skib/* 2235783Skib * Copyright �� 2010 Daniel Vetter 3235783Skib * 4235783Skib * Permission is hereby granted, free of charge, to any person obtaining a 5235783Skib * copy of this software and associated documentation files (the "Software"), 6235783Skib * to deal in the Software without restriction, including without limitation 7235783Skib * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8235783Skib * and/or sell copies of the Software, and to permit persons to whom the 9235783Skib * Software is furnished to do so, subject to the following conditions: 10235783Skib * 11235783Skib * The above copyright notice and this permission notice (including the next 12235783Skib * paragraph) shall be included in all copies or substantial portions of the 13235783Skib * Software. 14235783Skib * 15235783Skib * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16235783Skib * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17235783Skib * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18235783Skib * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19235783Skib * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20235783Skib * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21235783Skib * IN THE SOFTWARE. 22235783Skib * 23235783Skib */ 24235783Skib 25235783Skib#include <sys/cdefs.h> 26235783Skib__FBSDID("$FreeBSD$"); 27235783Skib 28235783Skib#include <dev/drm2/drmP.h> 29235783Skib#include <dev/drm2/drm.h> 30235783Skib#include <dev/drm2/i915/i915_drm.h> 31235783Skib#include <dev/drm2/i915/i915_drv.h> 32235783Skib#include <dev/drm2/i915/intel_drv.h> 33235783Skib#include <sys/sched.h> 34235783Skib#include <sys/sf_buf.h> 35235783Skib 36235783Skib/* PPGTT support for Sandybdrige/Gen6 and later */ 37235783Skibstatic void 38235783Skibi915_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt, 39235783Skib unsigned first_entry, unsigned num_entries) 40235783Skib{ 41235783Skib uint32_t *pt_vaddr; 42235783Skib uint32_t scratch_pte; 43235783Skib struct sf_buf *sf; 44235783Skib unsigned act_pd, first_pte, last_pte, i; 45235783Skib 46235783Skib act_pd = first_entry / I915_PPGTT_PT_ENTRIES; 47235783Skib first_pte = first_entry % I915_PPGTT_PT_ENTRIES; 48235783Skib 49235783Skib scratch_pte = GEN6_PTE_ADDR_ENCODE(ppgtt->scratch_page_dma_addr); 50235783Skib scratch_pte |= GEN6_PTE_VALID | GEN6_PTE_CACHE_LLC; 51235783Skib 52235783Skib while (num_entries) { 53235783Skib last_pte = first_pte + num_entries; 54235783Skib if (last_pte > I915_PPGTT_PT_ENTRIES) 55235783Skib last_pte = I915_PPGTT_PT_ENTRIES; 56235783Skib 57235783Skib sched_pin(); 58235783Skib sf = sf_buf_alloc(ppgtt->pt_pages[act_pd], SFB_CPUPRIVATE); 59235783Skib pt_vaddr = (uint32_t *)(uintptr_t)sf_buf_kva(sf); 60235783Skib 61235783Skib for (i = first_pte; i < last_pte; i++) 62235783Skib pt_vaddr[i] = scratch_pte; 63235783Skib 64235783Skib sf_buf_free(sf); 65235783Skib sched_unpin(); 66235783Skib 67235783Skib num_entries -= last_pte - first_pte; 68235783Skib first_pte = 0; 69235783Skib act_pd++; 70235783Skib } 71235783Skib 72235783Skib} 73235783Skib 74235783Skibint 75235783Skibi915_gem_init_aliasing_ppgtt(struct drm_device *dev) 76235783Skib{ 77235783Skib struct drm_i915_private *dev_priv; 78235783Skib struct i915_hw_ppgtt *ppgtt; 79235783Skib u_int first_pd_entry_in_global_pt, i; 80235783Skib 81235783Skib dev_priv = dev->dev_private; 82235783Skib 83235783Skib /* 84235783Skib * ppgtt PDEs reside in the global gtt pagetable, which has 512*1024 85235783Skib * entries. For aliasing ppgtt support we just steal them at the end for 86235783Skib * now. 87235783Skib */ 88235783Skib first_pd_entry_in_global_pt = 512 * 1024 - I915_PPGTT_PD_ENTRIES; 89235783Skib 90235783Skib ppgtt = malloc(sizeof(*ppgtt), DRM_I915_GEM, M_WAITOK | M_ZERO); 91235783Skib 92235783Skib ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES; 93235783Skib ppgtt->pt_pages = malloc(sizeof(vm_page_t) * ppgtt->num_pd_entries, 94235783Skib DRM_I915_GEM, M_WAITOK | M_ZERO); 95235783Skib 96235783Skib for (i = 0; i < ppgtt->num_pd_entries; i++) { 97235783Skib ppgtt->pt_pages[i] = vm_page_alloc(NULL, 0, 98235783Skib VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | 99235783Skib VM_ALLOC_ZERO); 100235783Skib if (ppgtt->pt_pages[i] == NULL) { 101235783Skib dev_priv->mm.aliasing_ppgtt = ppgtt; 102235783Skib i915_gem_cleanup_aliasing_ppgtt(dev); 103235783Skib return (-ENOMEM); 104235783Skib } 105235783Skib } 106235783Skib 107235783Skib ppgtt->scratch_page_dma_addr = dev_priv->mm.gtt.scratch_page_dma; 108235783Skib 109235783Skib i915_ppgtt_clear_range(ppgtt, 0, ppgtt->num_pd_entries * 110235783Skib I915_PPGTT_PT_ENTRIES); 111235783Skib ppgtt->pd_offset = (first_pd_entry_in_global_pt) * sizeof(uint32_t); 112235783Skib dev_priv->mm.aliasing_ppgtt = ppgtt; 113235783Skib return (0); 114235783Skib} 115235783Skib 116235783Skibstatic void 117235783Skibi915_ppgtt_insert_pages(struct i915_hw_ppgtt *ppgtt, unsigned first_entry, 118235783Skib unsigned num_entries, vm_page_t *pages, uint32_t pte_flags) 119235783Skib{ 120235783Skib uint32_t *pt_vaddr, pte; 121235783Skib struct sf_buf *sf; 122235783Skib unsigned act_pd, first_pte; 123235783Skib unsigned last_pte, i; 124235783Skib vm_paddr_t page_addr; 125235783Skib 126235783Skib act_pd = first_entry / I915_PPGTT_PT_ENTRIES; 127235783Skib first_pte = first_entry % I915_PPGTT_PT_ENTRIES; 128235783Skib 129235783Skib while (num_entries) { 130235783Skib last_pte = first_pte + num_entries; 131235783Skib if (last_pte > I915_PPGTT_PT_ENTRIES) 132235783Skib last_pte = I915_PPGTT_PT_ENTRIES; 133235783Skib 134235783Skib sched_pin(); 135235783Skib sf = sf_buf_alloc(ppgtt->pt_pages[act_pd], SFB_CPUPRIVATE); 136235783Skib pt_vaddr = (uint32_t *)(uintptr_t)sf_buf_kva(sf); 137235783Skib 138235783Skib for (i = first_pte; i < last_pte; i++) { 139235783Skib page_addr = VM_PAGE_TO_PHYS(*pages); 140235783Skib pte = GEN6_PTE_ADDR_ENCODE(page_addr); 141235783Skib pt_vaddr[i] = pte | pte_flags; 142235783Skib 143235783Skib pages++; 144235783Skib } 145235783Skib 146235783Skib sf_buf_free(sf); 147235783Skib sched_unpin(); 148235783Skib 149235783Skib num_entries -= last_pte - first_pte; 150235783Skib first_pte = 0; 151235783Skib act_pd++; 152235783Skib } 153235783Skib} 154235783Skib 155235783Skibvoid 156235783Skibi915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt, 157235783Skib struct drm_i915_gem_object *obj, enum i915_cache_level cache_level) 158235783Skib{ 159235783Skib struct drm_device *dev; 160235783Skib struct drm_i915_private *dev_priv; 161235783Skib uint32_t pte_flags; 162235783Skib 163235783Skib dev = obj->base.dev; 164235783Skib dev_priv = dev->dev_private; 165235783Skib pte_flags = GEN6_PTE_VALID; 166235783Skib 167235783Skib switch (cache_level) { 168235783Skib case I915_CACHE_LLC_MLC: 169235783Skib pte_flags |= GEN6_PTE_CACHE_LLC_MLC; 170235783Skib break; 171235783Skib case I915_CACHE_LLC: 172235783Skib pte_flags |= GEN6_PTE_CACHE_LLC; 173235783Skib break; 174235783Skib case I915_CACHE_NONE: 175235783Skib pte_flags |= GEN6_PTE_UNCACHED; 176235783Skib break; 177235783Skib default: 178235783Skib panic("cache mode"); 179235783Skib } 180235783Skib 181235783Skib i915_ppgtt_insert_pages(ppgtt, obj->gtt_space->start >> PAGE_SHIFT, 182235783Skib obj->base.size >> PAGE_SHIFT, obj->pages, pte_flags); 183235783Skib} 184235783Skib 185235783Skibvoid i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt, 186235783Skib struct drm_i915_gem_object *obj) 187235783Skib{ 188235783Skib i915_ppgtt_clear_range(ppgtt, obj->gtt_space->start >> PAGE_SHIFT, 189235783Skib obj->base.size >> PAGE_SHIFT); 190235783Skib} 191235783Skib 192235783Skibvoid 193235783Skibi915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev) 194235783Skib{ 195235783Skib struct drm_i915_private *dev_priv; 196235783Skib struct i915_hw_ppgtt *ppgtt; 197235783Skib vm_page_t m; 198235783Skib int i; 199235783Skib 200235783Skib dev_priv = dev->dev_private; 201235783Skib ppgtt = dev_priv->mm.aliasing_ppgtt; 202235783Skib if (ppgtt == NULL) 203235783Skib return; 204235783Skib dev_priv->mm.aliasing_ppgtt = NULL; 205235783Skib 206235783Skib for (i = 0; i < ppgtt->num_pd_entries; i++) { 207235783Skib m = ppgtt->pt_pages[i]; 208235783Skib if (m != NULL) { 209235783Skib vm_page_unwire(m, 0); 210235783Skib vm_page_free(m); 211235783Skib } 212235783Skib } 213235783Skib free(ppgtt->pt_pages, DRM_I915_GEM); 214235783Skib free(ppgtt, DRM_I915_GEM); 215235783Skib} 216235783Skib 217235783Skib 218235783Skibstatic unsigned int 219235783Skibcache_level_to_agp_type(struct drm_device *dev, enum i915_cache_level 220235783Skib cache_level) 221235783Skib{ 222235783Skib 223235783Skib switch (cache_level) { 224235783Skib case I915_CACHE_LLC_MLC: 225235783Skib if (INTEL_INFO(dev)->gen >= 6) 226235783Skib return (AGP_USER_CACHED_MEMORY_LLC_MLC); 227235783Skib /* 228235783Skib * Older chipsets do not have this extra level of CPU 229235783Skib * cacheing, so fallthrough and request the PTE simply 230235783Skib * as cached. 231235783Skib */ 232235783Skib case I915_CACHE_LLC: 233235783Skib return (AGP_USER_CACHED_MEMORY); 234235783Skib 235235783Skib default: 236235783Skib case I915_CACHE_NONE: 237235783Skib return (AGP_USER_MEMORY); 238235783Skib } 239235783Skib} 240235783Skib 241235783Skibstatic bool 242235783Skibdo_idling(struct drm_i915_private *dev_priv) 243235783Skib{ 244235783Skib bool ret = dev_priv->mm.interruptible; 245235783Skib 246235783Skib if (dev_priv->mm.gtt.do_idle_maps) { 247235783Skib dev_priv->mm.interruptible = false; 248235783Skib if (i915_gpu_idle(dev_priv->dev, false)) { 249235783Skib DRM_ERROR("Couldn't idle GPU\n"); 250235783Skib /* Wait a bit, in hopes it avoids the hang */ 251235783Skib DELAY(10); 252235783Skib } 253235783Skib } 254235783Skib 255235783Skib return ret; 256235783Skib} 257235783Skib 258235783Skibstatic void 259235783Skibundo_idling(struct drm_i915_private *dev_priv, bool interruptible) 260235783Skib{ 261235783Skib 262235783Skib if (dev_priv->mm.gtt.do_idle_maps) 263235783Skib dev_priv->mm.interruptible = interruptible; 264235783Skib} 265235783Skib 266235783Skibvoid 267235783Skibi915_gem_restore_gtt_mappings(struct drm_device *dev) 268235783Skib{ 269235783Skib struct drm_i915_private *dev_priv; 270235783Skib struct drm_i915_gem_object *obj; 271235783Skib 272235783Skib dev_priv = dev->dev_private; 273235783Skib 274235783Skib /* First fill our portion of the GTT with scratch pages */ 275235783Skib intel_gtt_clear_range(dev_priv->mm.gtt_start / PAGE_SIZE, 276235783Skib (dev_priv->mm.gtt_end - dev_priv->mm.gtt_start) / PAGE_SIZE); 277235783Skib 278235783Skib list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) { 279235783Skib i915_gem_clflush_object(obj); 280235783Skib i915_gem_gtt_rebind_object(obj, obj->cache_level); 281235783Skib } 282235783Skib 283235783Skib intel_gtt_chipset_flush(); 284235783Skib} 285235783Skib 286235783Skibint 287235783Skibi915_gem_gtt_bind_object(struct drm_i915_gem_object *obj) 288235783Skib{ 289235783Skib unsigned int agp_type; 290235783Skib 291235783Skib agp_type = cache_level_to_agp_type(obj->base.dev, obj->cache_level); 292235783Skib intel_gtt_insert_pages(obj->gtt_space->start >> PAGE_SHIFT, 293235783Skib obj->base.size >> PAGE_SHIFT, obj->pages, agp_type); 294271816Sdumbbell 295271816Sdumbbell obj->has_global_gtt_mapping = 1; 296271816Sdumbbell 297235783Skib return (0); 298235783Skib} 299235783Skib 300235783Skibvoid 301235783Skibi915_gem_gtt_rebind_object(struct drm_i915_gem_object *obj, 302235783Skib enum i915_cache_level cache_level) 303235783Skib{ 304235783Skib struct drm_device *dev; 305235783Skib struct drm_i915_private *dev_priv; 306235783Skib unsigned int agp_type; 307235783Skib 308235783Skib dev = obj->base.dev; 309235783Skib dev_priv = dev->dev_private; 310235783Skib agp_type = cache_level_to_agp_type(dev, cache_level); 311235783Skib 312235783Skib intel_gtt_insert_pages(obj->gtt_space->start >> PAGE_SHIFT, 313235783Skib obj->base.size >> PAGE_SHIFT, obj->pages, agp_type); 314271816Sdumbbell 315271816Sdumbbell obj->has_global_gtt_mapping = 0; 316235783Skib} 317235783Skib 318235783Skibvoid 319235783Skibi915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj) 320235783Skib{ 321235783Skib struct drm_device *dev = obj->base.dev; 322235783Skib struct drm_i915_private *dev_priv = dev->dev_private; 323235783Skib bool interruptible; 324235783Skib 325235783Skib dev = obj->base.dev; 326235783Skib dev_priv = dev->dev_private; 327235783Skib 328235783Skib interruptible = do_idling(dev_priv); 329235783Skib 330235783Skib intel_gtt_clear_range(obj->gtt_space->start >> PAGE_SHIFT, 331235783Skib obj->base.size >> PAGE_SHIFT); 332235783Skib 333235783Skib undo_idling(dev_priv, interruptible); 334235783Skib} 335