1203288Srnoland/* via_dma.c -- DMA support for the VIA Unichrome/Pro 2203288Srnoland * 3203288Srnoland * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas. 4203288Srnoland * All Rights Reserved. 5203288Srnoland * 6203288Srnoland * Copyright 2004 Digeo, Inc., Palo Alto, CA, U.S.A. 7203288Srnoland * All Rights Reserved. 8203288Srnoland * 9203288Srnoland * Copyright 2004 The Unichrome project. 10203288Srnoland * All Rights Reserved. 11203288Srnoland * 12203288Srnoland * Permission is hereby granted, free of charge, to any person obtaining a 13203288Srnoland * copy of this software and associated documentation files (the "Software"), 14203288Srnoland * to deal in the Software without restriction, including without limitation 15203288Srnoland * the rights to use, copy, modify, merge, publish, distribute, sub license, 16203288Srnoland * and/or sell copies of the Software, and to permit persons to whom the 17203288Srnoland * Software is furnished to do so, subject to the following conditions: 18203288Srnoland * 19203288Srnoland * The above copyright notice and this permission notice (including the 20203288Srnoland * next paragraph) shall be included in all copies or substantial portions 21203288Srnoland * of the Software. 22203288Srnoland * 23203288Srnoland * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 24203288Srnoland * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 25203288Srnoland * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 26203288Srnoland * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 27203288Srnoland * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 28203288Srnoland * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 29203288Srnoland * USE OR OTHER DEALINGS IN THE SOFTWARE. 30203288Srnoland * 31203288Srnoland * Authors: 32203288Srnoland * Tungsten Graphics, 33203288Srnoland * Erdi Chen, 34203288Srnoland * Thomas Hellstrom. 35203288Srnoland */ 36203288Srnoland 37203288Srnoland#include <sys/cdefs.h> 38203288Srnoland__FBSDID("$FreeBSD$"); 39203288Srnoland 40203288Srnoland#include "dev/drm/drmP.h" 41203288Srnoland#include "dev/drm/drm.h" 42203288Srnoland#include "dev/drm/via_drm.h" 43203288Srnoland#include "dev/drm/via_drv.h" 44203288Srnoland#include "dev/drm/via_3d_reg.h" 45203288Srnoland 46203288Srnoland#define CMDBUF_ALIGNMENT_SIZE (0x100) 47203288Srnoland#define CMDBUF_ALIGNMENT_MASK (0x0ff) 48203288Srnoland 49203288Srnoland/* defines for VIA 3D registers */ 50203288Srnoland#define VIA_REG_STATUS 0x400 51203288Srnoland#define VIA_REG_TRANSET 0x43C 52203288Srnoland#define VIA_REG_TRANSPACE 0x440 53203288Srnoland 54203288Srnoland/* VIA_REG_STATUS(0x400): Engine Status */ 55203288Srnoland#define VIA_CMD_RGTR_BUSY 0x00000080 /* Command Regulator is busy */ 56203288Srnoland#define VIA_2D_ENG_BUSY 0x00000001 /* 2D Engine is busy */ 57203288Srnoland#define VIA_3D_ENG_BUSY 0x00000002 /* 3D Engine is busy */ 58203288Srnoland#define VIA_VR_QUEUE_BUSY 0x00020000 /* Virtual Queue is busy */ 59203288Srnoland 60203288Srnoland#define SetReg2DAGP(nReg, nData) { \ 61203288Srnoland *((uint32_t *)(vb)) = ((nReg) >> 2) | HALCYON_HEADER1; \ 62203288Srnoland *((uint32_t *)(vb) + 1) = (nData); \ 63203288Srnoland vb = ((uint32_t *)vb) + 2; \ 64203288Srnoland dev_priv->dma_low +=8; \ 65203288Srnoland} 66203288Srnoland 67203288Srnoland#define via_flush_write_combine() DRM_MEMORYBARRIER() 68203288Srnoland 69203288Srnoland#define VIA_OUT_RING_QW(w1,w2) \ 70203288Srnoland *vb++ = (w1); \ 71203288Srnoland *vb++ = (w2); \ 72203288Srnoland dev_priv->dma_low += 8; 73203288Srnoland 74203288Srnolandstatic void via_cmdbuf_start(drm_via_private_t * dev_priv); 75203288Srnolandstatic void via_cmdbuf_pause(drm_via_private_t * dev_priv); 76203288Srnolandstatic void via_cmdbuf_reset(drm_via_private_t * dev_priv); 77203288Srnolandstatic void via_cmdbuf_rewind(drm_via_private_t * dev_priv); 78203288Srnolandstatic int via_wait_idle(drm_via_private_t * dev_priv); 79203288Srnolandstatic void via_pad_cache(drm_via_private_t * dev_priv, int qwords); 80203288Srnoland 81203288Srnoland/* 82203288Srnoland * Free space in command buffer. 83203288Srnoland */ 84203288Srnoland 85203288Srnolandstatic uint32_t via_cmdbuf_space(drm_via_private_t * dev_priv) 86203288Srnoland{ 87203288Srnoland uint32_t agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr; 88203288Srnoland uint32_t hw_addr = *(dev_priv->hw_addr_ptr) - agp_base; 89203288Srnoland 90203288Srnoland return ((hw_addr <= dev_priv->dma_low) ? 91203288Srnoland (dev_priv->dma_high + hw_addr - dev_priv->dma_low) : 92203288Srnoland (hw_addr - dev_priv->dma_low)); 93203288Srnoland} 94203288Srnoland 95203288Srnoland/* 96203288Srnoland * How much does the command regulator lag behind? 97203288Srnoland */ 98203288Srnoland 99203288Srnolandstatic uint32_t via_cmdbuf_lag(drm_via_private_t * dev_priv) 100203288Srnoland{ 101203288Srnoland uint32_t agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr; 102203288Srnoland uint32_t hw_addr = *(dev_priv->hw_addr_ptr) - agp_base; 103203288Srnoland 104203288Srnoland return ((hw_addr <= dev_priv->dma_low) ? 105203288Srnoland (dev_priv->dma_low - hw_addr) : 106203288Srnoland (dev_priv->dma_wrap + dev_priv->dma_low - hw_addr)); 107203288Srnoland} 108203288Srnoland 109203288Srnoland/* 110203288Srnoland * Check that the given size fits in the buffer, otherwise wait. 111203288Srnoland */ 112203288Srnoland 113203288Srnolandstatic inline int 114203288Srnolandvia_cmdbuf_wait(drm_via_private_t * dev_priv, unsigned int size) 115203288Srnoland{ 116203288Srnoland uint32_t agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr; 117203288Srnoland uint32_t cur_addr, hw_addr, next_addr; 118203288Srnoland volatile uint32_t *hw_addr_ptr; 119203288Srnoland uint32_t count; 120203288Srnoland hw_addr_ptr = dev_priv->hw_addr_ptr; 121203288Srnoland cur_addr = dev_priv->dma_low; 122203288Srnoland next_addr = cur_addr + size + 512 * 1024; 123203288Srnoland count = 1000000; 124203288Srnoland do { 125203288Srnoland hw_addr = *hw_addr_ptr - agp_base; 126203288Srnoland if (count-- == 0) { 127203288Srnoland DRM_ERROR 128203288Srnoland ("via_cmdbuf_wait timed out hw %x cur_addr %x next_addr %x\n", 129203288Srnoland hw_addr, cur_addr, next_addr); 130203288Srnoland return -1; 131203288Srnoland } 132203288Srnoland if ((cur_addr < hw_addr) && (next_addr >= hw_addr)) 133203288Srnoland DRM_UDELAY(1000); 134203288Srnoland } while ((cur_addr < hw_addr) && (next_addr >= hw_addr)); 135203288Srnoland return 0; 136203288Srnoland} 137203288Srnoland 138203288Srnoland/* 139203288Srnoland * Checks whether buffer head has reach the end. Rewind the ring buffer 140203288Srnoland * when necessary. 141203288Srnoland * 142203288Srnoland * Returns virtual pointer to ring buffer. 143203288Srnoland */ 144203288Srnoland 145203288Srnolandstatic inline uint32_t *via_check_dma(drm_via_private_t * dev_priv, 146203288Srnoland unsigned int size) 147203288Srnoland{ 148203288Srnoland if ((dev_priv->dma_low + size + 4 * CMDBUF_ALIGNMENT_SIZE) > 149203288Srnoland dev_priv->dma_high) { 150203288Srnoland via_cmdbuf_rewind(dev_priv); 151203288Srnoland } 152203288Srnoland if (via_cmdbuf_wait(dev_priv, size) != 0) { 153203288Srnoland return NULL; 154203288Srnoland } 155203288Srnoland 156203288Srnoland return (uint32_t *) (dev_priv->dma_ptr + dev_priv->dma_low); 157203288Srnoland} 158203288Srnoland 159203288Srnolandint via_dma_cleanup(struct drm_device * dev) 160203288Srnoland{ 161207066Srnoland drm_via_blitq_t *blitq; 162207066Srnoland int i; 163207066Srnoland 164203288Srnoland if (dev->dev_private) { 165203288Srnoland drm_via_private_t *dev_priv = 166203288Srnoland (drm_via_private_t *) dev->dev_private; 167203288Srnoland 168203288Srnoland if (dev_priv->ring.virtual_start) { 169203288Srnoland via_cmdbuf_reset(dev_priv); 170203288Srnoland 171203288Srnoland drm_core_ioremapfree(&dev_priv->ring.map, dev); 172203288Srnoland dev_priv->ring.virtual_start = NULL; 173203288Srnoland } 174203288Srnoland 175207066Srnoland for (i=0; i< VIA_NUM_BLIT_ENGINES; ++i) { 176207066Srnoland blitq = dev_priv->blit_queues + i; 177207066Srnoland mtx_destroy(&blitq->blit_lock); 178207066Srnoland } 179203288Srnoland } 180203288Srnoland 181203288Srnoland return 0; 182203288Srnoland} 183203288Srnoland 184203288Srnolandstatic int via_initialize(struct drm_device * dev, 185203288Srnoland drm_via_private_t * dev_priv, 186203288Srnoland drm_via_dma_init_t * init) 187203288Srnoland{ 188203288Srnoland if (!dev_priv || !dev_priv->mmio) { 189203288Srnoland DRM_ERROR("via_dma_init called before via_map_init\n"); 190203288Srnoland return -EFAULT; 191203288Srnoland } 192203288Srnoland 193203288Srnoland if (dev_priv->ring.virtual_start != NULL) { 194203288Srnoland DRM_ERROR("called again without calling cleanup\n"); 195203288Srnoland return -EFAULT; 196203288Srnoland } 197203288Srnoland 198203288Srnoland if (!dev->agp || !dev->agp->base) { 199203288Srnoland DRM_ERROR("called with no agp memory available\n"); 200203288Srnoland return -EFAULT; 201203288Srnoland } 202203288Srnoland 203203288Srnoland if (dev_priv->chipset == VIA_DX9_0) { 204203288Srnoland DRM_ERROR("AGP DMA is not supported on this chip\n"); 205203288Srnoland return -EINVAL; 206203288Srnoland } 207203288Srnoland 208203288Srnoland dev_priv->ring.map.offset = dev->agp->base + init->offset; 209203288Srnoland dev_priv->ring.map.size = init->size; 210203288Srnoland dev_priv->ring.map.type = 0; 211203288Srnoland dev_priv->ring.map.flags = 0; 212203288Srnoland dev_priv->ring.map.mtrr = 0; 213203288Srnoland 214203288Srnoland drm_core_ioremap_wc(&dev_priv->ring.map, dev); 215203288Srnoland 216207066Srnoland if (dev_priv->ring.map.virtual == NULL) { 217203288Srnoland via_dma_cleanup(dev); 218203288Srnoland DRM_ERROR("can not ioremap virtual address for" 219203288Srnoland " ring buffer\n"); 220203288Srnoland return -ENOMEM; 221203288Srnoland } 222203288Srnoland 223207066Srnoland dev_priv->ring.virtual_start = dev_priv->ring.map.virtual; 224203288Srnoland 225203288Srnoland dev_priv->dma_ptr = dev_priv->ring.virtual_start; 226203288Srnoland dev_priv->dma_low = 0; 227203288Srnoland dev_priv->dma_high = init->size; 228203288Srnoland dev_priv->dma_wrap = init->size; 229203288Srnoland dev_priv->dma_offset = init->offset; 230203288Srnoland dev_priv->last_pause_ptr = NULL; 231203288Srnoland dev_priv->hw_addr_ptr = 232207066Srnoland (volatile uint32_t *)((char *)dev_priv->mmio->virtual + 233203288Srnoland init->reg_pause_addr); 234203288Srnoland 235203288Srnoland via_cmdbuf_start(dev_priv); 236203288Srnoland 237203288Srnoland return 0; 238203288Srnoland} 239203288Srnoland 240203288Srnolandstatic int via_dma_init(struct drm_device *dev, void *data, struct drm_file *file_priv) 241203288Srnoland{ 242203288Srnoland drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private; 243203288Srnoland drm_via_dma_init_t *init = data; 244203288Srnoland int retcode = 0; 245203288Srnoland 246203288Srnoland switch (init->func) { 247203288Srnoland case VIA_INIT_DMA: 248203288Srnoland if (!DRM_SUSER(DRM_CURPROC)) 249203288Srnoland retcode = -EPERM; 250203288Srnoland else 251203288Srnoland retcode = via_initialize(dev, dev_priv, init); 252203288Srnoland break; 253203288Srnoland case VIA_CLEANUP_DMA: 254203288Srnoland if (!DRM_SUSER(DRM_CURPROC)) 255203288Srnoland retcode = -EPERM; 256203288Srnoland else 257203288Srnoland retcode = via_dma_cleanup(dev); 258203288Srnoland break; 259203288Srnoland case VIA_DMA_INITIALIZED: 260203288Srnoland retcode = (dev_priv->ring.virtual_start != NULL) ? 261203288Srnoland 0 : -EFAULT; 262203288Srnoland break; 263203288Srnoland default: 264203288Srnoland retcode = -EINVAL; 265203288Srnoland break; 266203288Srnoland } 267203288Srnoland 268203288Srnoland return retcode; 269203288Srnoland} 270203288Srnoland 271203288Srnolandstatic int via_dispatch_cmdbuffer(struct drm_device * dev, drm_via_cmdbuffer_t * cmd) 272203288Srnoland{ 273203288Srnoland drm_via_private_t *dev_priv; 274203288Srnoland uint32_t *vb; 275203288Srnoland int ret; 276203288Srnoland 277203288Srnoland dev_priv = (drm_via_private_t *) dev->dev_private; 278203288Srnoland 279203288Srnoland if (dev_priv->ring.virtual_start == NULL) { 280203288Srnoland DRM_ERROR("called without initializing AGP ring buffer.\n"); 281203288Srnoland return -EFAULT; 282203288Srnoland } 283203288Srnoland 284203288Srnoland if (cmd->size > VIA_PCI_BUF_SIZE) { 285203288Srnoland return -ENOMEM; 286203288Srnoland } 287203288Srnoland 288203288Srnoland if (DRM_COPY_FROM_USER(dev_priv->pci_buf, cmd->buf, cmd->size)) 289203288Srnoland return -EFAULT; 290203288Srnoland 291203288Srnoland /* 292203288Srnoland * Running this function on AGP memory is dead slow. Therefore 293203288Srnoland * we run it on a temporary cacheable system memory buffer and 294203288Srnoland * copy it to AGP memory when ready. 295203288Srnoland */ 296203288Srnoland 297203288Srnoland if ((ret = 298203288Srnoland via_verify_command_stream((uint32_t *) dev_priv->pci_buf, 299203288Srnoland cmd->size, dev, 1))) { 300203288Srnoland return ret; 301203288Srnoland } 302203288Srnoland 303203288Srnoland vb = via_check_dma(dev_priv, (cmd->size < 0x100) ? 0x102 : cmd->size); 304203288Srnoland if (vb == NULL) { 305203288Srnoland return -EAGAIN; 306203288Srnoland } 307203288Srnoland 308203288Srnoland memcpy(vb, dev_priv->pci_buf, cmd->size); 309203288Srnoland 310203288Srnoland dev_priv->dma_low += cmd->size; 311203288Srnoland 312203288Srnoland /* 313203288Srnoland * Small submissions somehow stalls the CPU. (AGP cache effects?) 314203288Srnoland * pad to greater size. 315203288Srnoland */ 316203288Srnoland 317203288Srnoland if (cmd->size < 0x100) 318203288Srnoland via_pad_cache(dev_priv, (0x100 - cmd->size) >> 3); 319203288Srnoland via_cmdbuf_pause(dev_priv); 320203288Srnoland 321203288Srnoland return 0; 322203288Srnoland} 323203288Srnoland 324203288Srnolandint via_driver_dma_quiescent(struct drm_device * dev) 325203288Srnoland{ 326203288Srnoland drm_via_private_t *dev_priv = dev->dev_private; 327203288Srnoland 328203288Srnoland if (!via_wait_idle(dev_priv)) { 329203288Srnoland return -EBUSY; 330203288Srnoland } 331203288Srnoland return 0; 332203288Srnoland} 333203288Srnoland 334203288Srnolandstatic int via_flush_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) 335203288Srnoland{ 336203288Srnoland 337203288Srnoland LOCK_TEST_WITH_RETURN(dev, file_priv); 338203288Srnoland 339203288Srnoland return via_driver_dma_quiescent(dev); 340203288Srnoland} 341203288Srnoland 342203288Srnolandstatic int via_cmdbuffer(struct drm_device *dev, void *data, struct drm_file *file_priv) 343203288Srnoland{ 344203288Srnoland drm_via_cmdbuffer_t *cmdbuf = data; 345203288Srnoland int ret; 346203288Srnoland 347203288Srnoland LOCK_TEST_WITH_RETURN(dev, file_priv); 348203288Srnoland 349203288Srnoland DRM_DEBUG("buf %p size %lu\n", cmdbuf->buf, cmdbuf->size); 350203288Srnoland 351203288Srnoland ret = via_dispatch_cmdbuffer(dev, cmdbuf); 352203288Srnoland if (ret) { 353203288Srnoland return ret; 354203288Srnoland } 355203288Srnoland 356203288Srnoland return 0; 357203288Srnoland} 358203288Srnoland 359203288Srnolandstatic int via_dispatch_pci_cmdbuffer(struct drm_device * dev, 360203288Srnoland drm_via_cmdbuffer_t * cmd) 361203288Srnoland{ 362203288Srnoland drm_via_private_t *dev_priv = dev->dev_private; 363203288Srnoland int ret; 364203288Srnoland 365203288Srnoland if (cmd->size > VIA_PCI_BUF_SIZE) { 366203288Srnoland return -ENOMEM; 367203288Srnoland } 368203288Srnoland if (DRM_COPY_FROM_USER(dev_priv->pci_buf, cmd->buf, cmd->size)) 369203288Srnoland return -EFAULT; 370203288Srnoland 371203288Srnoland if ((ret = 372203288Srnoland via_verify_command_stream((uint32_t *) dev_priv->pci_buf, 373203288Srnoland cmd->size, dev, 0))) { 374203288Srnoland return ret; 375203288Srnoland } 376203288Srnoland 377203288Srnoland ret = 378203288Srnoland via_parse_command_stream(dev, (const uint32_t *)dev_priv->pci_buf, 379203288Srnoland cmd->size); 380203288Srnoland return ret; 381203288Srnoland} 382203288Srnoland 383203288Srnolandstatic int via_pci_cmdbuffer(struct drm_device *dev, void *data, struct drm_file *file_priv) 384203288Srnoland{ 385203288Srnoland drm_via_cmdbuffer_t *cmdbuf = data; 386203288Srnoland int ret; 387203288Srnoland 388203288Srnoland LOCK_TEST_WITH_RETURN(dev, file_priv); 389203288Srnoland 390203288Srnoland DRM_DEBUG("buf %p size %lu\n", cmdbuf->buf, cmdbuf->size); 391203288Srnoland 392203288Srnoland ret = via_dispatch_pci_cmdbuffer(dev, cmdbuf); 393203288Srnoland if (ret) { 394203288Srnoland return ret; 395203288Srnoland } 396203288Srnoland 397203288Srnoland return 0; 398203288Srnoland} 399203288Srnoland 400203288Srnolandstatic inline uint32_t *via_align_buffer(drm_via_private_t * dev_priv, 401203288Srnoland uint32_t * vb, int qw_count) 402203288Srnoland{ 403203288Srnoland for (; qw_count > 0; --qw_count) { 404203288Srnoland VIA_OUT_RING_QW(HC_DUMMY, HC_DUMMY); 405203288Srnoland } 406203288Srnoland return vb; 407203288Srnoland} 408203288Srnoland 409203288Srnoland/* 410203288Srnoland * This function is used internally by ring buffer management code. 411203288Srnoland * 412203288Srnoland * Returns virtual pointer to ring buffer. 413203288Srnoland */ 414203288Srnolandstatic inline uint32_t *via_get_dma(drm_via_private_t * dev_priv) 415203288Srnoland{ 416203288Srnoland return (uint32_t *) (dev_priv->dma_ptr + dev_priv->dma_low); 417203288Srnoland} 418203288Srnoland 419203288Srnoland/* 420203288Srnoland * Hooks a segment of data into the tail of the ring-buffer by 421203288Srnoland * modifying the pause address stored in the buffer itself. If 422203288Srnoland * the regulator has already paused, restart it. 423203288Srnoland */ 424203288Srnolandstatic int via_hook_segment(drm_via_private_t * dev_priv, 425203288Srnoland uint32_t pause_addr_hi, uint32_t pause_addr_lo, 426203288Srnoland int no_pci_fire) 427203288Srnoland{ 428203288Srnoland int paused, count; 429203288Srnoland volatile uint32_t *paused_at = dev_priv->last_pause_ptr; 430203288Srnoland uint32_t reader,ptr; 431203288Srnoland uint32_t diff; 432203288Srnoland 433203288Srnoland paused = 0; 434203288Srnoland via_flush_write_combine(); 435203288Srnoland (void) *(volatile uint32_t *)(via_get_dma(dev_priv) -1); 436203288Srnoland 437203288Srnoland *paused_at = pause_addr_lo; 438203288Srnoland via_flush_write_combine(); 439203288Srnoland (void) *paused_at; 440203288Srnoland 441203288Srnoland reader = *(dev_priv->hw_addr_ptr); 442203288Srnoland ptr = ((volatile char *)paused_at - dev_priv->dma_ptr) + 443203288Srnoland dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr + 4; 444203288Srnoland 445203288Srnoland dev_priv->last_pause_ptr = via_get_dma(dev_priv) - 1; 446203288Srnoland 447203288Srnoland /* 448203288Srnoland * If there is a possibility that the command reader will 449203288Srnoland * miss the new pause address and pause on the old one, 450203288Srnoland * In that case we need to program the new start address 451203288Srnoland * using PCI. 452203288Srnoland */ 453203288Srnoland 454203288Srnoland diff = (uint32_t) (ptr - reader) - dev_priv->dma_diff; 455203288Srnoland count = 10000000; 456203288Srnoland while(diff == 0 && count--) { 457203288Srnoland paused = (VIA_READ(0x41c) & 0x80000000); 458203288Srnoland if (paused) 459203288Srnoland break; 460203288Srnoland reader = *(dev_priv->hw_addr_ptr); 461203288Srnoland diff = (uint32_t) (ptr - reader) - dev_priv->dma_diff; 462203288Srnoland } 463203288Srnoland 464203288Srnoland paused = VIA_READ(0x41c) & 0x80000000; 465203288Srnoland 466203288Srnoland if (paused && !no_pci_fire) { 467203288Srnoland reader = *(dev_priv->hw_addr_ptr); 468203288Srnoland diff = (uint32_t) (ptr - reader) - dev_priv->dma_diff; 469203288Srnoland diff &= (dev_priv->dma_high - 1); 470203288Srnoland if (diff != 0 && diff < (dev_priv->dma_high >> 1)) { 471203288Srnoland DRM_ERROR("Paused at incorrect address. " 472203288Srnoland "0x%08x, 0x%08x 0x%08x\n", 473203288Srnoland ptr, reader, dev_priv->dma_diff); 474203288Srnoland } else if (diff == 0) { 475203288Srnoland /* 476203288Srnoland * There is a concern that these writes may stall the PCI bus 477203288Srnoland * if the GPU is not idle. However, idling the GPU first 478203288Srnoland * doesn't make a difference. 479203288Srnoland */ 480203288Srnoland 481203288Srnoland VIA_WRITE(VIA_REG_TRANSET, (HC_ParaType_PreCR << 16)); 482203288Srnoland VIA_WRITE(VIA_REG_TRANSPACE, pause_addr_hi); 483203288Srnoland VIA_WRITE(VIA_REG_TRANSPACE, pause_addr_lo); 484242825Srdivacky (void)VIA_READ(VIA_REG_TRANSPACE); 485203288Srnoland } 486203288Srnoland } 487203288Srnoland return paused; 488203288Srnoland} 489203288Srnoland 490203288Srnolandstatic int via_wait_idle(drm_via_private_t * dev_priv) 491203288Srnoland{ 492203288Srnoland int count = 10000000; 493203288Srnoland 494203288Srnoland while (!(VIA_READ(VIA_REG_STATUS) & VIA_VR_QUEUE_BUSY) && --count) 495203288Srnoland ; 496203288Srnoland 497203288Srnoland while (count && (VIA_READ(VIA_REG_STATUS) & 498203288Srnoland (VIA_CMD_RGTR_BUSY | VIA_2D_ENG_BUSY | 499203288Srnoland VIA_3D_ENG_BUSY))) 500203288Srnoland --count; 501203288Srnoland return count; 502203288Srnoland} 503203288Srnoland 504203288Srnolandstatic uint32_t *via_align_cmd(drm_via_private_t * dev_priv, uint32_t cmd_type, 505203288Srnoland uint32_t addr, uint32_t * cmd_addr_hi, 506203288Srnoland uint32_t * cmd_addr_lo, int skip_wait) 507203288Srnoland{ 508203288Srnoland uint32_t agp_base; 509203288Srnoland uint32_t cmd_addr, addr_lo, addr_hi; 510203288Srnoland uint32_t *vb; 511203288Srnoland uint32_t qw_pad_count; 512203288Srnoland 513203288Srnoland if (!skip_wait) 514203288Srnoland via_cmdbuf_wait(dev_priv, 2 * CMDBUF_ALIGNMENT_SIZE); 515203288Srnoland 516203288Srnoland vb = via_get_dma(dev_priv); 517203288Srnoland VIA_OUT_RING_QW(HC_HEADER2 | ((VIA_REG_TRANSET >> 2) << 12) | 518203288Srnoland (VIA_REG_TRANSPACE >> 2), HC_ParaType_PreCR << 16); 519203288Srnoland agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr; 520203288Srnoland qw_pad_count = (CMDBUF_ALIGNMENT_SIZE >> 3) - 521203288Srnoland ((dev_priv->dma_low & CMDBUF_ALIGNMENT_MASK) >> 3); 522203288Srnoland 523203288Srnoland cmd_addr = (addr) ? addr : 524203288Srnoland agp_base + dev_priv->dma_low - 8 + (qw_pad_count << 3); 525203288Srnoland addr_lo = ((HC_SubA_HAGPBpL << 24) | (cmd_type & HC_HAGPBpID_MASK) | 526203288Srnoland (cmd_addr & HC_HAGPBpL_MASK)); 527203288Srnoland addr_hi = ((HC_SubA_HAGPBpH << 24) | (cmd_addr >> 24)); 528203288Srnoland 529203288Srnoland vb = via_align_buffer(dev_priv, vb, qw_pad_count - 1); 530203288Srnoland VIA_OUT_RING_QW(*cmd_addr_hi = addr_hi, *cmd_addr_lo = addr_lo); 531203288Srnoland return vb; 532203288Srnoland} 533203288Srnoland 534203288Srnolandstatic void via_cmdbuf_start(drm_via_private_t * dev_priv) 535203288Srnoland{ 536203288Srnoland uint32_t pause_addr_lo, pause_addr_hi; 537203288Srnoland uint32_t start_addr, start_addr_lo; 538203288Srnoland uint32_t end_addr, end_addr_lo; 539203288Srnoland uint32_t command; 540203288Srnoland uint32_t agp_base; 541203288Srnoland uint32_t ptr; 542203288Srnoland uint32_t reader; 543203288Srnoland int count; 544203288Srnoland 545203288Srnoland dev_priv->dma_low = 0; 546203288Srnoland 547203288Srnoland agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr; 548203288Srnoland start_addr = agp_base; 549203288Srnoland end_addr = agp_base + dev_priv->dma_high; 550203288Srnoland 551203288Srnoland start_addr_lo = ((HC_SubA_HAGPBstL << 24) | (start_addr & 0xFFFFFF)); 552203288Srnoland end_addr_lo = ((HC_SubA_HAGPBendL << 24) | (end_addr & 0xFFFFFF)); 553203288Srnoland command = ((HC_SubA_HAGPCMNT << 24) | (start_addr >> 24) | 554203288Srnoland ((end_addr & 0xff000000) >> 16)); 555203288Srnoland 556203288Srnoland dev_priv->last_pause_ptr = 557203288Srnoland via_align_cmd(dev_priv, HC_HAGPBpID_PAUSE, 0, 558203288Srnoland &pause_addr_hi, &pause_addr_lo, 1) - 1; 559203288Srnoland 560203288Srnoland via_flush_write_combine(); 561203288Srnoland (void) *(volatile uint32_t *)dev_priv->last_pause_ptr; 562203288Srnoland 563203288Srnoland VIA_WRITE(VIA_REG_TRANSET, (HC_ParaType_PreCR << 16)); 564203288Srnoland VIA_WRITE(VIA_REG_TRANSPACE, command); 565203288Srnoland VIA_WRITE(VIA_REG_TRANSPACE, start_addr_lo); 566203288Srnoland VIA_WRITE(VIA_REG_TRANSPACE, end_addr_lo); 567203288Srnoland 568203288Srnoland VIA_WRITE(VIA_REG_TRANSPACE, pause_addr_hi); 569203288Srnoland VIA_WRITE(VIA_REG_TRANSPACE, pause_addr_lo); 570203288Srnoland DRM_WRITEMEMORYBARRIER(); 571203288Srnoland VIA_WRITE(VIA_REG_TRANSPACE, command | HC_HAGPCMNT_MASK); 572242825Srdivacky (void)VIA_READ(VIA_REG_TRANSPACE); 573203288Srnoland 574203288Srnoland dev_priv->dma_diff = 0; 575203288Srnoland 576203288Srnoland count = 10000000; 577203288Srnoland while (!(VIA_READ(0x41c) & 0x80000000) && count--); 578203288Srnoland 579203288Srnoland reader = *(dev_priv->hw_addr_ptr); 580203288Srnoland ptr = ((volatile char *)dev_priv->last_pause_ptr - dev_priv->dma_ptr) + 581203288Srnoland dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr + 4; 582203288Srnoland 583203288Srnoland /* 584203288Srnoland * This is the difference between where we tell the 585203288Srnoland * command reader to pause and where it actually pauses. 586203288Srnoland * This differs between hw implementation so we need to 587203288Srnoland * detect it. 588203288Srnoland */ 589203288Srnoland 590203288Srnoland dev_priv->dma_diff = ptr - reader; 591203288Srnoland} 592203288Srnoland 593203288Srnolandstatic void via_pad_cache(drm_via_private_t * dev_priv, int qwords) 594203288Srnoland{ 595203288Srnoland uint32_t *vb; 596203288Srnoland 597203288Srnoland via_cmdbuf_wait(dev_priv, qwords + 2); 598203288Srnoland vb = via_get_dma(dev_priv); 599203288Srnoland VIA_OUT_RING_QW(HC_HEADER2, HC_ParaType_NotTex << 16); 600203288Srnoland via_align_buffer(dev_priv, vb, qwords); 601203288Srnoland} 602203288Srnoland 603203288Srnolandstatic inline void via_dummy_bitblt(drm_via_private_t * dev_priv) 604203288Srnoland{ 605203288Srnoland uint32_t *vb = via_get_dma(dev_priv); 606203288Srnoland SetReg2DAGP(0x0C, (0 | (0 << 16))); 607203288Srnoland SetReg2DAGP(0x10, 0 | (0 << 16)); 608203288Srnoland SetReg2DAGP(0x0, 0x1 | 0x2000 | 0xAA000000); 609203288Srnoland} 610203288Srnoland 611203288Srnolandstatic void via_cmdbuf_jump(drm_via_private_t * dev_priv) 612203288Srnoland{ 613203288Srnoland uint32_t agp_base; 614203288Srnoland uint32_t pause_addr_lo, pause_addr_hi; 615203288Srnoland uint32_t jump_addr_lo, jump_addr_hi; 616203288Srnoland volatile uint32_t *last_pause_ptr; 617203288Srnoland uint32_t dma_low_save1, dma_low_save2; 618203288Srnoland 619203288Srnoland agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr; 620203288Srnoland via_align_cmd(dev_priv, HC_HAGPBpID_JUMP, 0, &jump_addr_hi, 621203288Srnoland &jump_addr_lo, 0); 622203288Srnoland 623203288Srnoland dev_priv->dma_wrap = dev_priv->dma_low; 624203288Srnoland 625203288Srnoland /* 626203288Srnoland * Wrap command buffer to the beginning. 627203288Srnoland */ 628203288Srnoland 629203288Srnoland dev_priv->dma_low = 0; 630203288Srnoland if (via_cmdbuf_wait(dev_priv, CMDBUF_ALIGNMENT_SIZE) != 0) { 631203288Srnoland DRM_ERROR("via_cmdbuf_jump failed\n"); 632203288Srnoland } 633203288Srnoland 634203288Srnoland via_dummy_bitblt(dev_priv); 635203288Srnoland via_dummy_bitblt(dev_priv); 636203288Srnoland 637203288Srnoland last_pause_ptr = 638203288Srnoland via_align_cmd(dev_priv, HC_HAGPBpID_PAUSE, 0, &pause_addr_hi, 639203288Srnoland &pause_addr_lo, 0) - 1; 640203288Srnoland via_align_cmd(dev_priv, HC_HAGPBpID_PAUSE, 0, &pause_addr_hi, 641203288Srnoland &pause_addr_lo, 0); 642203288Srnoland 643203288Srnoland *last_pause_ptr = pause_addr_lo; 644203288Srnoland dma_low_save1 = dev_priv->dma_low; 645203288Srnoland 646203288Srnoland /* 647203288Srnoland * Now, set a trap that will pause the regulator if it tries to rerun the old 648203288Srnoland * command buffer. (Which may happen if via_hook_segment detecs a command regulator pause 649203288Srnoland * and reissues the jump command over PCI, while the regulator has already taken the jump 650203288Srnoland * and actually paused at the current buffer end). 651203288Srnoland * There appears to be no other way to detect this condition, since the hw_addr_pointer 652203288Srnoland * does not seem to get updated immediately when a jump occurs. 653203288Srnoland */ 654203288Srnoland 655203288Srnoland last_pause_ptr = 656203288Srnoland via_align_cmd(dev_priv, HC_HAGPBpID_PAUSE, 0, &pause_addr_hi, 657203288Srnoland &pause_addr_lo, 0) - 1; 658203288Srnoland via_align_cmd(dev_priv, HC_HAGPBpID_PAUSE, 0, &pause_addr_hi, 659203288Srnoland &pause_addr_lo, 0); 660203288Srnoland *last_pause_ptr = pause_addr_lo; 661203288Srnoland 662203288Srnoland dma_low_save2 = dev_priv->dma_low; 663203288Srnoland dev_priv->dma_low = dma_low_save1; 664203288Srnoland via_hook_segment(dev_priv, jump_addr_hi, jump_addr_lo, 0); 665203288Srnoland dev_priv->dma_low = dma_low_save2; 666203288Srnoland via_hook_segment(dev_priv, pause_addr_hi, pause_addr_lo, 0); 667203288Srnoland} 668203288Srnoland 669203288Srnoland 670203288Srnolandstatic void via_cmdbuf_rewind(drm_via_private_t * dev_priv) 671203288Srnoland{ 672203288Srnoland via_cmdbuf_jump(dev_priv); 673203288Srnoland} 674203288Srnoland 675203288Srnolandstatic void via_cmdbuf_flush(drm_via_private_t * dev_priv, uint32_t cmd_type) 676203288Srnoland{ 677203288Srnoland uint32_t pause_addr_lo, pause_addr_hi; 678203288Srnoland 679203288Srnoland via_align_cmd(dev_priv, cmd_type, 0, &pause_addr_hi, &pause_addr_lo, 0); 680203288Srnoland via_hook_segment(dev_priv, pause_addr_hi, pause_addr_lo, 0); 681203288Srnoland} 682203288Srnoland 683203288Srnolandstatic void via_cmdbuf_pause(drm_via_private_t * dev_priv) 684203288Srnoland{ 685203288Srnoland via_cmdbuf_flush(dev_priv, HC_HAGPBpID_PAUSE); 686203288Srnoland} 687203288Srnoland 688203288Srnolandstatic void via_cmdbuf_reset(drm_via_private_t * dev_priv) 689203288Srnoland{ 690203288Srnoland via_cmdbuf_flush(dev_priv, HC_HAGPBpID_STOP); 691203288Srnoland via_wait_idle(dev_priv); 692203288Srnoland} 693203288Srnoland 694203288Srnoland/* 695203288Srnoland * User interface to the space and lag functions. 696203288Srnoland */ 697203288Srnoland 698203288Srnolandstatic int via_cmdbuf_size(struct drm_device *dev, void *data, struct drm_file *file_priv) 699203288Srnoland{ 700203288Srnoland drm_via_cmdbuf_size_t *d_siz = data; 701203288Srnoland int ret = 0; 702203288Srnoland uint32_t tmp_size, count; 703203288Srnoland drm_via_private_t *dev_priv; 704203288Srnoland 705203288Srnoland DRM_DEBUG("\n"); 706203288Srnoland LOCK_TEST_WITH_RETURN(dev, file_priv); 707203288Srnoland 708203288Srnoland dev_priv = (drm_via_private_t *) dev->dev_private; 709203288Srnoland 710203288Srnoland if (dev_priv->ring.virtual_start == NULL) { 711203288Srnoland DRM_ERROR("called without initializing AGP ring buffer.\n"); 712203288Srnoland return -EFAULT; 713203288Srnoland } 714203288Srnoland 715203288Srnoland count = 1000000; 716203288Srnoland tmp_size = d_siz->size; 717203288Srnoland switch (d_siz->func) { 718203288Srnoland case VIA_CMDBUF_SPACE: 719203288Srnoland while (((tmp_size = via_cmdbuf_space(dev_priv)) < d_siz->size) 720203288Srnoland && --count) { 721203288Srnoland if (!d_siz->wait) { 722203288Srnoland break; 723203288Srnoland } 724203288Srnoland } 725203288Srnoland if (!count) { 726203288Srnoland DRM_ERROR("VIA_CMDBUF_SPACE timed out.\n"); 727203288Srnoland ret = -EAGAIN; 728203288Srnoland } 729203288Srnoland break; 730203288Srnoland case VIA_CMDBUF_LAG: 731203288Srnoland while (((tmp_size = via_cmdbuf_lag(dev_priv)) > d_siz->size) 732203288Srnoland && --count) { 733203288Srnoland if (!d_siz->wait) { 734203288Srnoland break; 735203288Srnoland } 736203288Srnoland } 737203288Srnoland if (!count) { 738203288Srnoland DRM_ERROR("VIA_CMDBUF_LAG timed out.\n"); 739203288Srnoland ret = -EAGAIN; 740203288Srnoland } 741203288Srnoland break; 742203288Srnoland default: 743203288Srnoland ret = -EFAULT; 744203288Srnoland } 745203288Srnoland d_siz->size = tmp_size; 746203288Srnoland 747203288Srnoland return ret; 748203288Srnoland} 749203288Srnoland 750203288Srnolandstruct drm_ioctl_desc via_ioctls[] = { 751203288Srnoland DRM_IOCTL_DEF(DRM_VIA_ALLOCMEM, via_mem_alloc, DRM_AUTH), 752203288Srnoland DRM_IOCTL_DEF(DRM_VIA_FREEMEM, via_mem_free, DRM_AUTH), 753203288Srnoland DRM_IOCTL_DEF(DRM_VIA_AGP_INIT, via_agp_init, DRM_AUTH|DRM_MASTER), 754203288Srnoland DRM_IOCTL_DEF(DRM_VIA_FB_INIT, via_fb_init, DRM_AUTH|DRM_MASTER), 755203288Srnoland DRM_IOCTL_DEF(DRM_VIA_MAP_INIT, via_map_init, DRM_AUTH|DRM_MASTER), 756203288Srnoland DRM_IOCTL_DEF(DRM_VIA_DEC_FUTEX, via_decoder_futex, DRM_AUTH), 757203288Srnoland DRM_IOCTL_DEF(DRM_VIA_DMA_INIT, via_dma_init, DRM_AUTH), 758203288Srnoland DRM_IOCTL_DEF(DRM_VIA_CMDBUFFER, via_cmdbuffer, DRM_AUTH), 759203288Srnoland DRM_IOCTL_DEF(DRM_VIA_FLUSH, via_flush_ioctl, DRM_AUTH), 760203288Srnoland DRM_IOCTL_DEF(DRM_VIA_PCICMD, via_pci_cmdbuffer, DRM_AUTH), 761203288Srnoland DRM_IOCTL_DEF(DRM_VIA_CMDBUF_SIZE, via_cmdbuf_size, DRM_AUTH), 762203288Srnoland DRM_IOCTL_DEF(DRM_VIA_WAIT_IRQ, via_wait_irq, DRM_AUTH), 763203288Srnoland DRM_IOCTL_DEF(DRM_VIA_DMA_BLIT, via_dma_blit, DRM_AUTH), 764203288Srnoland DRM_IOCTL_DEF(DRM_VIA_BLIT_SYNC, via_dma_blit_sync, DRM_AUTH) 765203288Srnoland}; 766203288Srnoland 767203288Srnolandint via_max_ioctl = DRM_ARRAY_SIZE(via_ioctls); 768