1254885Sdumbbell/* r300_cmdbuf.c -- Command buffer emission for R300 -*- linux-c -*- 2254885Sdumbbell * 3254885Sdumbbell * Copyright (C) The Weather Channel, Inc. 2002. 4254885Sdumbbell * Copyright (C) 2004 Nicolai Haehnle. 5254885Sdumbbell * All Rights Reserved. 6254885Sdumbbell * 7254885Sdumbbell * The Weather Channel (TM) funded Tungsten Graphics to develop the 8254885Sdumbbell * initial release of the Radeon 8500 driver under the XFree86 license. 9254885Sdumbbell * This notice must be preserved. 10254885Sdumbbell * 11254885Sdumbbell * Permission is hereby granted, free of charge, to any person obtaining a 12254885Sdumbbell * copy of this software and associated documentation files (the "Software"), 13254885Sdumbbell * to deal in the Software without restriction, including without limitation 14254885Sdumbbell * the rights to use, copy, modify, merge, publish, distribute, sublicense, 15254885Sdumbbell * and/or sell copies of the Software, and to permit persons to whom the 16254885Sdumbbell * Software is furnished to do so, subject to the following conditions: 17254885Sdumbbell * 18254885Sdumbbell * The above copyright notice and this permission notice (including the next 19254885Sdumbbell * paragraph) shall be included in all copies or substantial portions of the 20254885Sdumbbell * Software. 21254885Sdumbbell * 22254885Sdumbbell * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 23254885Sdumbbell * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 24254885Sdumbbell * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 25254885Sdumbbell * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 26254885Sdumbbell * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 27254885Sdumbbell * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 28254885Sdumbbell * DEALINGS IN THE SOFTWARE. 29254885Sdumbbell * 30254885Sdumbbell * Authors: 31254885Sdumbbell * Nicolai Haehnle <prefect_@gmx.net> 32254885Sdumbbell */ 33254885Sdumbbell 34254885Sdumbbell#include <sys/cdefs.h> 35254885Sdumbbell__FBSDID("$FreeBSD$"); 36254885Sdumbbell 37254885Sdumbbell#include <dev/drm2/drmP.h> 38254885Sdumbbell#include <dev/drm2/drm_buffer.h> 39254885Sdumbbell#include <dev/drm2/radeon/radeon_drm.h> 40254885Sdumbbell#include "radeon_drv.h" 41254885Sdumbbell#include "r300_reg.h" 42254885Sdumbbell 43254885Sdumbbell#define R300_SIMULTANEOUS_CLIPRECTS 4 44254885Sdumbbell 45254885Sdumbbell/* Values for R300_RE_CLIPRECT_CNTL depending on the number of cliprects 46254885Sdumbbell */ 47254885Sdumbbellstatic const int r300_cliprect_cntl[4] = { 48254885Sdumbbell 0xAAAA, 49254885Sdumbbell 0xEEEE, 50254885Sdumbbell 0xFEFE, 51254885Sdumbbell 0xFFFE 52254885Sdumbbell}; 53254885Sdumbbell 54254885Sdumbbell/** 55254885Sdumbbell * Emit up to R300_SIMULTANEOUS_CLIPRECTS cliprects from the given command 56254885Sdumbbell * buffer, starting with index n. 57254885Sdumbbell */ 58254885Sdumbbellstatic int r300_emit_cliprects(drm_radeon_private_t *dev_priv, 59254885Sdumbbell drm_radeon_kcmd_buffer_t *cmdbuf, int n) 60254885Sdumbbell{ 61254885Sdumbbell struct drm_clip_rect box; 62254885Sdumbbell int nr; 63254885Sdumbbell int i; 64254885Sdumbbell RING_LOCALS; 65254885Sdumbbell 66254885Sdumbbell nr = cmdbuf->nbox - n; 67254885Sdumbbell if (nr > R300_SIMULTANEOUS_CLIPRECTS) 68254885Sdumbbell nr = R300_SIMULTANEOUS_CLIPRECTS; 69254885Sdumbbell 70254885Sdumbbell DRM_DEBUG("%i cliprects\n", nr); 71254885Sdumbbell 72254885Sdumbbell if (nr) { 73254885Sdumbbell BEGIN_RING(6 + nr * 2); 74254885Sdumbbell OUT_RING(CP_PACKET0(R300_RE_CLIPRECT_TL_0, nr * 2 - 1)); 75254885Sdumbbell 76254885Sdumbbell for (i = 0; i < nr; ++i) { 77254885Sdumbbell if (DRM_COPY_FROM_USER_UNCHECKED 78254885Sdumbbell (&box, &cmdbuf->boxes[n + i], sizeof(box))) { 79254885Sdumbbell DRM_ERROR("copy cliprect faulted\n"); 80254885Sdumbbell return -EFAULT; 81254885Sdumbbell } 82254885Sdumbbell 83254885Sdumbbell box.x2--; /* Hardware expects inclusive bottom-right corner */ 84254885Sdumbbell box.y2--; 85254885Sdumbbell 86254885Sdumbbell if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV515) { 87254885Sdumbbell box.x1 = (box.x1) & 88254885Sdumbbell R300_CLIPRECT_MASK; 89254885Sdumbbell box.y1 = (box.y1) & 90254885Sdumbbell R300_CLIPRECT_MASK; 91254885Sdumbbell box.x2 = (box.x2) & 92254885Sdumbbell R300_CLIPRECT_MASK; 93254885Sdumbbell box.y2 = (box.y2) & 94254885Sdumbbell R300_CLIPRECT_MASK; 95254885Sdumbbell } else { 96254885Sdumbbell box.x1 = (box.x1 + R300_CLIPRECT_OFFSET) & 97254885Sdumbbell R300_CLIPRECT_MASK; 98254885Sdumbbell box.y1 = (box.y1 + R300_CLIPRECT_OFFSET) & 99254885Sdumbbell R300_CLIPRECT_MASK; 100254885Sdumbbell box.x2 = (box.x2 + R300_CLIPRECT_OFFSET) & 101254885Sdumbbell R300_CLIPRECT_MASK; 102254885Sdumbbell box.y2 = (box.y2 + R300_CLIPRECT_OFFSET) & 103254885Sdumbbell R300_CLIPRECT_MASK; 104254885Sdumbbell } 105254885Sdumbbell 106254885Sdumbbell OUT_RING((box.x1 << R300_CLIPRECT_X_SHIFT) | 107254885Sdumbbell (box.y1 << R300_CLIPRECT_Y_SHIFT)); 108254885Sdumbbell OUT_RING((box.x2 << R300_CLIPRECT_X_SHIFT) | 109254885Sdumbbell (box.y2 << R300_CLIPRECT_Y_SHIFT)); 110254885Sdumbbell 111254885Sdumbbell } 112254885Sdumbbell 113254885Sdumbbell OUT_RING_REG(R300_RE_CLIPRECT_CNTL, r300_cliprect_cntl[nr - 1]); 114254885Sdumbbell 115254885Sdumbbell /* TODO/SECURITY: Force scissors to a safe value, otherwise the 116254885Sdumbbell * client might be able to trample over memory. 117254885Sdumbbell * The impact should be very limited, but I'd rather be safe than 118254885Sdumbbell * sorry. 119254885Sdumbbell */ 120254885Sdumbbell OUT_RING(CP_PACKET0(R300_RE_SCISSORS_TL, 1)); 121254885Sdumbbell OUT_RING(0); 122254885Sdumbbell OUT_RING(R300_SCISSORS_X_MASK | R300_SCISSORS_Y_MASK); 123254885Sdumbbell ADVANCE_RING(); 124254885Sdumbbell } else { 125254885Sdumbbell /* Why we allow zero cliprect rendering: 126254885Sdumbbell * There are some commands in a command buffer that must be submitted 127254885Sdumbbell * even when there are no cliprects, e.g. DMA buffer discard 128254885Sdumbbell * or state setting (though state setting could be avoided by 129254885Sdumbbell * simulating a loss of context). 130254885Sdumbbell * 131254885Sdumbbell * Now since the cmdbuf interface is so chaotic right now (and is 132254885Sdumbbell * bound to remain that way for a bit until things settle down), 133254885Sdumbbell * it is basically impossible to filter out the commands that are 134254885Sdumbbell * necessary and those that aren't. 135254885Sdumbbell * 136254885Sdumbbell * So I choose the safe way and don't do any filtering at all; 137254885Sdumbbell * instead, I simply set up the engine so that all rendering 138254885Sdumbbell * can't produce any fragments. 139254885Sdumbbell */ 140254885Sdumbbell BEGIN_RING(2); 141254885Sdumbbell OUT_RING_REG(R300_RE_CLIPRECT_CNTL, 0); 142254885Sdumbbell ADVANCE_RING(); 143254885Sdumbbell } 144254885Sdumbbell 145254885Sdumbbell /* flus cache and wait idle clean after cliprect change */ 146254885Sdumbbell BEGIN_RING(2); 147254885Sdumbbell OUT_RING(CP_PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0)); 148254885Sdumbbell OUT_RING(R300_RB3D_DC_FLUSH); 149254885Sdumbbell ADVANCE_RING(); 150254885Sdumbbell BEGIN_RING(2); 151254885Sdumbbell OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0)); 152254885Sdumbbell OUT_RING(RADEON_WAIT_3D_IDLECLEAN); 153254885Sdumbbell ADVANCE_RING(); 154254885Sdumbbell /* set flush flag */ 155254885Sdumbbell dev_priv->track_flush |= RADEON_FLUSH_EMITED; 156254885Sdumbbell 157254885Sdumbbell return 0; 158254885Sdumbbell} 159254885Sdumbbell 160254885Sdumbbellstatic u8 r300_reg_flags[0x10000 >> 2]; 161254885Sdumbbell 162254885Sdumbbellvoid r300_init_reg_flags(struct drm_device *dev) 163254885Sdumbbell{ 164254885Sdumbbell int i; 165254885Sdumbbell drm_radeon_private_t *dev_priv = dev->dev_private; 166254885Sdumbbell 167254885Sdumbbell memset(r300_reg_flags, 0, 0x10000 >> 2); 168254885Sdumbbell#define ADD_RANGE_MARK(reg, count,mark) \ 169254885Sdumbbell for(i=((reg)>>2);i<((reg)>>2)+(count);i++)\ 170254885Sdumbbell r300_reg_flags[i]|=(mark); 171254885Sdumbbell 172254885Sdumbbell#define MARK_SAFE 1 173254885Sdumbbell#define MARK_CHECK_OFFSET 2 174254885Sdumbbell 175254885Sdumbbell#define ADD_RANGE(reg, count) ADD_RANGE_MARK(reg, count, MARK_SAFE) 176254885Sdumbbell 177254885Sdumbbell /* these match cmducs() command in r300_driver/r300/r300_cmdbuf.c */ 178254885Sdumbbell ADD_RANGE(R300_SE_VPORT_XSCALE, 6); 179254885Sdumbbell ADD_RANGE(R300_VAP_CNTL, 1); 180254885Sdumbbell ADD_RANGE(R300_SE_VTE_CNTL, 2); 181254885Sdumbbell ADD_RANGE(0x2134, 2); 182254885Sdumbbell ADD_RANGE(R300_VAP_CNTL_STATUS, 1); 183254885Sdumbbell ADD_RANGE(R300_VAP_INPUT_CNTL_0, 2); 184254885Sdumbbell ADD_RANGE(0x21DC, 1); 185254885Sdumbbell ADD_RANGE(R300_VAP_UNKNOWN_221C, 1); 186254885Sdumbbell ADD_RANGE(R300_VAP_CLIP_X_0, 4); 187254885Sdumbbell ADD_RANGE(R300_VAP_PVS_STATE_FLUSH_REG, 1); 188254885Sdumbbell ADD_RANGE(R300_VAP_UNKNOWN_2288, 1); 189254885Sdumbbell ADD_RANGE(R300_VAP_OUTPUT_VTX_FMT_0, 2); 190254885Sdumbbell ADD_RANGE(R300_VAP_PVS_CNTL_1, 3); 191254885Sdumbbell ADD_RANGE(R300_GB_ENABLE, 1); 192254885Sdumbbell ADD_RANGE(R300_GB_MSPOS0, 5); 193254885Sdumbbell ADD_RANGE(R300_TX_INVALTAGS, 1); 194254885Sdumbbell ADD_RANGE(R300_TX_ENABLE, 1); 195254885Sdumbbell ADD_RANGE(0x4200, 4); 196254885Sdumbbell ADD_RANGE(0x4214, 1); 197254885Sdumbbell ADD_RANGE(R300_RE_POINTSIZE, 1); 198254885Sdumbbell ADD_RANGE(0x4230, 3); 199254885Sdumbbell ADD_RANGE(R300_RE_LINE_CNT, 1); 200254885Sdumbbell ADD_RANGE(R300_RE_UNK4238, 1); 201254885Sdumbbell ADD_RANGE(0x4260, 3); 202254885Sdumbbell ADD_RANGE(R300_RE_SHADE, 4); 203254885Sdumbbell ADD_RANGE(R300_RE_POLYGON_MODE, 5); 204254885Sdumbbell ADD_RANGE(R300_RE_ZBIAS_CNTL, 1); 205254885Sdumbbell ADD_RANGE(R300_RE_ZBIAS_T_FACTOR, 4); 206254885Sdumbbell ADD_RANGE(R300_RE_OCCLUSION_CNTL, 1); 207254885Sdumbbell ADD_RANGE(R300_RE_CULL_CNTL, 1); 208254885Sdumbbell ADD_RANGE(0x42C0, 2); 209254885Sdumbbell ADD_RANGE(R300_RS_CNTL_0, 2); 210254885Sdumbbell 211254885Sdumbbell ADD_RANGE(R300_SU_REG_DEST, 1); 212254885Sdumbbell if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV530) 213254885Sdumbbell ADD_RANGE(RV530_FG_ZBREG_DEST, 1); 214254885Sdumbbell 215254885Sdumbbell ADD_RANGE(R300_SC_HYPERZ, 2); 216254885Sdumbbell ADD_RANGE(0x43E8, 1); 217254885Sdumbbell 218254885Sdumbbell ADD_RANGE(0x46A4, 5); 219254885Sdumbbell 220254885Sdumbbell ADD_RANGE(R300_RE_FOG_STATE, 1); 221254885Sdumbbell ADD_RANGE(R300_FOG_COLOR_R, 3); 222254885Sdumbbell ADD_RANGE(R300_PP_ALPHA_TEST, 2); 223254885Sdumbbell ADD_RANGE(0x4BD8, 1); 224254885Sdumbbell ADD_RANGE(R300_PFS_PARAM_0_X, 64); 225254885Sdumbbell ADD_RANGE(0x4E00, 1); 226254885Sdumbbell ADD_RANGE(R300_RB3D_CBLEND, 2); 227254885Sdumbbell ADD_RANGE(R300_RB3D_COLORMASK, 1); 228254885Sdumbbell ADD_RANGE(R300_RB3D_BLEND_COLOR, 3); 229254885Sdumbbell ADD_RANGE_MARK(R300_RB3D_COLOROFFSET0, 1, MARK_CHECK_OFFSET); /* check offset */ 230254885Sdumbbell ADD_RANGE(R300_RB3D_COLORPITCH0, 1); 231254885Sdumbbell ADD_RANGE(0x4E50, 9); 232254885Sdumbbell ADD_RANGE(0x4E88, 1); 233254885Sdumbbell ADD_RANGE(0x4EA0, 2); 234254885Sdumbbell ADD_RANGE(R300_ZB_CNTL, 3); 235254885Sdumbbell ADD_RANGE(R300_ZB_FORMAT, 4); 236254885Sdumbbell ADD_RANGE_MARK(R300_ZB_DEPTHOFFSET, 1, MARK_CHECK_OFFSET); /* check offset */ 237254885Sdumbbell ADD_RANGE(R300_ZB_DEPTHPITCH, 1); 238254885Sdumbbell ADD_RANGE(R300_ZB_DEPTHCLEARVALUE, 1); 239254885Sdumbbell ADD_RANGE(R300_ZB_ZMASK_OFFSET, 13); 240254885Sdumbbell ADD_RANGE(R300_ZB_ZPASS_DATA, 2); /* ZB_ZPASS_DATA, ZB_ZPASS_ADDR */ 241254885Sdumbbell 242254885Sdumbbell ADD_RANGE(R300_TX_FILTER_0, 16); 243254885Sdumbbell ADD_RANGE(R300_TX_FILTER1_0, 16); 244254885Sdumbbell ADD_RANGE(R300_TX_SIZE_0, 16); 245254885Sdumbbell ADD_RANGE(R300_TX_FORMAT_0, 16); 246254885Sdumbbell ADD_RANGE(R300_TX_PITCH_0, 16); 247254885Sdumbbell /* Texture offset is dangerous and needs more checking */ 248254885Sdumbbell ADD_RANGE_MARK(R300_TX_OFFSET_0, 16, MARK_CHECK_OFFSET); 249254885Sdumbbell ADD_RANGE(R300_TX_CHROMA_KEY_0, 16); 250254885Sdumbbell ADD_RANGE(R300_TX_BORDER_COLOR_0, 16); 251254885Sdumbbell 252254885Sdumbbell /* Sporadic registers used as primitives are emitted */ 253254885Sdumbbell ADD_RANGE(R300_ZB_ZCACHE_CTLSTAT, 1); 254254885Sdumbbell ADD_RANGE(R300_RB3D_DSTCACHE_CTLSTAT, 1); 255254885Sdumbbell ADD_RANGE(R300_VAP_INPUT_ROUTE_0_0, 8); 256254885Sdumbbell ADD_RANGE(R300_VAP_INPUT_ROUTE_1_0, 8); 257254885Sdumbbell 258254885Sdumbbell if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV515) { 259254885Sdumbbell ADD_RANGE(R500_VAP_INDEX_OFFSET, 1); 260254885Sdumbbell ADD_RANGE(R500_US_CONFIG, 2); 261254885Sdumbbell ADD_RANGE(R500_US_CODE_ADDR, 3); 262254885Sdumbbell ADD_RANGE(R500_US_FC_CTRL, 1); 263254885Sdumbbell ADD_RANGE(R500_RS_IP_0, 16); 264254885Sdumbbell ADD_RANGE(R500_RS_INST_0, 16); 265254885Sdumbbell ADD_RANGE(R500_RB3D_COLOR_CLEAR_VALUE_AR, 2); 266254885Sdumbbell ADD_RANGE(R500_RB3D_CONSTANT_COLOR_AR, 2); 267254885Sdumbbell ADD_RANGE(R500_ZB_FIFO_SIZE, 2); 268254885Sdumbbell } else { 269254885Sdumbbell ADD_RANGE(R300_PFS_CNTL_0, 3); 270254885Sdumbbell ADD_RANGE(R300_PFS_NODE_0, 4); 271254885Sdumbbell ADD_RANGE(R300_PFS_TEXI_0, 64); 272254885Sdumbbell ADD_RANGE(R300_PFS_INSTR0_0, 64); 273254885Sdumbbell ADD_RANGE(R300_PFS_INSTR1_0, 64); 274254885Sdumbbell ADD_RANGE(R300_PFS_INSTR2_0, 64); 275254885Sdumbbell ADD_RANGE(R300_PFS_INSTR3_0, 64); 276254885Sdumbbell ADD_RANGE(R300_RS_INTERP_0, 8); 277254885Sdumbbell ADD_RANGE(R300_RS_ROUTE_0, 8); 278254885Sdumbbell 279254885Sdumbbell } 280254885Sdumbbell} 281254885Sdumbbell 282254885Sdumbbellstatic __inline__ int r300_check_range(unsigned reg, int count) 283254885Sdumbbell{ 284254885Sdumbbell int i; 285254885Sdumbbell if (reg & ~0xffff) 286254885Sdumbbell return -1; 287254885Sdumbbell for (i = (reg >> 2); i < (reg >> 2) + count; i++) 288254885Sdumbbell if (r300_reg_flags[i] != MARK_SAFE) 289254885Sdumbbell return 1; 290254885Sdumbbell return 0; 291254885Sdumbbell} 292254885Sdumbbell 293254885Sdumbbellstatic __inline__ int r300_emit_carefully_checked_packet0(drm_radeon_private_t * 294254885Sdumbbell dev_priv, 295254885Sdumbbell drm_radeon_kcmd_buffer_t 296254885Sdumbbell * cmdbuf, 297254885Sdumbbell drm_r300_cmd_header_t 298254885Sdumbbell header) 299254885Sdumbbell{ 300254885Sdumbbell int reg; 301254885Sdumbbell int sz; 302254885Sdumbbell int i; 303254885Sdumbbell u32 *value; 304254885Sdumbbell RING_LOCALS; 305254885Sdumbbell 306254885Sdumbbell sz = header.packet0.count; 307254885Sdumbbell reg = (header.packet0.reghi << 8) | header.packet0.reglo; 308254885Sdumbbell 309254885Sdumbbell if ((sz > 64) || (sz < 0)) { 310254885Sdumbbell DRM_ERROR("Cannot emit more than 64 values at a time (reg=%04x sz=%d)\n", 311254885Sdumbbell reg, sz); 312254885Sdumbbell return -EINVAL; 313254885Sdumbbell } 314254885Sdumbbell 315254885Sdumbbell for (i = 0; i < sz; i++) { 316254885Sdumbbell switch (r300_reg_flags[(reg >> 2) + i]) { 317254885Sdumbbell case MARK_SAFE: 318254885Sdumbbell break; 319254885Sdumbbell case MARK_CHECK_OFFSET: 320254885Sdumbbell value = drm_buffer_pointer_to_dword(cmdbuf->buffer, i); 321254885Sdumbbell if (!radeon_check_offset(dev_priv, *value)) { 322254885Sdumbbell DRM_ERROR("Offset failed range check (reg=%04x sz=%d)\n", 323254885Sdumbbell reg, sz); 324254885Sdumbbell return -EINVAL; 325254885Sdumbbell } 326254885Sdumbbell break; 327254885Sdumbbell default: 328254885Sdumbbell DRM_ERROR("Register %04x failed check as flag=%02x\n", 329254885Sdumbbell reg + i * 4, r300_reg_flags[(reg >> 2) + i]); 330254885Sdumbbell return -EINVAL; 331254885Sdumbbell } 332254885Sdumbbell } 333254885Sdumbbell 334254885Sdumbbell BEGIN_RING(1 + sz); 335254885Sdumbbell OUT_RING(CP_PACKET0(reg, sz - 1)); 336254885Sdumbbell OUT_RING_DRM_BUFFER(cmdbuf->buffer, sz); 337254885Sdumbbell ADVANCE_RING(); 338254885Sdumbbell 339254885Sdumbbell return 0; 340254885Sdumbbell} 341254885Sdumbbell 342254885Sdumbbell/** 343254885Sdumbbell * Emits a packet0 setting arbitrary registers. 344254885Sdumbbell * Called by r300_do_cp_cmdbuf. 345254885Sdumbbell * 346254885Sdumbbell * Note that checks are performed on contents and addresses of the registers 347254885Sdumbbell */ 348254885Sdumbbellstatic __inline__ int r300_emit_packet0(drm_radeon_private_t *dev_priv, 349254885Sdumbbell drm_radeon_kcmd_buffer_t *cmdbuf, 350254885Sdumbbell drm_r300_cmd_header_t header) 351254885Sdumbbell{ 352254885Sdumbbell int reg; 353254885Sdumbbell int sz; 354254885Sdumbbell RING_LOCALS; 355254885Sdumbbell 356254885Sdumbbell sz = header.packet0.count; 357254885Sdumbbell reg = (header.packet0.reghi << 8) | header.packet0.reglo; 358254885Sdumbbell 359254885Sdumbbell if (!sz) 360254885Sdumbbell return 0; 361254885Sdumbbell 362254885Sdumbbell if (sz * 4 > drm_buffer_unprocessed(cmdbuf->buffer)) 363254885Sdumbbell return -EINVAL; 364254885Sdumbbell 365254885Sdumbbell if (reg + sz * 4 >= 0x10000) { 366254885Sdumbbell DRM_ERROR("No such registers in hardware reg=%04x sz=%d\n", reg, 367254885Sdumbbell sz); 368254885Sdumbbell return -EINVAL; 369254885Sdumbbell } 370254885Sdumbbell 371254885Sdumbbell if (r300_check_range(reg, sz)) { 372254885Sdumbbell /* go and check everything */ 373254885Sdumbbell return r300_emit_carefully_checked_packet0(dev_priv, cmdbuf, 374254885Sdumbbell header); 375254885Sdumbbell } 376254885Sdumbbell /* the rest of the data is safe to emit, whatever the values the user passed */ 377254885Sdumbbell 378254885Sdumbbell BEGIN_RING(1 + sz); 379254885Sdumbbell OUT_RING(CP_PACKET0(reg, sz - 1)); 380254885Sdumbbell OUT_RING_DRM_BUFFER(cmdbuf->buffer, sz); 381254885Sdumbbell ADVANCE_RING(); 382254885Sdumbbell 383254885Sdumbbell return 0; 384254885Sdumbbell} 385254885Sdumbbell 386254885Sdumbbell/** 387254885Sdumbbell * Uploads user-supplied vertex program instructions or parameters onto 388254885Sdumbbell * the graphics card. 389254885Sdumbbell * Called by r300_do_cp_cmdbuf. 390254885Sdumbbell */ 391254885Sdumbbellstatic __inline__ int r300_emit_vpu(drm_radeon_private_t *dev_priv, 392254885Sdumbbell drm_radeon_kcmd_buffer_t *cmdbuf, 393254885Sdumbbell drm_r300_cmd_header_t header) 394254885Sdumbbell{ 395254885Sdumbbell int sz; 396254885Sdumbbell int addr; 397254885Sdumbbell RING_LOCALS; 398254885Sdumbbell 399254885Sdumbbell sz = header.vpu.count; 400254885Sdumbbell addr = (header.vpu.adrhi << 8) | header.vpu.adrlo; 401254885Sdumbbell 402254885Sdumbbell if (!sz) 403254885Sdumbbell return 0; 404254885Sdumbbell if (sz * 16 > drm_buffer_unprocessed(cmdbuf->buffer)) 405254885Sdumbbell return -EINVAL; 406254885Sdumbbell 407254885Sdumbbell /* VAP is very sensitive so we purge cache before we program it 408254885Sdumbbell * and we also flush its state before & after */ 409254885Sdumbbell BEGIN_RING(6); 410254885Sdumbbell OUT_RING(CP_PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0)); 411254885Sdumbbell OUT_RING(R300_RB3D_DC_FLUSH); 412254885Sdumbbell OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0)); 413254885Sdumbbell OUT_RING(RADEON_WAIT_3D_IDLECLEAN); 414254885Sdumbbell OUT_RING(CP_PACKET0(R300_VAP_PVS_STATE_FLUSH_REG, 0)); 415254885Sdumbbell OUT_RING(0); 416254885Sdumbbell ADVANCE_RING(); 417254885Sdumbbell /* set flush flag */ 418254885Sdumbbell dev_priv->track_flush |= RADEON_FLUSH_EMITED; 419254885Sdumbbell 420254885Sdumbbell BEGIN_RING(3 + sz * 4); 421254885Sdumbbell OUT_RING_REG(R300_VAP_PVS_UPLOAD_ADDRESS, addr); 422254885Sdumbbell OUT_RING(CP_PACKET0_TABLE(R300_VAP_PVS_UPLOAD_DATA, sz * 4 - 1)); 423254885Sdumbbell OUT_RING_DRM_BUFFER(cmdbuf->buffer, sz * 4); 424254885Sdumbbell ADVANCE_RING(); 425254885Sdumbbell 426254885Sdumbbell BEGIN_RING(2); 427254885Sdumbbell OUT_RING(CP_PACKET0(R300_VAP_PVS_STATE_FLUSH_REG, 0)); 428254885Sdumbbell OUT_RING(0); 429254885Sdumbbell ADVANCE_RING(); 430254885Sdumbbell 431254885Sdumbbell return 0; 432254885Sdumbbell} 433254885Sdumbbell 434254885Sdumbbell/** 435254885Sdumbbell * Emit a clear packet from userspace. 436254885Sdumbbell * Called by r300_emit_packet3. 437254885Sdumbbell */ 438254885Sdumbbellstatic __inline__ int r300_emit_clear(drm_radeon_private_t *dev_priv, 439254885Sdumbbell drm_radeon_kcmd_buffer_t *cmdbuf) 440254885Sdumbbell{ 441254885Sdumbbell RING_LOCALS; 442254885Sdumbbell 443254885Sdumbbell if (8 * 4 > drm_buffer_unprocessed(cmdbuf->buffer)) 444254885Sdumbbell return -EINVAL; 445254885Sdumbbell 446254885Sdumbbell BEGIN_RING(10); 447254885Sdumbbell OUT_RING(CP_PACKET3(R200_3D_DRAW_IMMD_2, 8)); 448254885Sdumbbell OUT_RING(R300_PRIM_TYPE_POINT | R300_PRIM_WALK_RING | 449254885Sdumbbell (1 << R300_PRIM_NUM_VERTICES_SHIFT)); 450254885Sdumbbell OUT_RING_DRM_BUFFER(cmdbuf->buffer, 8); 451254885Sdumbbell ADVANCE_RING(); 452254885Sdumbbell 453254885Sdumbbell BEGIN_RING(4); 454254885Sdumbbell OUT_RING(CP_PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0)); 455254885Sdumbbell OUT_RING(R300_RB3D_DC_FLUSH); 456254885Sdumbbell OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0)); 457254885Sdumbbell OUT_RING(RADEON_WAIT_3D_IDLECLEAN); 458254885Sdumbbell ADVANCE_RING(); 459254885Sdumbbell /* set flush flag */ 460254885Sdumbbell dev_priv->track_flush |= RADEON_FLUSH_EMITED; 461254885Sdumbbell 462254885Sdumbbell return 0; 463254885Sdumbbell} 464254885Sdumbbell 465254885Sdumbbellstatic __inline__ int r300_emit_3d_load_vbpntr(drm_radeon_private_t *dev_priv, 466254885Sdumbbell drm_radeon_kcmd_buffer_t *cmdbuf, 467254885Sdumbbell u32 header) 468254885Sdumbbell{ 469254885Sdumbbell int count, i, k; 470254885Sdumbbell#define MAX_ARRAY_PACKET 64 471254885Sdumbbell u32 *data; 472254885Sdumbbell u32 narrays; 473254885Sdumbbell RING_LOCALS; 474254885Sdumbbell 475254885Sdumbbell count = (header & RADEON_CP_PACKET_COUNT_MASK) >> 16; 476254885Sdumbbell 477254885Sdumbbell if ((count + 1) > MAX_ARRAY_PACKET) { 478254885Sdumbbell DRM_ERROR("Too large payload in 3D_LOAD_VBPNTR (count=%d)\n", 479254885Sdumbbell count); 480254885Sdumbbell return -EINVAL; 481254885Sdumbbell } 482254885Sdumbbell /* carefully check packet contents */ 483254885Sdumbbell 484254885Sdumbbell /* We have already read the header so advance the buffer. */ 485254885Sdumbbell drm_buffer_advance(cmdbuf->buffer, 4); 486254885Sdumbbell 487254885Sdumbbell narrays = *(u32 *)drm_buffer_pointer_to_dword(cmdbuf->buffer, 0); 488254885Sdumbbell k = 0; 489254885Sdumbbell i = 1; 490254885Sdumbbell while ((k < narrays) && (i < (count + 1))) { 491254885Sdumbbell i++; /* skip attribute field */ 492254885Sdumbbell data = drm_buffer_pointer_to_dword(cmdbuf->buffer, i); 493254885Sdumbbell if (!radeon_check_offset(dev_priv, *data)) { 494254885Sdumbbell DRM_ERROR 495254885Sdumbbell ("Offset failed range check (k=%d i=%d) while processing 3D_LOAD_VBPNTR packet.\n", 496254885Sdumbbell k, i); 497254885Sdumbbell return -EINVAL; 498254885Sdumbbell } 499254885Sdumbbell k++; 500254885Sdumbbell i++; 501254885Sdumbbell if (k == narrays) 502254885Sdumbbell break; 503254885Sdumbbell /* have one more to process, they come in pairs */ 504254885Sdumbbell data = drm_buffer_pointer_to_dword(cmdbuf->buffer, i); 505254885Sdumbbell if (!radeon_check_offset(dev_priv, *data)) { 506254885Sdumbbell DRM_ERROR 507254885Sdumbbell ("Offset failed range check (k=%d i=%d) while processing 3D_LOAD_VBPNTR packet.\n", 508254885Sdumbbell k, i); 509254885Sdumbbell return -EINVAL; 510254885Sdumbbell } 511254885Sdumbbell k++; 512254885Sdumbbell i++; 513254885Sdumbbell } 514254885Sdumbbell /* do the counts match what we expect ? */ 515254885Sdumbbell if ((k != narrays) || (i != (count + 1))) { 516254885Sdumbbell DRM_ERROR 517254885Sdumbbell ("Malformed 3D_LOAD_VBPNTR packet (k=%d i=%d narrays=%d count+1=%d).\n", 518254885Sdumbbell k, i, narrays, count + 1); 519254885Sdumbbell return -EINVAL; 520254885Sdumbbell } 521254885Sdumbbell 522254885Sdumbbell /* all clear, output packet */ 523254885Sdumbbell 524254885Sdumbbell BEGIN_RING(count + 2); 525254885Sdumbbell OUT_RING(header); 526254885Sdumbbell OUT_RING_DRM_BUFFER(cmdbuf->buffer, count + 1); 527254885Sdumbbell ADVANCE_RING(); 528254885Sdumbbell 529254885Sdumbbell return 0; 530254885Sdumbbell} 531254885Sdumbbell 532254885Sdumbbellstatic __inline__ int r300_emit_bitblt_multi(drm_radeon_private_t *dev_priv, 533254885Sdumbbell drm_radeon_kcmd_buffer_t *cmdbuf) 534254885Sdumbbell{ 535254885Sdumbbell u32 *cmd = drm_buffer_pointer_to_dword(cmdbuf->buffer, 0); 536254885Sdumbbell int count, ret; 537254885Sdumbbell RING_LOCALS; 538254885Sdumbbell 539254885Sdumbbell 540254885Sdumbbell count = (*cmd & RADEON_CP_PACKET_COUNT_MASK) >> 16; 541254885Sdumbbell 542254885Sdumbbell if (*cmd & 0x8000) { 543254885Sdumbbell u32 offset; 544254885Sdumbbell u32 *cmd1 = drm_buffer_pointer_to_dword(cmdbuf->buffer, 1); 545254885Sdumbbell if (*cmd1 & (RADEON_GMC_SRC_PITCH_OFFSET_CNTL 546254885Sdumbbell | RADEON_GMC_DST_PITCH_OFFSET_CNTL)) { 547254885Sdumbbell 548254885Sdumbbell u32 *cmd2 = drm_buffer_pointer_to_dword(cmdbuf->buffer, 2); 549254885Sdumbbell offset = *cmd2 << 10; 550254885Sdumbbell ret = !radeon_check_offset(dev_priv, offset); 551254885Sdumbbell if (ret) { 552254885Sdumbbell DRM_ERROR("Invalid bitblt first offset is %08X\n", offset); 553254885Sdumbbell return -EINVAL; 554254885Sdumbbell } 555254885Sdumbbell } 556254885Sdumbbell 557254885Sdumbbell if ((*cmd1 & RADEON_GMC_SRC_PITCH_OFFSET_CNTL) && 558254885Sdumbbell (*cmd1 & RADEON_GMC_DST_PITCH_OFFSET_CNTL)) { 559254885Sdumbbell u32 *cmd3 = drm_buffer_pointer_to_dword(cmdbuf->buffer, 3); 560254885Sdumbbell offset = *cmd3 << 10; 561254885Sdumbbell ret = !radeon_check_offset(dev_priv, offset); 562254885Sdumbbell if (ret) { 563254885Sdumbbell DRM_ERROR("Invalid bitblt second offset is %08X\n", offset); 564254885Sdumbbell return -EINVAL; 565254885Sdumbbell } 566254885Sdumbbell 567254885Sdumbbell } 568254885Sdumbbell } 569254885Sdumbbell 570254885Sdumbbell BEGIN_RING(count+2); 571254885Sdumbbell OUT_RING_DRM_BUFFER(cmdbuf->buffer, count + 2); 572254885Sdumbbell ADVANCE_RING(); 573254885Sdumbbell 574254885Sdumbbell return 0; 575254885Sdumbbell} 576254885Sdumbbell 577254885Sdumbbellstatic __inline__ int r300_emit_draw_indx_2(drm_radeon_private_t *dev_priv, 578254885Sdumbbell drm_radeon_kcmd_buffer_t *cmdbuf) 579254885Sdumbbell{ 580254885Sdumbbell u32 *cmd = drm_buffer_pointer_to_dword(cmdbuf->buffer, 0); 581254885Sdumbbell u32 *cmd1 = drm_buffer_pointer_to_dword(cmdbuf->buffer, 1); 582254885Sdumbbell int count; 583254885Sdumbbell int expected_count; 584254885Sdumbbell RING_LOCALS; 585254885Sdumbbell 586254885Sdumbbell count = (*cmd & RADEON_CP_PACKET_COUNT_MASK) >> 16; 587254885Sdumbbell 588254885Sdumbbell expected_count = *cmd1 >> 16; 589254885Sdumbbell if (!(*cmd1 & R300_VAP_VF_CNTL__INDEX_SIZE_32bit)) 590254885Sdumbbell expected_count = (expected_count+1)/2; 591254885Sdumbbell 592254885Sdumbbell if (count && count != expected_count) { 593254885Sdumbbell DRM_ERROR("3D_DRAW_INDX_2: packet size %i, expected %i\n", 594254885Sdumbbell count, expected_count); 595254885Sdumbbell return -EINVAL; 596254885Sdumbbell } 597254885Sdumbbell 598254885Sdumbbell BEGIN_RING(count+2); 599254885Sdumbbell OUT_RING_DRM_BUFFER(cmdbuf->buffer, count + 2); 600254885Sdumbbell ADVANCE_RING(); 601254885Sdumbbell 602254885Sdumbbell if (!count) { 603254885Sdumbbell drm_r300_cmd_header_t stack_header, *header; 604254885Sdumbbell u32 *cmd1, *cmd2, *cmd3; 605254885Sdumbbell 606254885Sdumbbell if (drm_buffer_unprocessed(cmdbuf->buffer) 607254885Sdumbbell < 4*4 + sizeof(stack_header)) { 608254885Sdumbbell DRM_ERROR("3D_DRAW_INDX_2: expect subsequent INDX_BUFFER, but stream is too short.\n"); 609254885Sdumbbell return -EINVAL; 610254885Sdumbbell } 611254885Sdumbbell 612254885Sdumbbell header = drm_buffer_read_object(cmdbuf->buffer, 613254885Sdumbbell sizeof(stack_header), &stack_header); 614254885Sdumbbell 615254885Sdumbbell cmd = drm_buffer_pointer_to_dword(cmdbuf->buffer, 0); 616254885Sdumbbell cmd1 = drm_buffer_pointer_to_dword(cmdbuf->buffer, 1); 617254885Sdumbbell cmd2 = drm_buffer_pointer_to_dword(cmdbuf->buffer, 2); 618254885Sdumbbell cmd3 = drm_buffer_pointer_to_dword(cmdbuf->buffer, 3); 619254885Sdumbbell 620254885Sdumbbell if (header->header.cmd_type != R300_CMD_PACKET3 || 621254885Sdumbbell header->packet3.packet != R300_CMD_PACKET3_RAW || 622254885Sdumbbell *cmd != CP_PACKET3(RADEON_CP_INDX_BUFFER, 2)) { 623254885Sdumbbell DRM_ERROR("3D_DRAW_INDX_2: expect subsequent INDX_BUFFER.\n"); 624254885Sdumbbell return -EINVAL; 625254885Sdumbbell } 626254885Sdumbbell 627254885Sdumbbell if ((*cmd1 & 0x8000ffff) != 0x80000810) { 628254885Sdumbbell DRM_ERROR("Invalid indx_buffer reg address %08X\n", 629254885Sdumbbell *cmd1); 630254885Sdumbbell return -EINVAL; 631254885Sdumbbell } 632254885Sdumbbell if (!radeon_check_offset(dev_priv, *cmd2)) { 633254885Sdumbbell DRM_ERROR("Invalid indx_buffer offset is %08X\n", 634254885Sdumbbell *cmd2); 635254885Sdumbbell return -EINVAL; 636254885Sdumbbell } 637254885Sdumbbell if (*cmd3 != expected_count) { 638254885Sdumbbell DRM_ERROR("INDX_BUFFER: buffer size %i, expected %i\n", 639254885Sdumbbell *cmd3, expected_count); 640254885Sdumbbell return -EINVAL; 641254885Sdumbbell } 642254885Sdumbbell 643254885Sdumbbell BEGIN_RING(4); 644254885Sdumbbell OUT_RING_DRM_BUFFER(cmdbuf->buffer, 4); 645254885Sdumbbell ADVANCE_RING(); 646254885Sdumbbell } 647254885Sdumbbell 648254885Sdumbbell return 0; 649254885Sdumbbell} 650254885Sdumbbell 651254885Sdumbbellstatic __inline__ int r300_emit_raw_packet3(drm_radeon_private_t *dev_priv, 652254885Sdumbbell drm_radeon_kcmd_buffer_t *cmdbuf) 653254885Sdumbbell{ 654254885Sdumbbell u32 *header; 655254885Sdumbbell int count; 656254885Sdumbbell RING_LOCALS; 657254885Sdumbbell 658254885Sdumbbell if (4 > drm_buffer_unprocessed(cmdbuf->buffer)) 659254885Sdumbbell return -EINVAL; 660254885Sdumbbell 661254885Sdumbbell /* Fixme !! This simply emits a packet without much checking. 662254885Sdumbbell We need to be smarter. */ 663254885Sdumbbell 664254885Sdumbbell /* obtain first word - actual packet3 header */ 665254885Sdumbbell header = drm_buffer_pointer_to_dword(cmdbuf->buffer, 0); 666254885Sdumbbell 667254885Sdumbbell /* Is it packet 3 ? */ 668254885Sdumbbell if ((*header >> 30) != 0x3) { 669254885Sdumbbell DRM_ERROR("Not a packet3 header (0x%08x)\n", *header); 670254885Sdumbbell return -EINVAL; 671254885Sdumbbell } 672254885Sdumbbell 673254885Sdumbbell count = (*header >> 16) & 0x3fff; 674254885Sdumbbell 675254885Sdumbbell /* Check again now that we know how much data to expect */ 676254885Sdumbbell if ((count + 2) * 4 > drm_buffer_unprocessed(cmdbuf->buffer)) { 677254885Sdumbbell DRM_ERROR 678254885Sdumbbell ("Expected packet3 of length %d but have only %d bytes left\n", 679254885Sdumbbell (count + 2) * 4, drm_buffer_unprocessed(cmdbuf->buffer)); 680254885Sdumbbell return -EINVAL; 681254885Sdumbbell } 682254885Sdumbbell 683254885Sdumbbell /* Is it a packet type we know about ? */ 684254885Sdumbbell switch (*header & 0xff00) { 685254885Sdumbbell case RADEON_3D_LOAD_VBPNTR: /* load vertex array pointers */ 686254885Sdumbbell return r300_emit_3d_load_vbpntr(dev_priv, cmdbuf, *header); 687254885Sdumbbell 688254885Sdumbbell case RADEON_CNTL_BITBLT_MULTI: 689254885Sdumbbell return r300_emit_bitblt_multi(dev_priv, cmdbuf); 690254885Sdumbbell 691254885Sdumbbell case RADEON_CP_INDX_BUFFER: 692254885Sdumbbell DRM_ERROR("packet3 INDX_BUFFER without preceding 3D_DRAW_INDX_2 is illegal.\n"); 693254885Sdumbbell return -EINVAL; 694254885Sdumbbell case RADEON_CP_3D_DRAW_IMMD_2: 695254885Sdumbbell /* triggers drawing using in-packet vertex data */ 696254885Sdumbbell case RADEON_CP_3D_DRAW_VBUF_2: 697254885Sdumbbell /* triggers drawing of vertex buffers setup elsewhere */ 698254885Sdumbbell dev_priv->track_flush &= ~(RADEON_FLUSH_EMITED | 699254885Sdumbbell RADEON_PURGE_EMITED); 700254885Sdumbbell break; 701254885Sdumbbell case RADEON_CP_3D_DRAW_INDX_2: 702254885Sdumbbell /* triggers drawing using indices to vertex buffer */ 703254885Sdumbbell /* whenever we send vertex we clear flush & purge */ 704254885Sdumbbell dev_priv->track_flush &= ~(RADEON_FLUSH_EMITED | 705254885Sdumbbell RADEON_PURGE_EMITED); 706254885Sdumbbell return r300_emit_draw_indx_2(dev_priv, cmdbuf); 707254885Sdumbbell case RADEON_WAIT_FOR_IDLE: 708254885Sdumbbell case RADEON_CP_NOP: 709254885Sdumbbell /* these packets are safe */ 710254885Sdumbbell break; 711254885Sdumbbell default: 712254885Sdumbbell DRM_ERROR("Unknown packet3 header (0x%08x)\n", *header); 713254885Sdumbbell return -EINVAL; 714254885Sdumbbell } 715254885Sdumbbell 716254885Sdumbbell BEGIN_RING(count + 2); 717254885Sdumbbell OUT_RING_DRM_BUFFER(cmdbuf->buffer, count + 2); 718254885Sdumbbell ADVANCE_RING(); 719254885Sdumbbell 720254885Sdumbbell return 0; 721254885Sdumbbell} 722254885Sdumbbell 723254885Sdumbbell/** 724254885Sdumbbell * Emit a rendering packet3 from userspace. 725254885Sdumbbell * Called by r300_do_cp_cmdbuf. 726254885Sdumbbell */ 727254885Sdumbbellstatic __inline__ int r300_emit_packet3(drm_radeon_private_t *dev_priv, 728254885Sdumbbell drm_radeon_kcmd_buffer_t *cmdbuf, 729254885Sdumbbell drm_r300_cmd_header_t header) 730254885Sdumbbell{ 731254885Sdumbbell int n; 732254885Sdumbbell int ret; 733254885Sdumbbell int orig_iter = cmdbuf->buffer->iterator; 734254885Sdumbbell 735254885Sdumbbell /* This is a do-while-loop so that we run the interior at least once, 736254885Sdumbbell * even if cmdbuf->nbox is 0. Compare r300_emit_cliprects for rationale. 737254885Sdumbbell */ 738254885Sdumbbell n = 0; 739254885Sdumbbell do { 740254885Sdumbbell if (cmdbuf->nbox > R300_SIMULTANEOUS_CLIPRECTS) { 741254885Sdumbbell ret = r300_emit_cliprects(dev_priv, cmdbuf, n); 742254885Sdumbbell if (ret) 743254885Sdumbbell return ret; 744254885Sdumbbell 745254885Sdumbbell cmdbuf->buffer->iterator = orig_iter; 746254885Sdumbbell } 747254885Sdumbbell 748254885Sdumbbell switch (header.packet3.packet) { 749254885Sdumbbell case R300_CMD_PACKET3_CLEAR: 750254885Sdumbbell DRM_DEBUG("R300_CMD_PACKET3_CLEAR\n"); 751254885Sdumbbell ret = r300_emit_clear(dev_priv, cmdbuf); 752254885Sdumbbell if (ret) { 753254885Sdumbbell DRM_ERROR("r300_emit_clear failed\n"); 754254885Sdumbbell return ret; 755254885Sdumbbell } 756254885Sdumbbell break; 757254885Sdumbbell 758254885Sdumbbell case R300_CMD_PACKET3_RAW: 759254885Sdumbbell DRM_DEBUG("R300_CMD_PACKET3_RAW\n"); 760254885Sdumbbell ret = r300_emit_raw_packet3(dev_priv, cmdbuf); 761254885Sdumbbell if (ret) { 762254885Sdumbbell DRM_ERROR("r300_emit_raw_packet3 failed\n"); 763254885Sdumbbell return ret; 764254885Sdumbbell } 765254885Sdumbbell break; 766254885Sdumbbell 767254885Sdumbbell default: 768254885Sdumbbell DRM_ERROR("bad packet3 type %i at byte %d\n", 769254885Sdumbbell header.packet3.packet, 770254885Sdumbbell cmdbuf->buffer->iterator - (int)sizeof(header)); 771254885Sdumbbell return -EINVAL; 772254885Sdumbbell } 773254885Sdumbbell 774254885Sdumbbell n += R300_SIMULTANEOUS_CLIPRECTS; 775254885Sdumbbell } while (n < cmdbuf->nbox); 776254885Sdumbbell 777254885Sdumbbell return 0; 778254885Sdumbbell} 779254885Sdumbbell 780254885Sdumbbell/* Some of the R300 chips seem to be extremely touchy about the two registers 781254885Sdumbbell * that are configured in r300_pacify. 782254885Sdumbbell * Among the worst offenders seems to be the R300 ND (0x4E44): When userspace 783254885Sdumbbell * sends a command buffer that contains only state setting commands and a 784254885Sdumbbell * vertex program/parameter upload sequence, this will eventually lead to a 785254885Sdumbbell * lockup, unless the sequence is bracketed by calls to r300_pacify. 786254885Sdumbbell * So we should take great care to *always* call r300_pacify before 787254885Sdumbbell * *anything* 3D related, and again afterwards. This is what the 788254885Sdumbbell * call bracket in r300_do_cp_cmdbuf is for. 789254885Sdumbbell */ 790254885Sdumbbell 791254885Sdumbbell/** 792254885Sdumbbell * Emit the sequence to pacify R300. 793254885Sdumbbell */ 794254885Sdumbbellstatic void r300_pacify(drm_radeon_private_t *dev_priv) 795254885Sdumbbell{ 796254885Sdumbbell uint32_t cache_z, cache_3d, cache_2d; 797254885Sdumbbell RING_LOCALS; 798254885Sdumbbell 799254885Sdumbbell cache_z = R300_ZC_FLUSH; 800254885Sdumbbell cache_2d = R300_RB2D_DC_FLUSH; 801254885Sdumbbell cache_3d = R300_RB3D_DC_FLUSH; 802254885Sdumbbell if (!(dev_priv->track_flush & RADEON_PURGE_EMITED)) { 803254885Sdumbbell /* we can purge, primitive where draw since last purge */ 804254885Sdumbbell cache_z |= R300_ZC_FREE; 805254885Sdumbbell cache_2d |= R300_RB2D_DC_FREE; 806254885Sdumbbell cache_3d |= R300_RB3D_DC_FREE; 807254885Sdumbbell } 808254885Sdumbbell 809254885Sdumbbell /* flush & purge zbuffer */ 810254885Sdumbbell BEGIN_RING(2); 811254885Sdumbbell OUT_RING(CP_PACKET0(R300_ZB_ZCACHE_CTLSTAT, 0)); 812254885Sdumbbell OUT_RING(cache_z); 813254885Sdumbbell ADVANCE_RING(); 814254885Sdumbbell /* flush & purge 3d */ 815254885Sdumbbell BEGIN_RING(2); 816254885Sdumbbell OUT_RING(CP_PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0)); 817254885Sdumbbell OUT_RING(cache_3d); 818254885Sdumbbell ADVANCE_RING(); 819254885Sdumbbell /* flush & purge texture */ 820254885Sdumbbell BEGIN_RING(2); 821254885Sdumbbell OUT_RING(CP_PACKET0(R300_TX_INVALTAGS, 0)); 822254885Sdumbbell OUT_RING(0); 823254885Sdumbbell ADVANCE_RING(); 824254885Sdumbbell /* FIXME: is this one really needed ? */ 825254885Sdumbbell BEGIN_RING(2); 826254885Sdumbbell OUT_RING(CP_PACKET0(R300_RB3D_AARESOLVE_CTL, 0)); 827254885Sdumbbell OUT_RING(0); 828254885Sdumbbell ADVANCE_RING(); 829254885Sdumbbell BEGIN_RING(2); 830254885Sdumbbell OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0)); 831254885Sdumbbell OUT_RING(RADEON_WAIT_3D_IDLECLEAN); 832254885Sdumbbell ADVANCE_RING(); 833254885Sdumbbell /* flush & purge 2d through E2 as RB2D will trigger lockup */ 834254885Sdumbbell BEGIN_RING(4); 835254885Sdumbbell OUT_RING(CP_PACKET0(R300_DSTCACHE_CTLSTAT, 0)); 836254885Sdumbbell OUT_RING(cache_2d); 837254885Sdumbbell OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0)); 838254885Sdumbbell OUT_RING(RADEON_WAIT_2D_IDLECLEAN | 839254885Sdumbbell RADEON_WAIT_HOST_IDLECLEAN); 840254885Sdumbbell ADVANCE_RING(); 841254885Sdumbbell /* set flush & purge flags */ 842254885Sdumbbell dev_priv->track_flush |= RADEON_FLUSH_EMITED | RADEON_PURGE_EMITED; 843254885Sdumbbell} 844254885Sdumbbell 845254885Sdumbbell/** 846254885Sdumbbell * Called by r300_do_cp_cmdbuf to update the internal buffer age and state. 847254885Sdumbbell * The actual age emit is done by r300_do_cp_cmdbuf, which is why you must 848254885Sdumbbell * be careful about how this function is called. 849254885Sdumbbell */ 850254885Sdumbbellstatic void r300_discard_buffer(struct drm_device *dev, struct drm_master *master, struct drm_buf *buf) 851254885Sdumbbell{ 852254885Sdumbbell drm_radeon_buf_priv_t *buf_priv = buf->dev_private; 853254885Sdumbbell struct drm_radeon_master_private *master_priv = master->driver_priv; 854254885Sdumbbell 855254885Sdumbbell buf_priv->age = ++master_priv->sarea_priv->last_dispatch; 856254885Sdumbbell buf->pending = 1; 857254885Sdumbbell buf->used = 0; 858254885Sdumbbell} 859254885Sdumbbell 860254885Sdumbbellstatic void r300_cmd_wait(drm_radeon_private_t * dev_priv, 861254885Sdumbbell drm_r300_cmd_header_t header) 862254885Sdumbbell{ 863254885Sdumbbell u32 wait_until; 864254885Sdumbbell RING_LOCALS; 865254885Sdumbbell 866254885Sdumbbell if (!header.wait.flags) 867254885Sdumbbell return; 868254885Sdumbbell 869254885Sdumbbell wait_until = 0; 870254885Sdumbbell 871254885Sdumbbell switch(header.wait.flags) { 872254885Sdumbbell case R300_WAIT_2D: 873254885Sdumbbell wait_until = RADEON_WAIT_2D_IDLE; 874254885Sdumbbell break; 875254885Sdumbbell case R300_WAIT_3D: 876254885Sdumbbell wait_until = RADEON_WAIT_3D_IDLE; 877254885Sdumbbell break; 878254885Sdumbbell case R300_NEW_WAIT_2D_3D: 879254885Sdumbbell wait_until = RADEON_WAIT_2D_IDLE|RADEON_WAIT_3D_IDLE; 880254885Sdumbbell break; 881254885Sdumbbell case R300_NEW_WAIT_2D_2D_CLEAN: 882254885Sdumbbell wait_until = RADEON_WAIT_2D_IDLE|RADEON_WAIT_2D_IDLECLEAN; 883254885Sdumbbell break; 884254885Sdumbbell case R300_NEW_WAIT_3D_3D_CLEAN: 885254885Sdumbbell wait_until = RADEON_WAIT_3D_IDLE|RADEON_WAIT_3D_IDLECLEAN; 886254885Sdumbbell break; 887254885Sdumbbell case R300_NEW_WAIT_2D_2D_CLEAN_3D_3D_CLEAN: 888254885Sdumbbell wait_until = RADEON_WAIT_2D_IDLE|RADEON_WAIT_2D_IDLECLEAN; 889254885Sdumbbell wait_until |= RADEON_WAIT_3D_IDLE|RADEON_WAIT_3D_IDLECLEAN; 890254885Sdumbbell break; 891254885Sdumbbell default: 892254885Sdumbbell return; 893254885Sdumbbell } 894254885Sdumbbell 895254885Sdumbbell BEGIN_RING(2); 896254885Sdumbbell OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0)); 897254885Sdumbbell OUT_RING(wait_until); 898254885Sdumbbell ADVANCE_RING(); 899254885Sdumbbell} 900254885Sdumbbell 901254885Sdumbbellstatic int r300_scratch(drm_radeon_private_t *dev_priv, 902254885Sdumbbell drm_radeon_kcmd_buffer_t *cmdbuf, 903254885Sdumbbell drm_r300_cmd_header_t header) 904254885Sdumbbell{ 905254885Sdumbbell u32 *ref_age_base; 906254885Sdumbbell u32 i, *buf_idx, h_pending; 907254885Sdumbbell u64 *ptr_addr; 908254885Sdumbbell u64 stack_ptr_addr; 909254885Sdumbbell RING_LOCALS; 910254885Sdumbbell 911254885Sdumbbell if (drm_buffer_unprocessed(cmdbuf->buffer) < 912254885Sdumbbell (sizeof(u64) + header.scratch.n_bufs * sizeof(*buf_idx))) { 913254885Sdumbbell return -EINVAL; 914254885Sdumbbell } 915254885Sdumbbell 916254885Sdumbbell if (header.scratch.reg >= 5) { 917254885Sdumbbell return -EINVAL; 918254885Sdumbbell } 919254885Sdumbbell 920254885Sdumbbell dev_priv->scratch_ages[header.scratch.reg]++; 921254885Sdumbbell 922254885Sdumbbell ptr_addr = drm_buffer_read_object(cmdbuf->buffer, 923254885Sdumbbell sizeof(stack_ptr_addr), &stack_ptr_addr); 924254885Sdumbbell ref_age_base = (u32 *)(unsigned long)get_unaligned(ptr_addr); 925254885Sdumbbell 926254885Sdumbbell for (i=0; i < header.scratch.n_bufs; i++) { 927254885Sdumbbell buf_idx = drm_buffer_pointer_to_dword(cmdbuf->buffer, 0); 928254885Sdumbbell *buf_idx *= 2; /* 8 bytes per buf */ 929254885Sdumbbell 930254885Sdumbbell if (DRM_COPY_TO_USER(ref_age_base + *buf_idx, 931254885Sdumbbell &dev_priv->scratch_ages[header.scratch.reg], 932254885Sdumbbell sizeof(u32))) 933254885Sdumbbell return -EINVAL; 934254885Sdumbbell 935254885Sdumbbell if (DRM_COPY_FROM_USER(&h_pending, 936254885Sdumbbell ref_age_base + *buf_idx + 1, 937254885Sdumbbell sizeof(u32))) 938254885Sdumbbell return -EINVAL; 939254885Sdumbbell 940254885Sdumbbell if (h_pending == 0) 941254885Sdumbbell return -EINVAL; 942254885Sdumbbell 943254885Sdumbbell h_pending--; 944254885Sdumbbell 945254885Sdumbbell if (DRM_COPY_TO_USER(ref_age_base + *buf_idx + 1, 946254885Sdumbbell &h_pending, 947254885Sdumbbell sizeof(u32))) 948254885Sdumbbell return -EINVAL; 949254885Sdumbbell 950254885Sdumbbell drm_buffer_advance(cmdbuf->buffer, sizeof(*buf_idx)); 951254885Sdumbbell } 952254885Sdumbbell 953254885Sdumbbell BEGIN_RING(2); 954254885Sdumbbell OUT_RING( CP_PACKET0( RADEON_SCRATCH_REG0 + header.scratch.reg * 4, 0 ) ); 955254885Sdumbbell OUT_RING( dev_priv->scratch_ages[header.scratch.reg] ); 956254885Sdumbbell ADVANCE_RING(); 957254885Sdumbbell 958254885Sdumbbell return 0; 959254885Sdumbbell} 960254885Sdumbbell 961254885Sdumbbell/** 962254885Sdumbbell * Uploads user-supplied vertex program instructions or parameters onto 963254885Sdumbbell * the graphics card. 964254885Sdumbbell * Called by r300_do_cp_cmdbuf. 965254885Sdumbbell */ 966254885Sdumbbellstatic inline int r300_emit_r500fp(drm_radeon_private_t *dev_priv, 967254885Sdumbbell drm_radeon_kcmd_buffer_t *cmdbuf, 968254885Sdumbbell drm_r300_cmd_header_t header) 969254885Sdumbbell{ 970254885Sdumbbell int sz; 971254885Sdumbbell int addr; 972254885Sdumbbell int type; 973254885Sdumbbell int isclamp; 974254885Sdumbbell int stride; 975254885Sdumbbell RING_LOCALS; 976254885Sdumbbell 977254885Sdumbbell sz = header.r500fp.count; 978254885Sdumbbell /* address is 9 bits 0 - 8, bit 1 of flags is part of address */ 979254885Sdumbbell addr = ((header.r500fp.adrhi_flags & 1) << 8) | header.r500fp.adrlo; 980254885Sdumbbell 981254885Sdumbbell type = !!(header.r500fp.adrhi_flags & R500FP_CONSTANT_TYPE); 982254885Sdumbbell isclamp = !!(header.r500fp.adrhi_flags & R500FP_CONSTANT_CLAMP); 983254885Sdumbbell 984254885Sdumbbell addr |= (type << 16); 985254885Sdumbbell addr |= (isclamp << 17); 986254885Sdumbbell 987254885Sdumbbell stride = type ? 4 : 6; 988254885Sdumbbell 989254885Sdumbbell DRM_DEBUG("r500fp %d %d type: %d\n", sz, addr, type); 990254885Sdumbbell if (!sz) 991254885Sdumbbell return 0; 992254885Sdumbbell if (sz * stride * 4 > drm_buffer_unprocessed(cmdbuf->buffer)) 993254885Sdumbbell return -EINVAL; 994254885Sdumbbell 995254885Sdumbbell BEGIN_RING(3 + sz * stride); 996254885Sdumbbell OUT_RING_REG(R500_GA_US_VECTOR_INDEX, addr); 997254885Sdumbbell OUT_RING(CP_PACKET0_TABLE(R500_GA_US_VECTOR_DATA, sz * stride - 1)); 998254885Sdumbbell OUT_RING_DRM_BUFFER(cmdbuf->buffer, sz * stride); 999254885Sdumbbell 1000254885Sdumbbell ADVANCE_RING(); 1001254885Sdumbbell 1002254885Sdumbbell return 0; 1003254885Sdumbbell} 1004254885Sdumbbell 1005254885Sdumbbell 1006254885Sdumbbell/** 1007254885Sdumbbell * Parses and validates a user-supplied command buffer and emits appropriate 1008254885Sdumbbell * commands on the DMA ring buffer. 1009254885Sdumbbell * Called by the ioctl handler function radeon_cp_cmdbuf. 1010254885Sdumbbell */ 1011254885Sdumbbellint r300_do_cp_cmdbuf(struct drm_device *dev, 1012254885Sdumbbell struct drm_file *file_priv, 1013254885Sdumbbell drm_radeon_kcmd_buffer_t *cmdbuf) 1014254885Sdumbbell{ 1015254885Sdumbbell drm_radeon_private_t *dev_priv = dev->dev_private; 1016254885Sdumbbell struct drm_radeon_master_private *master_priv = file_priv->masterp->driver_priv; 1017254885Sdumbbell struct drm_device_dma *dma = dev->dma; 1018254885Sdumbbell struct drm_buf *buf = NULL; 1019254885Sdumbbell int emit_dispatch_age = 0; 1020254885Sdumbbell int ret = 0; 1021254885Sdumbbell 1022254885Sdumbbell DRM_DEBUG("\n"); 1023254885Sdumbbell 1024254885Sdumbbell /* pacify */ 1025254885Sdumbbell r300_pacify(dev_priv); 1026254885Sdumbbell 1027254885Sdumbbell if (cmdbuf->nbox <= R300_SIMULTANEOUS_CLIPRECTS) { 1028254885Sdumbbell ret = r300_emit_cliprects(dev_priv, cmdbuf, 0); 1029254885Sdumbbell if (ret) 1030254885Sdumbbell goto cleanup; 1031254885Sdumbbell } 1032254885Sdumbbell 1033254885Sdumbbell while (drm_buffer_unprocessed(cmdbuf->buffer) 1034254885Sdumbbell >= sizeof(drm_r300_cmd_header_t)) { 1035254885Sdumbbell int idx; 1036254885Sdumbbell drm_r300_cmd_header_t *header, stack_header; 1037254885Sdumbbell 1038254885Sdumbbell header = drm_buffer_read_object(cmdbuf->buffer, 1039254885Sdumbbell sizeof(stack_header), &stack_header); 1040254885Sdumbbell 1041254885Sdumbbell switch (header->header.cmd_type) { 1042254885Sdumbbell case R300_CMD_PACKET0: 1043254885Sdumbbell DRM_DEBUG("R300_CMD_PACKET0\n"); 1044254885Sdumbbell ret = r300_emit_packet0(dev_priv, cmdbuf, *header); 1045254885Sdumbbell if (ret) { 1046254885Sdumbbell DRM_ERROR("r300_emit_packet0 failed\n"); 1047254885Sdumbbell goto cleanup; 1048254885Sdumbbell } 1049254885Sdumbbell break; 1050254885Sdumbbell 1051254885Sdumbbell case R300_CMD_VPU: 1052254885Sdumbbell DRM_DEBUG("R300_CMD_VPU\n"); 1053254885Sdumbbell ret = r300_emit_vpu(dev_priv, cmdbuf, *header); 1054254885Sdumbbell if (ret) { 1055254885Sdumbbell DRM_ERROR("r300_emit_vpu failed\n"); 1056254885Sdumbbell goto cleanup; 1057254885Sdumbbell } 1058254885Sdumbbell break; 1059254885Sdumbbell 1060254885Sdumbbell case R300_CMD_PACKET3: 1061254885Sdumbbell DRM_DEBUG("R300_CMD_PACKET3\n"); 1062254885Sdumbbell ret = r300_emit_packet3(dev_priv, cmdbuf, *header); 1063254885Sdumbbell if (ret) { 1064254885Sdumbbell DRM_ERROR("r300_emit_packet3 failed\n"); 1065254885Sdumbbell goto cleanup; 1066254885Sdumbbell } 1067254885Sdumbbell break; 1068254885Sdumbbell 1069254885Sdumbbell case R300_CMD_END3D: 1070254885Sdumbbell DRM_DEBUG("R300_CMD_END3D\n"); 1071254885Sdumbbell /* TODO: 1072254885Sdumbbell Ideally userspace driver should not need to issue this call, 1073254885Sdumbbell i.e. the drm driver should issue it automatically and prevent 1074254885Sdumbbell lockups. 1075254885Sdumbbell 1076254885Sdumbbell In practice, we do not understand why this call is needed and what 1077254885Sdumbbell it does (except for some vague guesses that it has to do with cache 1078254885Sdumbbell coherence) and so the user space driver does it. 1079254885Sdumbbell 1080254885Sdumbbell Once we are sure which uses prevent lockups the code could be moved 1081254885Sdumbbell into the kernel and the userspace driver will not 1082254885Sdumbbell need to use this command. 1083254885Sdumbbell 1084254885Sdumbbell Note that issuing this command does not hurt anything 1085254885Sdumbbell except, possibly, performance */ 1086254885Sdumbbell r300_pacify(dev_priv); 1087254885Sdumbbell break; 1088254885Sdumbbell 1089254885Sdumbbell case R300_CMD_CP_DELAY: 1090254885Sdumbbell /* simple enough, we can do it here */ 1091254885Sdumbbell DRM_DEBUG("R300_CMD_CP_DELAY\n"); 1092254885Sdumbbell { 1093254885Sdumbbell int i; 1094254885Sdumbbell RING_LOCALS; 1095254885Sdumbbell 1096254885Sdumbbell BEGIN_RING(header->delay.count); 1097254885Sdumbbell for (i = 0; i < header->delay.count; i++) 1098254885Sdumbbell OUT_RING(RADEON_CP_PACKET2); 1099254885Sdumbbell ADVANCE_RING(); 1100254885Sdumbbell } 1101254885Sdumbbell break; 1102254885Sdumbbell 1103254885Sdumbbell case R300_CMD_DMA_DISCARD: 1104254885Sdumbbell DRM_DEBUG("RADEON_CMD_DMA_DISCARD\n"); 1105254885Sdumbbell idx = header->dma.buf_idx; 1106254885Sdumbbell if (idx < 0 || idx >= dma->buf_count) { 1107254885Sdumbbell DRM_ERROR("buffer index %d (of %d max)\n", 1108254885Sdumbbell idx, dma->buf_count - 1); 1109254885Sdumbbell ret = -EINVAL; 1110254885Sdumbbell goto cleanup; 1111254885Sdumbbell } 1112254885Sdumbbell 1113254885Sdumbbell buf = dma->buflist[idx]; 1114254885Sdumbbell if (buf->file_priv != file_priv || buf->pending) { 1115254885Sdumbbell DRM_ERROR("bad buffer %p %p %d\n", 1116254885Sdumbbell buf->file_priv, file_priv, 1117254885Sdumbbell buf->pending); 1118254885Sdumbbell ret = -EINVAL; 1119254885Sdumbbell goto cleanup; 1120254885Sdumbbell } 1121254885Sdumbbell 1122254885Sdumbbell emit_dispatch_age = 1; 1123254885Sdumbbell r300_discard_buffer(dev, file_priv->masterp, buf); 1124254885Sdumbbell break; 1125254885Sdumbbell 1126254885Sdumbbell case R300_CMD_WAIT: 1127254885Sdumbbell DRM_DEBUG("R300_CMD_WAIT\n"); 1128254885Sdumbbell r300_cmd_wait(dev_priv, *header); 1129254885Sdumbbell break; 1130254885Sdumbbell 1131254885Sdumbbell case R300_CMD_SCRATCH: 1132254885Sdumbbell DRM_DEBUG("R300_CMD_SCRATCH\n"); 1133254885Sdumbbell ret = r300_scratch(dev_priv, cmdbuf, *header); 1134254885Sdumbbell if (ret) { 1135254885Sdumbbell DRM_ERROR("r300_scratch failed\n"); 1136254885Sdumbbell goto cleanup; 1137254885Sdumbbell } 1138254885Sdumbbell break; 1139254885Sdumbbell 1140254885Sdumbbell case R300_CMD_R500FP: 1141254885Sdumbbell if ((dev_priv->flags & RADEON_FAMILY_MASK) < CHIP_RV515) { 1142254885Sdumbbell DRM_ERROR("Calling r500 command on r300 card\n"); 1143254885Sdumbbell ret = -EINVAL; 1144254885Sdumbbell goto cleanup; 1145254885Sdumbbell } 1146254885Sdumbbell DRM_DEBUG("R300_CMD_R500FP\n"); 1147254885Sdumbbell ret = r300_emit_r500fp(dev_priv, cmdbuf, *header); 1148254885Sdumbbell if (ret) { 1149254885Sdumbbell DRM_ERROR("r300_emit_r500fp failed\n"); 1150254885Sdumbbell goto cleanup; 1151254885Sdumbbell } 1152254885Sdumbbell break; 1153254885Sdumbbell default: 1154254885Sdumbbell DRM_ERROR("bad cmd_type %i at byte %d\n", 1155254885Sdumbbell header->header.cmd_type, 1156254885Sdumbbell cmdbuf->buffer->iterator - (int)sizeof(*header)); 1157254885Sdumbbell ret = -EINVAL; 1158254885Sdumbbell goto cleanup; 1159254885Sdumbbell } 1160254885Sdumbbell } 1161254885Sdumbbell 1162254885Sdumbbell DRM_DEBUG("END\n"); 1163254885Sdumbbell 1164254885Sdumbbell cleanup: 1165254885Sdumbbell r300_pacify(dev_priv); 1166254885Sdumbbell 1167254885Sdumbbell /* We emit the vertex buffer age here, outside the pacifier "brackets" 1168254885Sdumbbell * for two reasons: 1169254885Sdumbbell * (1) This may coalesce multiple age emissions into a single one and 1170254885Sdumbbell * (2) more importantly, some chips lock up hard when scratch registers 1171254885Sdumbbell * are written inside the pacifier bracket. 1172254885Sdumbbell */ 1173254885Sdumbbell if (emit_dispatch_age) { 1174254885Sdumbbell RING_LOCALS; 1175254885Sdumbbell 1176254885Sdumbbell /* Emit the vertex buffer age */ 1177254885Sdumbbell BEGIN_RING(2); 1178254885Sdumbbell RADEON_DISPATCH_AGE(master_priv->sarea_priv->last_dispatch); 1179254885Sdumbbell ADVANCE_RING(); 1180254885Sdumbbell } 1181254885Sdumbbell 1182254885Sdumbbell COMMIT_RING(); 1183254885Sdumbbell 1184254885Sdumbbell return ret; 1185254885Sdumbbell} 1186