1/*- 2 * Copyright (c) 2001 Jake Burkholder. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 */ 28 29#ifndef _MACHINE_CPUFUNC_H_ 30#define _MACHINE_CPUFUNC_H_ 31 32#include <machine/asi.h> 33#include <machine/pstate.h> 34 35struct thread; 36 37/* 38 * Membar operand macros for use in other macros when # is a special 39 * character. Keep these in sync with what the hardware expects. 40 */ 41#define C_Lookaside (0) 42#define C_MemIssue (1) 43#define C_Sync (2) 44#define M_LoadLoad (0) 45#define M_StoreLoad (1) 46#define M_LoadStore (2) 47#define M_StoreStore (3) 48 49#define CMASK_SHIFT (4) 50#define MMASK_SHIFT (0) 51 52#define CMASK_GEN(bit) ((1 << (bit)) << CMASK_SHIFT) 53#define MMASK_GEN(bit) ((1 << (bit)) << MMASK_SHIFT) 54 55#define Lookaside CMASK_GEN(C_Lookaside) 56#define MemIssue CMASK_GEN(C_MemIssue) 57#define Sync CMASK_GEN(C_Sync) 58#define LoadLoad MMASK_GEN(M_LoadLoad) 59#define StoreLoad MMASK_GEN(M_StoreLoad) 60#define LoadStore MMASK_GEN(M_LoadStore) 61#define StoreStore MMASK_GEN(M_StoreStore) 62 63#define casa(rs1, rs2, rd, asi) ({ \ 64 u_int __rd = (uint32_t)(rd); \ 65 __asm __volatile("casa [%2] %3, %4, %0" \ 66 : "+r" (__rd), "=m" (*rs1) \ 67 : "r" (rs1), "n" (asi), "r" (rs2), "m" (*rs1)); \ 68 __rd; \ 69}) 70 71#define casxa(rs1, rs2, rd, asi) ({ \ 72 u_long __rd = (uint64_t)(rd); \ 73 __asm __volatile("casxa [%2] %3, %4, %0" \ 74 : "+r" (__rd), "=m" (*rs1) \ 75 : "r" (rs1), "n" (asi), "r" (rs2), "m" (*rs1)); \ 76 __rd; \ 77}) 78 79#define flush(va) do { \ 80 __asm __volatile("flush %0" : : "r" (va)); \ 81} while (0) 82 83#define flushw() do { \ 84 __asm __volatile("flushw" : :); \ 85} while (0) 86 87#define mov(val, reg) do { \ 88 __asm __volatile("mov %0, %" __XSTRING(reg) : : "r" (val)); \ 89} while (0) 90 91/* Generate ld*a/st*a functions for non-constant ASIs. */ 92#define LDNC_GEN(tp, o) \ 93 static __inline tp \ 94 o ## _nc(caddr_t va, int asi) \ 95 { \ 96 tp r; \ 97 __asm __volatile("wr %2, 0, %%asi;" #o " [%1] %%asi, %0"\ 98 : "=r" (r) : "r" (va), "r" (asi)); \ 99 return (r); \ 100 } 101 102LDNC_GEN(u_char, lduba); 103LDNC_GEN(u_short, lduha); 104LDNC_GEN(u_int, lduwa); 105LDNC_GEN(u_long, ldxa); 106 107#define LD_GENERIC(va, asi, op, type) ({ \ 108 type __r; \ 109 __asm __volatile(#op " [%1] %2, %0" \ 110 : "=r" (__r) : "r" (va), "n" (asi)); \ 111 __r; \ 112}) 113 114#define lduba(va, asi) LD_GENERIC(va, asi, lduba, u_char) 115#define lduha(va, asi) LD_GENERIC(va, asi, lduha, u_short) 116#define lduwa(va, asi) LD_GENERIC(va, asi, lduwa, u_int) 117#define ldxa(va, asi) LD_GENERIC(va, asi, ldxa, u_long) 118 119#define STNC_GEN(tp, o) \ 120 static __inline void \ 121 o ## _nc(caddr_t va, int asi, tp val) \ 122 { \ 123 __asm __volatile("wr %2, 0, %%asi;" #o " %0, [%1] %%asi"\ 124 : : "r" (val), "r" (va), "r" (asi)); \ 125 } 126 127STNC_GEN(u_char, stba); 128STNC_GEN(u_short, stha); 129STNC_GEN(u_int, stwa); 130STNC_GEN(u_long, stxa); 131 132#define ST_GENERIC(va, asi, val, op) \ 133 __asm __volatile(#op " %0, [%1] %2" \ 134 : : "r" (val), "r" (va), "n" (asi)); \ 135 136#define stba(va, asi, val) ST_GENERIC(va, asi, val, stba) 137#define stha(va, asi, val) ST_GENERIC(va, asi, val, stha) 138#define stwa(va, asi, val) ST_GENERIC(va, asi, val, stwa) 139#define stxa(va, asi, val) ST_GENERIC(va, asi, val, stxa) 140 141/* 142 * Attempt to read from addr, val. If a Data Access Error trap happens, 143 * they return -1 and the contents of val is undefined. A return of 0 144 * means no trap happened, and the contents of val is valid. 145 */ 146int fasword8(u_long asi, void *addr, uint8_t *val); 147int fasword16(u_long asi, void *addr, uint16_t *val); 148int fasword32(u_long asi, void *addr, uint32_t *val); 149 150#define membar(mask) do { \ 151 __asm __volatile("membar %0" : : "n" (mask) : "memory"); \ 152} while (0) 153 154#define rd(name) ({ \ 155 uint64_t __sr; \ 156 __asm __volatile("rd %%" #name ", %0" : "=r" (__sr) :); \ 157 __sr; \ 158}) 159 160#define wr(name, val, xorval) do { \ 161 __asm __volatile("wr %0, %1, %%" #name \ 162 : : "r" (val), "rI" (xorval)); \ 163} while (0) 164 165#define rdpr(name) ({ \ 166 uint64_t __pr; \ 167 __asm __volatile("rdpr %%" #name", %0" : "=r" (__pr) :); \ 168 __pr; \ 169}) 170 171#define wrpr(name, val, xorval) do { \ 172 __asm __volatile("wrpr %0, %1, %%" #name \ 173 : : "r" (val), "rI" (xorval)); \ 174} while (0) 175 176/* 177 * Trick GAS/GCC into compiling access to TICK/(S)TICK_COMPARE independently 178 * of the selected instruction set. 179 */ 180#define rdtickcmpr() rd(asr23) 181#define rdstick() rd(asr24) 182#define rdstickcmpr() rd(asr25) 183#define wrtickcmpr(val, xorval) wr(asr23, (val), (xorval)) 184#define wrstick(val, xorval) wr(asr24, (val), (xorval)) 185#define wrstickcmpr(val, xorval) wr(asr25, (val), (xorval)) 186 187/* 188 * Macro intended to be used instead of wr(asr23, val, xorval) for writing to 189 * the TICK_COMPARE register in order to avoid a bug in BlackBird CPUs that 190 * can cause these writes to fail under certain conditions which in turn 191 * causes the hardclock to stop. The workaround is to read the TICK_COMPARE 192 * register back immediately after writing to it with these two instructions 193 * aligned to a quadword boundary in order to ensure that I$ misses won't 194 * split them up. 195 */ 196#define wrtickcmpr_bbwar(val, xorval) ({ \ 197 __asm __volatile( \ 198 " ba,pt %%xcc, 1f ; " \ 199 " nop ; " \ 200 " .align 128 ; " \ 201 "1: wr %0, %1, %%asr23 ; " \ 202 " rd %%asr23, %%g0 ; " \ 203 : : "r" (val), "rI" (xorval)); \ 204}) 205 206static __inline void 207breakpoint(void) 208{ 209 210 __asm __volatile("ta %%xcc, 1" : :); 211} 212 213static __inline register_t 214intr_disable(void) 215{ 216 register_t s; 217 218 s = rdpr(pstate); 219 wrpr(pstate, s & ~PSTATE_IE, 0); 220 return (s); 221} 222#define intr_restore(s) wrpr(pstate, (s), 0) 223 224/* 225 * In some places, it is required that the store is directly followed by a 226 * membar #Sync. Don't trust the compiler to not insert instructions in 227 * between. We also need to disable interrupts completely. 228 */ 229#define stxa_sync(va, asi, val) do { \ 230 register_t s; \ 231 s = intr_disable(); \ 232 __asm __volatile("stxa %0, [%1] %2; membar #Sync" \ 233 : : "r" (val), "r" (va), "n" (asi)); \ 234 intr_restore(s); \ 235} while (0) 236 237void ascopy(u_long asi, vm_offset_t src, vm_offset_t dst, size_t len); 238void ascopyfrom(u_long sasi, vm_offset_t src, caddr_t dst, size_t len); 239void ascopyto(caddr_t src, u_long dasi, vm_offset_t dst, size_t len); 240void aszero(u_long asi, vm_offset_t dst, size_t len); 241 242/* 243 * Ultrasparc II doesn't implement popc in hardware. 244 */ 245#if 0 246#define HAVE_INLINE_FFS 247/* 248 * See page 202 of the SPARC v9 Architecture Manual. 249 */ 250static __inline int 251ffs(int mask) 252{ 253 int result; 254 int neg; 255 int tmp; 256 257 __asm __volatile( 258 " neg %3, %1 ; " 259 " xnor %3, %1, %2 ; " 260 " popc %2, %0 ; " 261 " movrz %3, %%g0, %0 ; " 262 : "=r" (result), "=r" (neg), "=r" (tmp) : "r" (mask)); 263 return (result); 264} 265#endif 266 267#undef LDNC_GEN 268#undef STNC_GEN 269 270#endif /* !_MACHINE_CPUFUNC_H_ */ 271