180708Sjake/*- 280708Sjake * Copyright (c) 2001 Jake Burkholder. 380708Sjake * All rights reserved. 480708Sjake * 580708Sjake * Redistribution and use in source and binary forms, with or without 680708Sjake * modification, are permitted provided that the following conditions 780708Sjake * are met: 880708Sjake * 1. Redistributions of source code must retain the above copyright 980708Sjake * notice, this list of conditions and the following disclaimer. 1080708Sjake * 2. Redistributions in binary form must reproduce the above copyright 1180708Sjake * notice, this list of conditions and the following disclaimer in the 1280708Sjake * documentation and/or other materials provided with the distribution. 1380708Sjake * 1481334Sobrien * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1580708Sjake * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1680708Sjake * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1781334Sobrien * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1880708Sjake * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1980708Sjake * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2080708Sjake * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2180708Sjake * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2280708Sjake * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2380708Sjake * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2480708Sjake * SUCH DAMAGE. 2580708Sjake * 2680708Sjake * $FreeBSD$ 2780708Sjake */ 2880708Sjake 2980708Sjake#ifndef _MACHINE_CPUFUNC_H_ 3080708Sjake#define _MACHINE_CPUFUNC_H_ 3180708Sjake 3280709Sjake#include <machine/asi.h> 3380709Sjake#include <machine/pstate.h> 3480709Sjake 3593264Sdillonstruct thread; 3693264Sdillon 3780709Sjake/* 38181701Smarius * Membar operand macros for use in other macros when # is a special 3980709Sjake * character. Keep these in sync with what the hardware expects. 4080709Sjake */ 4180709Sjake#define C_Lookaside (0) 4280709Sjake#define C_MemIssue (1) 4380709Sjake#define C_Sync (2) 4480709Sjake#define M_LoadLoad (0) 4580709Sjake#define M_StoreLoad (1) 4680709Sjake#define M_LoadStore (2) 4780709Sjake#define M_StoreStore (3) 4880709Sjake 4980709Sjake#define CMASK_SHIFT (4) 5080709Sjake#define MMASK_SHIFT (0) 5180709Sjake 5280709Sjake#define CMASK_GEN(bit) ((1 << (bit)) << CMASK_SHIFT) 5380709Sjake#define MMASK_GEN(bit) ((1 << (bit)) << MMASK_SHIFT) 5480709Sjake 5580709Sjake#define Lookaside CMASK_GEN(C_Lookaside) 5680709Sjake#define MemIssue CMASK_GEN(C_MemIssue) 5780709Sjake#define Sync CMASK_GEN(C_Sync) 5880709Sjake#define LoadLoad MMASK_GEN(M_LoadLoad) 5980709Sjake#define StoreLoad MMASK_GEN(M_StoreLoad) 6080709Sjake#define LoadStore MMASK_GEN(M_LoadStore) 6180709Sjake#define StoreStore MMASK_GEN(M_StoreStore) 6280709Sjake 6380709Sjake#define casa(rs1, rs2, rd, asi) ({ \ 64129568Smarius u_int __rd = (uint32_t)(rd); \ 65148453Sjhb __asm __volatile("casa [%2] %3, %4, %0" \ 66148453Sjhb : "+r" (__rd), "=m" (*rs1) \ 67148453Sjhb : "r" (rs1), "n" (asi), "r" (rs2), "m" (*rs1)); \ 6880709Sjake __rd; \ 6980709Sjake}) 7080709Sjake 7180709Sjake#define casxa(rs1, rs2, rd, asi) ({ \ 72129568Smarius u_long __rd = (uint64_t)(rd); \ 73148453Sjhb __asm __volatile("casxa [%2] %3, %4, %0" \ 74148453Sjhb : "+r" (__rd), "=m" (*rs1) \ 75148453Sjhb : "r" (rs1), "n" (asi), "r" (rs2), "m" (*rs1)); \ 7680709Sjake __rd; \ 7780709Sjake}) 7880709Sjake 7980709Sjake#define flush(va) do { \ 8080709Sjake __asm __volatile("flush %0" : : "r" (va)); \ 8180709Sjake} while (0) 8280709Sjake 8382896Sjake#define flushw() do { \ 8482896Sjake __asm __volatile("flushw" : :); \ 8582896Sjake} while (0) 8682896Sjake 8789033Sjake#define mov(val, reg) do { \ 8889033Sjake __asm __volatile("mov %0, %" __XSTRING(reg) : : "r" (val)); \ 8989033Sjake} while (0) 9089033Sjake 91181701Smarius/* Generate ld*a/st*a functions for non-constant ASIs. */ 92181701Smarius#define LDNC_GEN(tp, o) \ 9386228Stmm static __inline tp \ 9486228Stmm o ## _nc(caddr_t va, int asi) \ 9586228Stmm { \ 9686228Stmm tp r; \ 9786228Stmm __asm __volatile("wr %2, 0, %%asi;" #o " [%1] %%asi, %0"\ 9886228Stmm : "=r" (r) : "r" (va), "r" (asi)); \ 9986228Stmm return (r); \ 10086228Stmm } 10186228Stmm 10286228StmmLDNC_GEN(u_char, lduba); 10386228StmmLDNC_GEN(u_short, lduha); 10486228StmmLDNC_GEN(u_int, lduwa); 10586228StmmLDNC_GEN(u_long, ldxa); 10686228Stmm 10786228Stmm#define LD_GENERIC(va, asi, op, type) ({ \ 10886228Stmm type __r; \ 10986228Stmm __asm __volatile(#op " [%1] %2, %0" \ 11080709Sjake : "=r" (__r) : "r" (va), "n" (asi)); \ 11180709Sjake __r; \ 11280709Sjake}) 11380709Sjake 11486228Stmm#define lduba(va, asi) LD_GENERIC(va, asi, lduba, u_char) 11586228Stmm#define lduha(va, asi) LD_GENERIC(va, asi, lduha, u_short) 11686228Stmm#define lduwa(va, asi) LD_GENERIC(va, asi, lduwa, u_int) 11786228Stmm#define ldxa(va, asi) LD_GENERIC(va, asi, ldxa, u_long) 11886228Stmm 119181701Smarius#define STNC_GEN(tp, o) \ 12086228Stmm static __inline void \ 12186228Stmm o ## _nc(caddr_t va, int asi, tp val) \ 12286228Stmm { \ 12386228Stmm __asm __volatile("wr %2, 0, %%asi;" #o " %0, [%1] %%asi"\ 12486228Stmm : : "r" (val), "r" (va), "r" (asi)); \ 12586228Stmm } 12686228Stmm 12786228StmmSTNC_GEN(u_char, stba); 12886228StmmSTNC_GEN(u_short, stha); 12986228StmmSTNC_GEN(u_int, stwa); 13086228StmmSTNC_GEN(u_long, stxa); 13186228Stmm 13286228Stmm#define ST_GENERIC(va, asi, val, op) \ 13386228Stmm __asm __volatile(#op " %0, [%1] %2" \ 13480709Sjake : : "r" (val), "r" (va), "n" (asi)); \ 13580709Sjake 13686228Stmm#define stba(va, asi, val) ST_GENERIC(va, asi, val, stba) 13786228Stmm#define stha(va, asi, val) ST_GENERIC(va, asi, val, stha) 13886228Stmm#define stwa(va, asi, val) ST_GENERIC(va, asi, val, stwa) 13986228Stmm#define stxa(va, asi, val) ST_GENERIC(va, asi, val, stxa) 14086228Stmm 141116659Sjmg/* 142116659Sjmg * Attempt to read from addr, val. If a Data Access Error trap happens, 143116659Sjmg * they return -1 and the contents of val is undefined. A return of 0 144116659Sjmg * means no trap happened, and the contents of val is valid. 145116659Sjmg */ 146116659Sjmgint fasword8(u_long asi, void *addr, uint8_t *val); 147116659Sjmgint fasword16(u_long asi, void *addr, uint16_t *val); 148116659Sjmgint fasword32(u_long asi, void *addr, uint32_t *val); 149116659Sjmg 15080709Sjake#define membar(mask) do { \ 15188620Sjake __asm __volatile("membar %0" : : "n" (mask) : "memory"); \ 15280709Sjake} while (0) 15380709Sjake 15480709Sjake#define rd(name) ({ \ 155129568Smarius uint64_t __sr; \ 15680709Sjake __asm __volatile("rd %%" #name ", %0" : "=r" (__sr) :); \ 15780709Sjake __sr; \ 15880709Sjake}) 15980709Sjake 160216801Smarius#define wr(name, val, xorval) do { \ 16180709Sjake __asm __volatile("wr %0, %1, %%" #name \ 162216801Smarius : : "r" (val), "rI" (xorval)); \ 16380709Sjake} while (0) 16480709Sjake 16580709Sjake#define rdpr(name) ({ \ 166129568Smarius uint64_t __pr; \ 16780709Sjake __asm __volatile("rdpr %%" #name", %0" : "=r" (__pr) :); \ 16880709Sjake __pr; \ 16980709Sjake}) 17080709Sjake 171216801Smarius#define wrpr(name, val, xorval) do { \ 17280709Sjake __asm __volatile("wrpr %0, %1, %%" #name \ 173216801Smarius : : "r" (val), "rI" (xorval)); \ 17480709Sjake} while (0) 17580709Sjake 176145150Smarius/* 177216628Smarius * Trick GAS/GCC into compiling access to TICK/(S)TICK_COMPARE independently 178182730Smarius * of the selected instruction set. 179182730Smarius */ 180216801Smarius#define rdtickcmpr() rd(asr23) 181216801Smarius#define rdstick() rd(asr24) 182216801Smarius#define rdstickcmpr() rd(asr25) 183216801Smarius#define wrtickcmpr(val, xorval) wr(asr23, (val), (xorval)) 184216801Smarius#define wrstick(val, xorval) wr(asr24, (val), (xorval)) 185216801Smarius#define wrstickcmpr(val, xorval) wr(asr25, (val), (xorval)) 186182730Smarius 187182730Smarius/* 188216801Smarius * Macro intended to be used instead of wr(asr23, val, xorval) for writing to 189181701Smarius * the TICK_COMPARE register in order to avoid a bug in BlackBird CPUs that 190220939Smarius * can cause these writes to fail under certain conditions which in turn 191182078Smarius * causes the hardclock to stop. The workaround is to read the TICK_COMPARE 192182078Smarius * register back immediately after writing to it with these two instructions 193182078Smarius * aligned to a quadword boundary in order to ensure that I$ misses won't 194182078Smarius * split them up. 195145150Smarius */ 196216801Smarius#define wrtickcmpr_bbwar(val, xorval) ({ \ 197145150Smarius __asm __volatile( \ 198145150Smarius " ba,pt %%xcc, 1f ; " \ 199145150Smarius " nop ; " \ 200182078Smarius " .align 128 ; " \ 201145150Smarius "1: wr %0, %1, %%asr23 ; " \ 202145150Smarius " rd %%asr23, %%g0 ; " \ 203216801Smarius : : "r" (val), "rI" (xorval)); \ 204145150Smarius}) 205145150Smarius 20680709Sjakestatic __inline void 20780709Sjakebreakpoint(void) 20880709Sjake{ 209181701Smarius 21081894Sjake __asm __volatile("ta %%xcc, 1" : :); 21180709Sjake} 21280709Sjake 21392861Simpstatic __inline register_t 21490611Stmmintr_disable(void) 21590611Stmm{ 216181701Smarius register_t s; 21790611Stmm 21890611Stmm s = rdpr(pstate); 21990611Stmm wrpr(pstate, s & ~PSTATE_IE, 0); 22090611Stmm return (s); 22190611Stmm} 22290611Stmm#define intr_restore(s) wrpr(pstate, (s), 0) 22390611Stmm 22490611Stmm/* 22590611Stmm * In some places, it is required that the store is directly followed by a 226181701Smarius * membar #Sync. Don't trust the compiler to not insert instructions in 227181701Smarius * between. We also need to disable interrupts completely. 22890611Stmm */ 22990611Stmm#define stxa_sync(va, asi, val) do { \ 230181701Smarius register_t s; \ 231145149Smarius s = intr_disable(); \ 23290611Stmm __asm __volatile("stxa %0, [%1] %2; membar #Sync" \ 23390611Stmm : : "r" (val), "r" (va), "n" (asi)); \ 23490611Stmm intr_restore(s); \ 23590611Stmm} while (0) 23690611Stmm 23786520Sjakevoid ascopy(u_long asi, vm_offset_t src, vm_offset_t dst, size_t len); 23886228Stmmvoid ascopyfrom(u_long sasi, vm_offset_t src, caddr_t dst, size_t len); 23986228Stmmvoid ascopyto(caddr_t src, u_long dasi, vm_offset_t dst, size_t len); 24086520Sjakevoid aszero(u_long asi, vm_offset_t dst, size_t len); 24186228Stmm 24281894Sjake/* 243181701Smarius * Ultrasparc II doesn't implement popc in hardware. 24481894Sjake */ 24580709Sjake#if 0 24680709Sjake#define HAVE_INLINE_FFS 24780709Sjake/* 24880709Sjake * See page 202 of the SPARC v9 Architecture Manual. 24980709Sjake */ 25080709Sjakestatic __inline int 25180709Sjakeffs(int mask) 25280709Sjake{ 25380709Sjake int result; 25480709Sjake int neg; 25580709Sjake int tmp; 25680709Sjake 25780709Sjake __asm __volatile( 25880709Sjake " neg %3, %1 ; " 25980709Sjake " xnor %3, %1, %2 ; " 26080709Sjake " popc %2, %0 ; " 26180709Sjake " movrz %3, %%g0, %0 ; " 26280709Sjake : "=r" (result), "=r" (neg), "=r" (tmp) : "r" (mask)); 26380709Sjake return (result); 26480709Sjake} 26580709Sjake#endif 26680709Sjake 26786228Stmm#undef LDNC_GEN 26886228Stmm#undef STNC_GEN 26986228Stmm 27080708Sjake#endif /* !_MACHINE_CPUFUNC_H_ */ 271