11817Sdg/*- 2122296Speter * Copyright (c) 2003 Peter Wemm. 31817Sdg * Copyright (c) 1993 The Regents of the University of California. 41817Sdg * All rights reserved. 51817Sdg * 61817Sdg * Redistribution and use in source and binary forms, with or without 71817Sdg * modification, are permitted provided that the following conditions 81817Sdg * are met: 91817Sdg * 1. Redistributions of source code must retain the above copyright 101817Sdg * notice, this list of conditions and the following disclaimer. 111817Sdg * 2. Redistributions in binary form must reproduce the above copyright 121817Sdg * notice, this list of conditions and the following disclaimer in the 131817Sdg * documentation and/or other materials provided with the distribution. 141817Sdg * 4. Neither the name of the University nor the names of its contributors 151817Sdg * may be used to endorse or promote products derived from this software 161817Sdg * without specific prior written permission. 171817Sdg * 181817Sdg * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 191817Sdg * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 201817Sdg * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 211817Sdg * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 221817Sdg * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 231817Sdg * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 241817Sdg * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 251817Sdg * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 261817Sdg * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 271817Sdg * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 281817Sdg * SUCH DAMAGE. 291817Sdg * 3050477Speter * $FreeBSD: stable/10/sys/amd64/include/cpufunc.h 313150 2017-02-03 12:20:44Z kib $ 311817Sdg */ 321817Sdg 334Srgrimes/* 344Srgrimes * Functions to provide access to special i386 instructions. 3591497Smarkm * This in included in sys/systm.h, and that file should be 3691497Smarkm * used in preference to this. 374Srgrimes */ 384Srgrimes 39719Swollman#ifndef _MACHINE_CPUFUNC_H_ 404479Sbde#define _MACHINE_CPUFUNC_H_ 41719Swollman 42143063Sjoerg#ifndef _SYS_CDEFS_H_ 43143063Sjoerg#error this file needs sys/cdefs.h as a prerequisite 44143063Sjoerg#endif 45143063Sjoerg 46103778Speterstruct region_descriptor; 4793264Sdillon 48220628Sjkim#define readb(va) (*(volatile uint8_t *) (va)) 49220628Sjkim#define readw(va) (*(volatile uint16_t *) (va)) 50220628Sjkim#define readl(va) (*(volatile uint32_t *) (va)) 51220628Sjkim#define readq(va) (*(volatile uint64_t *) (va)) 5238392Sdfr 53220628Sjkim#define writeb(va, d) (*(volatile uint8_t *) (va) = (d)) 54220628Sjkim#define writew(va, d) (*(volatile uint16_t *) (va) = (d)) 55220628Sjkim#define writel(va, d) (*(volatile uint32_t *) (va) = (d)) 56220628Sjkim#define writeq(va, d) (*(volatile uint64_t *) (va) = (d)) 5738392Sdfr 58143063Sjoerg#if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE) 594Srgrimes 6015122Sbdestatic __inline void 6115122Sbdebreakpoint(void) 623102Sdg{ 634479Sbde __asm __volatile("int $3"); 643102Sdg} 653102Sdg 6655672Sbdestatic __inline u_int 6755672Sbdebsfl(u_int mask) 6855672Sbde{ 6955672Sbde u_int result; 7055672Sbde 7188118Sjhb __asm __volatile("bsfl %1,%0" : "=r" (result) : "rm" (mask)); 7255672Sbde return (result); 7355672Sbde} 7455672Sbde 75123181Speterstatic __inline u_long 76123181Speterbsfq(u_long mask) 77123181Speter{ 78123181Speter u_long result; 79123181Speter 80123181Speter __asm __volatile("bsfq %1,%0" : "=r" (result) : "rm" (mask)); 81123181Speter return (result); 82123181Speter} 83123181Speter 8455672Sbdestatic __inline u_int 8555672Sbdebsrl(u_int mask) 8655672Sbde{ 8755672Sbde u_int result; 8855672Sbde 8988118Sjhb __asm __volatile("bsrl %1,%0" : "=r" (result) : "rm" (mask)); 9055672Sbde return (result); 9155672Sbde} 9255672Sbde 93123181Speterstatic __inline u_long 94123181Speterbsrq(u_long mask) 95123181Speter{ 96123181Speter u_long result; 97123181Speter 98123181Speter __asm __volatile("bsrq %1,%0" : "=r" (result) : "rm" (mask)); 99123181Speter return (result); 100123181Speter} 101123181Speter 1023102Sdgstatic __inline void 103195820Skibclflush(u_long addr) 104195820Skib{ 105195820Skib 106195820Skib __asm __volatile("clflush %0" : : "m" (*(char *)addr)); 107195820Skib} 108195820Skib 109195820Skibstatic __inline void 110290189Skibclflushopt(u_long addr) 111290189Skib{ 112290189Skib 113290189Skib __asm __volatile(".byte 0x66;clflush %0" : : "m" (*(char *)addr)); 114290189Skib} 115290189Skib 116290189Skibstatic __inline void 117238311Sjhbclts(void) 118238311Sjhb{ 119238311Sjhb 120238311Sjhb __asm __volatile("clts"); 121238311Sjhb} 122238311Sjhb 123238311Sjhbstatic __inline void 1244479Sbdedisable_intr(void) 1253102Sdg{ 12610004Sdyson __asm __volatile("cli" : : : "memory"); 1273102Sdg} 1283102Sdg 1293102Sdgstatic __inline void 13094386Sdwmalonedo_cpuid(u_int ax, u_int *p) 13194386Sdwmalone{ 13294386Sdwmalone __asm __volatile("cpuid" 13394386Sdwmalone : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 13494386Sdwmalone : "0" (ax)); 13594386Sdwmalone} 13694386Sdwmalone 13794386Sdwmalonestatic __inline void 138146170Snectarcpuid_count(u_int ax, u_int cx, u_int *p) 139146170Snectar{ 140146170Snectar __asm __volatile("cpuid" 141146170Snectar : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 142146170Snectar : "0" (ax), "c" (cx)); 143146170Snectar} 144146170Snectar 145146170Snectarstatic __inline void 1464479Sbdeenable_intr(void) 1473102Sdg{ 14810342Sbde __asm __volatile("sti"); 1493102Sdg} 1503102Sdg 151126846Sbde#ifdef _KERNEL 152126846Sbde 15350054Speter#define HAVE_INLINE_FFS 154132888Sps#define ffs(x) __builtin_ffs(x) 15550054Speter 156123181Speter#define HAVE_INLINE_FFSL 157123181Speter 158123181Speterstatic __inline int 159123181Speterffsl(long mask) 160123181Speter{ 161123181Speter return (mask == 0 ? mask : (int)bsfq((u_long)mask) + 1); 162123181Speter} 163123181Speter 164289818Savg#define HAVE_INLINE_FFSLL 165289818Savg 166289818Savgstatic __inline int 167289818Savgffsll(long long mask) 168289818Savg{ 169289818Savg return (ffsl((long)mask)); 170289818Savg} 171289818Savg 17217384Swollman#define HAVE_INLINE_FLS 17317384Swollman 17417384Swollmanstatic __inline int 17517384Swollmanfls(int mask) 17617384Swollman{ 177100078Smarkm return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1); 17817384Swollman} 17917384Swollman 180123181Speter#define HAVE_INLINE_FLSL 181123181Speter 182123181Speterstatic __inline int 183123181Speterflsl(long mask) 184123181Speter{ 185123181Speter return (mask == 0 ? mask : (int)bsrq((u_long)mask) + 1); 186123181Speter} 187123181Speter 188289818Savg#define HAVE_INLINE_FLSLL 189289818Savg 190289818Savgstatic __inline int 191289818Savgflsll(long long mask) 192289818Savg{ 193289818Savg return (flsl((long)mask)); 194289818Savg} 195289818Savg 196126846Sbde#endif /* _KERNEL */ 197126846Sbde 198103749Smarkmstatic __inline void 199103749Smarkmhalt(void) 200103749Smarkm{ 201103749Smarkm __asm __volatile("hlt"); 202103749Smarkm} 203103749Smarkm 2044479Sbdestatic __inline u_char 205190919Sedinb(u_int port) 2063102Sdg{ 2074479Sbde u_char data; 2084479Sbde 209220629Sjkim __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port)); 2104479Sbde return (data); 2113102Sdg} 2123102Sdg 21337552Sbdestatic __inline u_int 2144479Sbdeinl(u_int port) 2152826Sdg{ 21637552Sbde u_int data; 2172826Sdg 218220629Sjkim __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port)); 2194479Sbde return (data); 2202826Sdg} 2212826Sdg 2224479Sbdestatic __inline void 223201369Sobrieninsb(u_int port, void *addr, size_t count) 2242826Sdg{ 2254479Sbde __asm __volatile("cld; rep; insb" 226201369Sobrien : "+D" (addr), "+c" (count) 22788118Sjhb : "d" (port) 22842427Sbde : "memory"); 2294479Sbde} 2302826Sdg 2314479Sbdestatic __inline void 232201369Sobrieninsw(u_int port, void *addr, size_t count) 2334479Sbde{ 2344479Sbde __asm __volatile("cld; rep; insw" 235201369Sobrien : "+D" (addr), "+c" (count) 23688118Sjhb : "d" (port) 23742427Sbde : "memory"); 2382826Sdg} 2392826Sdg 2404479Sbdestatic __inline void 241201369Sobrieninsl(u_int port, void *addr, size_t count) 2422826Sdg{ 2434479Sbde __asm __volatile("cld; rep; insl" 244201369Sobrien : "+D" (addr), "+c" (count) 24588118Sjhb : "d" (port) 24642427Sbde : "memory"); 2472826Sdg} 2482826Sdg 24918567Sbdestatic __inline void 25024112Skatoinvd(void) 25124112Skato{ 25224112Skato __asm __volatile("invd"); 25324112Skato} 25424112Skato 2554479Sbdestatic __inline u_short 2564479Sbdeinw(u_int port) 2572826Sdg{ 2584479Sbde u_short data; 2594479Sbde 260220629Sjkim __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port)); 2614479Sbde return (data); 2622826Sdg} 2632826Sdg 2644479Sbdestatic __inline void 265190919Sedoutb(u_int port, u_char data) 2662826Sdg{ 267220629Sjkim __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port)); 2682826Sdg} 2692826Sdg 2704479Sbdestatic __inline void 27137552Sbdeoutl(u_int port, u_int data) 2724Srgrimes{ 273220629Sjkim __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port)); 2744Srgrimes} 2754Srgrimes 2764479Sbdestatic __inline void 277201369Sobrienoutsb(u_int port, const void *addr, size_t count) 2784Srgrimes{ 2794479Sbde __asm __volatile("cld; rep; outsb" 280201369Sobrien : "+S" (addr), "+c" (count) 28188118Sjhb : "d" (port)); 2824Srgrimes} 2834Srgrimes 2844479Sbdestatic __inline void 285201369Sobrienoutsw(u_int port, const void *addr, size_t count) 2864Srgrimes{ 2874479Sbde __asm __volatile("cld; rep; outsw" 288201369Sobrien : "+S" (addr), "+c" (count) 28988118Sjhb : "d" (port)); 2904Srgrimes} 2914Srgrimes 2924479Sbdestatic __inline void 293201369Sobrienoutsl(u_int port, const void *addr, size_t count) 2944Srgrimes{ 2954479Sbde __asm __volatile("cld; rep; outsl" 296201369Sobrien : "+S" (addr), "+c" (count) 29788118Sjhb : "d" (port)); 2984Srgrimes} 2994Srgrimes 3004479Sbdestatic __inline void 3014479Sbdeoutw(u_int port, u_short data) 3024Srgrimes{ 303220629Sjkim __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port)); 3044Srgrimes} 3054Srgrimes 306237855Salcstatic __inline u_long 307237855Salcpopcntq(u_long mask) 308237855Salc{ 309237855Salc u_long result; 310237855Salc 311237855Salc __asm __volatile("popcntq %1,%0" : "=r" (result) : "rm" (mask)); 312237855Salc return (result); 313237855Salc} 314237855Salc 31597114Sjhbstatic __inline void 316238972Skiblfence(void) 317238972Skib{ 318238972Skib 319238972Skib __asm __volatile("lfence" : : : "memory"); 320238972Skib} 321238972Skib 322238972Skibstatic __inline void 323195820Skibmfence(void) 324195820Skib{ 325195820Skib 326197647Savg __asm __volatile("mfence" : : : "memory"); 327195820Skib} 328195820Skib 329195820Skibstatic __inline void 330313150Skibsfence(void) 331313150Skib{ 332313150Skib 333313150Skib __asm __volatile("sfence" : : : "memory"); 334313150Skib} 335313150Skib 336313150Skibstatic __inline void 33797139Sjhbia32_pause(void) 33897114Sjhb{ 33997114Sjhb __asm __volatile("pause"); 34097114Sjhb} 34197114Sjhb 342114349Speterstatic __inline u_long 343114349Speterread_rflags(void) 3444479Sbde{ 345114349Speter u_long rf; 3464479Sbde 347114349Speter __asm __volatile("pushfq; popq %0" : "=r" (rf)); 348114349Speter return (rf); 3494479Sbde} 3504479Sbde 351220628Sjkimstatic __inline uint64_t 35215122Sbderdmsr(u_int msr) 3534479Sbde{ 354220628Sjkim uint32_t low, high; 35515122Sbde 356114349Speter __asm __volatile("rdmsr" : "=a" (low), "=d" (high) : "c" (msr)); 357220628Sjkim return (low | ((uint64_t)high << 32)); 3584479Sbde} 3594479Sbde 360220628Sjkimstatic __inline uint64_t 36115122Sbderdpmc(u_int pmc) 36214825Swollman{ 363220628Sjkim uint32_t low, high; 36415122Sbde 365114349Speter __asm __volatile("rdpmc" : "=a" (low), "=d" (high) : "c" (pmc)); 366220628Sjkim return (low | ((uint64_t)high << 32)); 36714825Swollman} 36814825Swollman 369220628Sjkimstatic __inline uint64_t 37014825Swollmanrdtsc(void) 37114825Swollman{ 372220628Sjkim uint32_t low, high; 37315122Sbde 374114349Speter __asm __volatile("rdtsc" : "=a" (low), "=d" (high)); 375220628Sjkim return (low | ((uint64_t)high << 32)); 37614825Swollman} 37714825Swollman 378220631Sjkimstatic __inline uint32_t 379220631Sjkimrdtsc32(void) 380220631Sjkim{ 381220631Sjkim uint32_t rv; 382220631Sjkim 383220631Sjkim __asm __volatile("rdtsc" : "=a" (rv) : : "edx"); 384220631Sjkim return (rv); 385220631Sjkim} 386220631Sjkim 38715122Sbdestatic __inline void 38824112Skatowbinvd(void) 38924112Skato{ 39024112Skato __asm __volatile("wbinvd"); 39124112Skato} 39224112Skato 39324112Skatostatic __inline void 394114349Speterwrite_rflags(u_long rf) 39514825Swollman{ 396114349Speter __asm __volatile("pushq %0; popfq" : : "r" (rf)); 39714825Swollman} 39814825Swollman 39914825Swollmanstatic __inline void 400220628Sjkimwrmsr(u_int msr, uint64_t newval) 40114825Swollman{ 402220628Sjkim uint32_t low, high; 403114349Speter 404114349Speter low = newval; 405114349Speter high = newval >> 32; 406114349Speter __asm __volatile("wrmsr" : : "a" (low), "d" (high), "c" (msr)); 40714825Swollman} 40814825Swollman 40999862Speterstatic __inline void 410114349Speterload_cr0(u_long data) 41199862Speter{ 41299862Speter 413114349Speter __asm __volatile("movq %0,%%cr0" : : "r" (data)); 41499862Speter} 41599862Speter 416114349Speterstatic __inline u_long 41799862Speterrcr0(void) 41899862Speter{ 419114349Speter u_long data; 42099862Speter 421114349Speter __asm __volatile("movq %%cr0,%0" : "=r" (data)); 42299862Speter return (data); 42399862Speter} 42499862Speter 425114349Speterstatic __inline u_long 42699862Speterrcr2(void) 42799862Speter{ 428114349Speter u_long data; 42999862Speter 430114349Speter __asm __volatile("movq %%cr2,%0" : "=r" (data)); 43199862Speter return (data); 43299862Speter} 43399862Speter 43499862Speterstatic __inline void 435114349Speterload_cr3(u_long data) 43699862Speter{ 43799862Speter 438114349Speter __asm __volatile("movq %0,%%cr3" : : "r" (data) : "memory"); 43999862Speter} 44099862Speter 441114349Speterstatic __inline u_long 44299862Speterrcr3(void) 44399862Speter{ 444114349Speter u_long data; 44599862Speter 446114349Speter __asm __volatile("movq %%cr3,%0" : "=r" (data)); 44799862Speter return (data); 44899862Speter} 44999862Speter 45099862Speterstatic __inline void 451114349Speterload_cr4(u_long data) 45299862Speter{ 453114349Speter __asm __volatile("movq %0,%%cr4" : : "r" (data)); 45499862Speter} 45599862Speter 456114349Speterstatic __inline u_long 45799862Speterrcr4(void) 45899862Speter{ 459114349Speter u_long data; 46099862Speter 461114349Speter __asm __volatile("movq %%cr4,%0" : "=r" (data)); 46299862Speter return (data); 46399862Speter} 46499862Speter 465238142Sjhbstatic __inline u_long 466238142Sjhbrxcr(u_int reg) 467238142Sjhb{ 468238142Sjhb u_int low, high; 469238142Sjhb 470238142Sjhb __asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg)); 471238142Sjhb return (low | ((uint64_t)high << 32)); 472238142Sjhb} 473238142Sjhb 474238142Sjhbstatic __inline void 475238142Sjhbload_xcr(u_int reg, u_long val) 476238142Sjhb{ 477238142Sjhb u_int low, high; 478238142Sjhb 479238142Sjhb low = val; 480238142Sjhb high = val >> 32; 481238142Sjhb __asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high)); 482238142Sjhb} 483238142Sjhb 48499862Speter/* 48599862Speter * Global TLB flush (except for thise for pages marked PG_G) 48699862Speter */ 48799862Speterstatic __inline void 48899862Speterinvltlb(void) 48999862Speter{ 49099862Speter 49199862Speter load_cr3(rcr3()); 49299862Speter} 49399862Speter 494255331Sgibbs#ifndef CR4_PGE 495255331Sgibbs#define CR4_PGE 0x00000080 /* Page global enable */ 496255331Sgibbs#endif 497255331Sgibbs 49899862Speter/* 499255331Sgibbs * Perform the guaranteed invalidation of all TLB entries. This 500255331Sgibbs * includes the global entries, and entries in all PCIDs, not only the 501255331Sgibbs * current context. The function works both on non-PCID CPUs and CPUs 502255331Sgibbs * with the PCID turned off or on. See IA-32 SDM Vol. 3a 4.10.4.1 503255331Sgibbs * Operations that Invalidate TLBs and Paging-Structure Caches. 504255331Sgibbs */ 505255331Sgibbsstatic __inline void 506255331Sgibbsinvltlb_globpcid(void) 507255331Sgibbs{ 508255331Sgibbs uint64_t cr4; 509255331Sgibbs 510255331Sgibbs cr4 = rcr4(); 511255331Sgibbs load_cr4(cr4 & ~CR4_PGE); 512255331Sgibbs /* 513255331Sgibbs * Although preemption at this point could be detrimental to 514255331Sgibbs * performance, it would not lead to an error. PG_G is simply 515255331Sgibbs * ignored if CR4.PGE is clear. Moreover, in case this block 516255331Sgibbs * is re-entered, the load_cr4() either above or below will 517255331Sgibbs * modify CR4.PGE flushing the TLB. 518255331Sgibbs */ 519255331Sgibbs load_cr4(cr4 | CR4_PGE); 520255331Sgibbs} 521255331Sgibbs 522255331Sgibbs/* 52399862Speter * TLB flush for an individual page (even if it has PG_G). 52499862Speter * Only works on 486+ CPUs (i386 does not have PG_G). 52599862Speter */ 52699862Speterstatic __inline void 527114349Speterinvlpg(u_long addr) 52899862Speter{ 52999862Speter 53099862Speter __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 53199862Speter} 53299862Speter 533255058Skib#define INVPCID_ADDR 0 534255058Skib#define INVPCID_CTX 1 535255058Skib#define INVPCID_CTXGLOB 2 536255058Skib#define INVPCID_ALLCTX 3 537255058Skib 538255058Skibstruct invpcid_descr { 539255058Skib uint64_t pcid:12 __packed; 540255058Skib uint64_t pad:52 __packed; 541255058Skib uint64_t addr; 542255058Skib} __packed; 543255058Skib 544255058Skibstatic __inline void 545255058Skibinvpcid(struct invpcid_descr *d, int type) 546255058Skib{ 547255058Skib 548255058Skib /* invpcid (%rdx),%rax */ 549255058Skib __asm __volatile(".byte 0x66,0x0f,0x38,0x82,0x02" 550255058Skib : : "d" (d), "a" ((u_long)type) : "memory"); 551255058Skib} 552255058Skib 553212177Srdivackystatic __inline u_short 55446129Sluoqirfs(void) 55546129Sluoqi{ 556212177Srdivacky u_short sel; 557212177Srdivacky __asm __volatile("movw %%fs,%0" : "=rm" (sel)); 55846129Sluoqi return (sel); 55946129Sluoqi} 56046129Sluoqi 561212177Srdivackystatic __inline u_short 56246129Sluoqirgs(void) 56346129Sluoqi{ 564212177Srdivacky u_short sel; 565212177Srdivacky __asm __volatile("movw %%gs,%0" : "=rm" (sel)); 56646129Sluoqi return (sel); 56746129Sluoqi} 56846129Sluoqi 569212177Srdivackystatic __inline u_short 570127973Speterrss(void) 571127973Speter{ 572212177Srdivacky u_short sel; 573212177Srdivacky __asm __volatile("movw %%ss,%0" : "=rm" (sel)); 574127973Speter return (sel); 575127973Speter} 576127973Speter 57746129Sluoqistatic __inline void 578212177Srdivackyload_ds(u_short sel) 579114987Speter{ 580212177Srdivacky __asm __volatile("movw %0,%%ds" : : "rm" (sel)); 581114987Speter} 582114987Speter 583114987Speterstatic __inline void 584212177Srdivackyload_es(u_short sel) 585114987Speter{ 586212177Srdivacky __asm __volatile("movw %0,%%es" : : "rm" (sel)); 587114987Speter} 588114987Speter 589197647Savgstatic __inline void 590223796Sjkimcpu_monitor(const void *addr, u_long extensions, u_int hints) 591178299Sjeff{ 592223796Sjkim 593223796Sjkim __asm __volatile("monitor" 594223796Sjkim : : "a" (addr), "c" (extensions), "d" (hints)); 595178299Sjeff} 596178299Sjeff 597197647Savgstatic __inline void 598223796Sjkimcpu_mwait(u_long extensions, u_int hints) 599178299Sjeff{ 600223796Sjkim 601223796Sjkim __asm __volatile("mwait" : : "a" (hints), "c" (extensions)); 602178299Sjeff} 603178299Sjeff 604115006Speter#ifdef _KERNEL 605115006Speter/* This is defined in <machine/specialreg.h> but is too painful to get to */ 606115006Speter#ifndef MSR_FSBASE 607115006Speter#define MSR_FSBASE 0xc0000100 608115006Speter#endif 609114987Speterstatic __inline void 610212177Srdivackyload_fs(u_short sel) 61146129Sluoqi{ 612115006Speter /* Preserve the fsbase value across the selector load */ 613212177Srdivacky __asm __volatile("rdmsr; movw %0,%%fs; wrmsr" 614190817Sed : : "rm" (sel), "c" (MSR_FSBASE) : "eax", "edx"); 615115006Speter} 616115006Speter 617115006Speter#ifndef MSR_GSBASE 618115006Speter#define MSR_GSBASE 0xc0000101 619115006Speter#endif 620115006Speterstatic __inline void 621212177Srdivackyload_gs(u_short sel) 622115006Speter{ 623115006Speter /* 624115006Speter * Preserve the gsbase value across the selector load. 625115006Speter * Note that we have to disable interrupts because the gsbase 626115006Speter * being trashed happens to be the kernel gsbase at the time. 627115006Speter */ 628212177Srdivacky __asm __volatile("pushfq; cli; rdmsr; movw %0,%%gs; wrmsr; popfq" 629190817Sed : : "rm" (sel), "c" (MSR_GSBASE) : "eax", "edx"); 630115006Speter} 631115006Speter#else 632115006Speter/* Usable by userland */ 633115006Speterstatic __inline void 634212177Srdivackyload_fs(u_short sel) 635115006Speter{ 636212177Srdivacky __asm __volatile("movw %0,%%fs" : : "rm" (sel)); 63746129Sluoqi} 63846129Sluoqi 63946129Sluoqistatic __inline void 640212177Srdivackyload_gs(u_short sel) 64146129Sluoqi{ 642212177Srdivacky __asm __volatile("movw %0,%%gs" : : "rm" (sel)); 64346129Sluoqi} 644115006Speter#endif 64546129Sluoqi 646103778Speterstatic __inline void 647103778Speterlidt(struct region_descriptor *addr) 648103778Speter{ 649103778Speter __asm __volatile("lidt (%0)" : : "r" (addr)); 650103778Speter} 651103778Speter 652103778Speterstatic __inline void 653103778Speterlldt(u_short sel) 654103778Speter{ 655103778Speter __asm __volatile("lldt %0" : : "r" (sel)); 656103778Speter} 657103778Speter 658103778Speterstatic __inline void 659103778Speterltr(u_short sel) 660103778Speter{ 661103778Speter __asm __volatile("ltr %0" : : "r" (sel)); 662103778Speter} 663103778Speter 664220628Sjkimstatic __inline uint64_t 665125175Speterrdr0(void) 666125175Speter{ 667220628Sjkim uint64_t data; 668125175Speter __asm __volatile("movq %%dr0,%0" : "=r" (data)); 669125175Speter return (data); 670125175Speter} 671125175Speter 672125175Speterstatic __inline void 673220628Sjkimload_dr0(uint64_t dr0) 674125175Speter{ 675125175Speter __asm __volatile("movq %0,%%dr0" : : "r" (dr0)); 676125175Speter} 677125175Speter 678220628Sjkimstatic __inline uint64_t 679125175Speterrdr1(void) 680125175Speter{ 681220628Sjkim uint64_t data; 682125175Speter __asm __volatile("movq %%dr1,%0" : "=r" (data)); 683125175Speter return (data); 684125175Speter} 685125175Speter 686125175Speterstatic __inline void 687220628Sjkimload_dr1(uint64_t dr1) 688125175Speter{ 689125175Speter __asm __volatile("movq %0,%%dr1" : : "r" (dr1)); 690125175Speter} 691125175Speter 692220628Sjkimstatic __inline uint64_t 693125175Speterrdr2(void) 694125175Speter{ 695220628Sjkim uint64_t data; 696125175Speter __asm __volatile("movq %%dr2,%0" : "=r" (data)); 697125175Speter return (data); 698125175Speter} 699125175Speter 700125175Speterstatic __inline void 701220628Sjkimload_dr2(uint64_t dr2) 702125175Speter{ 703125175Speter __asm __volatile("movq %0,%%dr2" : : "r" (dr2)); 704125175Speter} 705125175Speter 706220628Sjkimstatic __inline uint64_t 707125175Speterrdr3(void) 708125175Speter{ 709220628Sjkim uint64_t data; 710125175Speter __asm __volatile("movq %%dr3,%0" : "=r" (data)); 711125175Speter return (data); 712125175Speter} 713125175Speter 714125175Speterstatic __inline void 715220628Sjkimload_dr3(uint64_t dr3) 716125175Speter{ 717125175Speter __asm __volatile("movq %0,%%dr3" : : "r" (dr3)); 718125175Speter} 719125175Speter 720220628Sjkimstatic __inline uint64_t 721125175Speterrdr4(void) 722125175Speter{ 723220628Sjkim uint64_t data; 724125175Speter __asm __volatile("movq %%dr4,%0" : "=r" (data)); 725125175Speter return (data); 726125175Speter} 727125175Speter 728125175Speterstatic __inline void 729220628Sjkimload_dr4(uint64_t dr4) 730125175Speter{ 731125175Speter __asm __volatile("movq %0,%%dr4" : : "r" (dr4)); 732125175Speter} 733125175Speter 734220628Sjkimstatic __inline uint64_t 735125175Speterrdr5(void) 736125175Speter{ 737220628Sjkim uint64_t data; 738125175Speter __asm __volatile("movq %%dr5,%0" : "=r" (data)); 739125175Speter return (data); 740125175Speter} 741125175Speter 742125175Speterstatic __inline void 743220628Sjkimload_dr5(uint64_t dr5) 744125175Speter{ 745125175Speter __asm __volatile("movq %0,%%dr5" : : "r" (dr5)); 746125175Speter} 747125175Speter 748220628Sjkimstatic __inline uint64_t 749125175Speterrdr6(void) 750125175Speter{ 751220628Sjkim uint64_t data; 752125175Speter __asm __volatile("movq %%dr6,%0" : "=r" (data)); 753125175Speter return (data); 754125175Speter} 755125175Speter 756125175Speterstatic __inline void 757220628Sjkimload_dr6(uint64_t dr6) 758125175Speter{ 759125175Speter __asm __volatile("movq %0,%%dr6" : : "r" (dr6)); 760125175Speter} 761125175Speter 762220628Sjkimstatic __inline uint64_t 763125175Speterrdr7(void) 764125175Speter{ 765220628Sjkim uint64_t data; 766125175Speter __asm __volatile("movq %%dr7,%0" : "=r" (data)); 767125175Speter return (data); 768125175Speter} 769125175Speter 770125175Speterstatic __inline void 771220628Sjkimload_dr7(uint64_t dr7) 772125175Speter{ 773125175Speter __asm __volatile("movq %0,%%dr7" : : "r" (dr7)); 774125175Speter} 775125175Speter 77692860Simpstatic __inline register_t 77792860Simpintr_disable(void) 77892860Simp{ 779114349Speter register_t rflags; 78092860Simp 781114349Speter rflags = read_rflags(); 78292860Simp disable_intr(); 783114349Speter return (rflags); 78492860Simp} 78592860Simp 78692860Simpstatic __inline void 787114349Speterintr_restore(register_t rflags) 78892860Simp{ 789114349Speter write_rflags(rflags); 79092860Simp} 79192860Simp 792143063Sjoerg#else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */ 7934Srgrimes 79492819Simpint breakpoint(void); 79592819Simpu_int bsfl(u_int mask); 79692819Simpu_int bsrl(u_int mask); 797238311Sjhbvoid clflush(u_long addr); 798238311Sjhbvoid clts(void); 799238311Sjhbvoid cpuid_count(u_int ax, u_int cx, u_int *p); 80092819Simpvoid disable_intr(void); 80192819Simpvoid do_cpuid(u_int ax, u_int *p); 80292819Simpvoid enable_intr(void); 803103749Smarkmvoid halt(void); 804126734Spetervoid ia32_pause(void); 80592819Simpu_char inb(u_int port); 80692819Simpu_int inl(u_int port); 807201369Sobrienvoid insb(u_int port, void *addr, size_t count); 808201369Sobrienvoid insl(u_int port, void *addr, size_t count); 809201369Sobrienvoid insw(u_int port, void *addr, size_t count); 810126734Speterregister_t intr_disable(void); 811126734Spetervoid intr_restore(register_t rf); 81292819Simpvoid invd(void); 81392819Simpvoid invlpg(u_int addr); 81492819Simpvoid invltlb(void); 81592819Simpu_short inw(u_int port); 816126734Spetervoid lidt(struct region_descriptor *addr); 817126734Spetervoid lldt(u_short sel); 818126734Spetervoid load_cr0(u_long cr0); 819126734Spetervoid load_cr3(u_long cr3); 820126734Spetervoid load_cr4(u_long cr4); 821220628Sjkimvoid load_dr0(uint64_t dr0); 822220628Sjkimvoid load_dr1(uint64_t dr1); 823220628Sjkimvoid load_dr2(uint64_t dr2); 824220628Sjkimvoid load_dr3(uint64_t dr3); 825220628Sjkimvoid load_dr4(uint64_t dr4); 826220628Sjkimvoid load_dr5(uint64_t dr5); 827220628Sjkimvoid load_dr6(uint64_t dr6); 828220628Sjkimvoid load_dr7(uint64_t dr7); 829212177Srdivackyvoid load_fs(u_short sel); 830212177Srdivackyvoid load_gs(u_short sel); 831103778Spetervoid ltr(u_short sel); 83292819Simpvoid outb(u_int port, u_char data); 83392819Simpvoid outl(u_int port, u_int data); 834201369Sobrienvoid outsb(u_int port, const void *addr, size_t count); 835201369Sobrienvoid outsl(u_int port, const void *addr, size_t count); 836201369Sobrienvoid outsw(u_int port, const void *addr, size_t count); 83792819Simpvoid outw(u_int port, u_short data); 838126734Speteru_long rcr0(void); 839126734Speteru_long rcr2(void); 840126734Speteru_long rcr3(void); 841126734Speteru_long rcr4(void); 842220628Sjkimuint64_t rdmsr(u_int msr); 843220628Sjkimuint64_t rdpmc(u_int pmc); 844220628Sjkimuint64_t rdr0(void); 845220628Sjkimuint64_t rdr1(void); 846220628Sjkimuint64_t rdr2(void); 847220628Sjkimuint64_t rdr3(void); 848220628Sjkimuint64_t rdr4(void); 849220628Sjkimuint64_t rdr5(void); 850220628Sjkimuint64_t rdr6(void); 851220628Sjkimuint64_t rdr7(void); 852220628Sjkimuint64_t rdtsc(void); 853232227Sjhbu_long read_rflags(void); 854126734Speteru_int rfs(void); 855126734Speteru_int rgs(void); 85692819Simpvoid wbinvd(void); 857114349Spetervoid write_rflags(u_int rf); 858220628Sjkimvoid wrmsr(u_int msr, uint64_t newval); 8594Srgrimes 860143063Sjoerg#endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */ 8614Srgrimes 862125175Spetervoid reset_dbregs(void); 86393264Sdillon 864181430Sstas#ifdef _KERNEL 865181430Sstasint rdmsr_safe(u_int msr, uint64_t *val); 866181430Sstasint wrmsr_safe(u_int msr, uint64_t newval); 867181430Sstas#endif 868181430Sstas 8694479Sbde#endif /* !_MACHINE_CPUFUNC_H_ */ 870