cpufunc.h revision 123181
1/*- 2 * Copyright (c) 2003 Peter Wemm. 3 * Copyright (c) 1993 The Regents of the University of California. 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. All advertising materials mentioning features or use of this software 15 * must display the following acknowledgement: 16 * This product includes software developed by the University of 17 * California, Berkeley and its contributors. 18 * 4. Neither the name of the University nor the names of its contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 * 34 * $FreeBSD: head/sys/amd64/include/cpufunc.h 123181 2003-12-06 23:22:43Z peter $ 35 */ 36 37/* 38 * Functions to provide access to special i386 instructions. 39 * This in included in sys/systm.h, and that file should be 40 * used in preference to this. 41 */ 42 43#ifndef _MACHINE_CPUFUNC_H_ 44#define _MACHINE_CPUFUNC_H_ 45 46#include <sys/cdefs.h> 47#include <machine/psl.h> 48 49struct thread; 50struct region_descriptor; 51 52__BEGIN_DECLS 53#define readb(va) (*(volatile u_int8_t *) (va)) 54#define readw(va) (*(volatile u_int16_t *) (va)) 55#define readl(va) (*(volatile u_int32_t *) (va)) 56#define readq(va) (*(volatile u_int64_t *) (va)) 57 58#define writeb(va, d) (*(volatile u_int8_t *) (va) = (d)) 59#define writew(va, d) (*(volatile u_int16_t *) (va) = (d)) 60#define writel(va, d) (*(volatile u_int32_t *) (va) = (d)) 61#define writeq(va, d) (*(volatile u_int64_t *) (va) = (d)) 62 63#ifdef __GNUC__ 64 65static __inline void 66breakpoint(void) 67{ 68 __asm __volatile("int $3"); 69} 70 71static __inline u_int 72bsfl(u_int mask) 73{ 74 u_int result; 75 76 __asm __volatile("bsfl %1,%0" : "=r" (result) : "rm" (mask)); 77 return (result); 78} 79 80static __inline u_long 81bsfq(u_long mask) 82{ 83 u_long result; 84 85 __asm __volatile("bsfq %1,%0" : "=r" (result) : "rm" (mask)); 86 return (result); 87} 88 89static __inline u_int 90bsrl(u_int mask) 91{ 92 u_int result; 93 94 __asm __volatile("bsrl %1,%0" : "=r" (result) : "rm" (mask)); 95 return (result); 96} 97 98static __inline u_long 99bsrq(u_long mask) 100{ 101 u_long result; 102 103 __asm __volatile("bsrq %1,%0" : "=r" (result) : "rm" (mask)); 104 return (result); 105} 106 107static __inline void 108disable_intr(void) 109{ 110 __asm __volatile("cli" : : : "memory"); 111} 112 113static __inline void 114do_cpuid(u_int ax, u_int *p) 115{ 116 __asm __volatile("cpuid" 117 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 118 : "0" (ax)); 119} 120 121static __inline void 122enable_intr(void) 123{ 124 __asm __volatile("sti"); 125} 126 127#define HAVE_INLINE_FFS 128 129static __inline int 130ffs(int mask) 131{ 132#if 0 133 /* 134 * Note that gcc-2's builtin ffs would be used if we didn't declare 135 * this inline or turn off the builtin. The builtin is faster but 136 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later 137 * versions. 138 */ 139 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1); 140#else 141 /* Actually, the above is way out of date. The builtins use cmov etc */ 142 return (__builtin_ffs(mask)); 143#endif 144} 145 146#define HAVE_INLINE_FFSL 147 148static __inline int 149ffsl(long mask) 150{ 151 return (mask == 0 ? mask : (int)bsfq((u_long)mask) + 1); 152} 153 154#define HAVE_INLINE_FLS 155 156static __inline int 157fls(int mask) 158{ 159 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1); 160} 161 162#define HAVE_INLINE_FLSL 163 164static __inline int 165flsl(long mask) 166{ 167 return (mask == 0 ? mask : (int)bsrq((u_long)mask) + 1); 168} 169 170static __inline void 171halt(void) 172{ 173 __asm __volatile("hlt"); 174} 175 176#if __GNUC__ < 2 177 178#define inb(port) inbv(port) 179#define outb(port, data) outbv(port, data) 180 181#else /* __GNUC >= 2 */ 182 183/* 184 * The following complications are to get around gcc not having a 185 * constraint letter for the range 0..255. We still put "d" in the 186 * constraint because "i" isn't a valid constraint when the port 187 * isn't constant. This only matters for -O0 because otherwise 188 * the non-working version gets optimized away. 189 * 190 * Use an expression-statement instead of a conditional expression 191 * because gcc-2.6.0 would promote the operands of the conditional 192 * and produce poor code for "if ((inb(var) & const1) == const2)". 193 * 194 * The unnecessary test `(port) < 0x10000' is to generate a warning if 195 * the `port' has type u_short or smaller. Such types are pessimal. 196 * This actually only works for signed types. The range check is 197 * careful to avoid generating warnings. 198 */ 199#define inb(port) __extension__ ({ \ 200 u_char _data; \ 201 if (__builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \ 202 && (port) < 0x10000) \ 203 _data = inbc(port); \ 204 else \ 205 _data = inbv(port); \ 206 _data; }) 207 208#define outb(port, data) ( \ 209 __builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \ 210 && (port) < 0x10000 \ 211 ? outbc(port, data) : outbv(port, data)) 212 213static __inline u_char 214inbc(u_int port) 215{ 216 u_char data; 217 218 __asm __volatile("inb %1,%0" : "=a" (data) : "id" ((u_short)(port))); 219 return (data); 220} 221 222static __inline void 223outbc(u_int port, u_char data) 224{ 225 __asm __volatile("outb %0,%1" : : "a" (data), "id" ((u_short)(port))); 226} 227 228#endif /* __GNUC <= 2 */ 229 230static __inline u_char 231inbv(u_int port) 232{ 233 u_char data; 234 /* 235 * We use %%dx and not %1 here because i/o is done at %dx and not at 236 * %edx, while gcc generates inferior code (movw instead of movl) 237 * if we tell it to load (u_short) port. 238 */ 239 __asm __volatile("inb %%dx,%0" : "=a" (data) : "d" (port)); 240 return (data); 241} 242 243static __inline u_int 244inl(u_int port) 245{ 246 u_int data; 247 248 __asm __volatile("inl %%dx,%0" : "=a" (data) : "d" (port)); 249 return (data); 250} 251 252static __inline void 253insb(u_int port, void *addr, size_t cnt) 254{ 255 __asm __volatile("cld; rep; insb" 256 : "+D" (addr), "+c" (cnt) 257 : "d" (port) 258 : "memory"); 259} 260 261static __inline void 262insw(u_int port, void *addr, size_t cnt) 263{ 264 __asm __volatile("cld; rep; insw" 265 : "+D" (addr), "+c" (cnt) 266 : "d" (port) 267 : "memory"); 268} 269 270static __inline void 271insl(u_int port, void *addr, size_t cnt) 272{ 273 __asm __volatile("cld; rep; insl" 274 : "+D" (addr), "+c" (cnt) 275 : "d" (port) 276 : "memory"); 277} 278 279static __inline void 280invd(void) 281{ 282 __asm __volatile("invd"); 283} 284 285static __inline u_short 286inw(u_int port) 287{ 288 u_short data; 289 290 __asm __volatile("inw %%dx,%0" : "=a" (data) : "d" (port)); 291 return (data); 292} 293 294static __inline void 295outbv(u_int port, u_char data) 296{ 297 u_char al; 298 /* 299 * Use an unnecessary assignment to help gcc's register allocator. 300 * This make a large difference for gcc-1.40 and a tiny difference 301 * for gcc-2.6.0. For gcc-1.40, al had to be ``asm("ax")'' for 302 * best results. gcc-2.6.0 can't handle this. 303 */ 304 al = data; 305 __asm __volatile("outb %0,%%dx" : : "a" (al), "d" (port)); 306} 307 308static __inline void 309outl(u_int port, u_int data) 310{ 311 /* 312 * outl() and outw() aren't used much so we haven't looked at 313 * possible micro-optimizations such as the unnecessary 314 * assignment for them. 315 */ 316 __asm __volatile("outl %0,%%dx" : : "a" (data), "d" (port)); 317} 318 319static __inline void 320outsb(u_int port, const void *addr, size_t cnt) 321{ 322 __asm __volatile("cld; rep; outsb" 323 : "+S" (addr), "+c" (cnt) 324 : "d" (port)); 325} 326 327static __inline void 328outsw(u_int port, const void *addr, size_t cnt) 329{ 330 __asm __volatile("cld; rep; outsw" 331 : "+S" (addr), "+c" (cnt) 332 : "d" (port)); 333} 334 335static __inline void 336outsl(u_int port, const void *addr, size_t cnt) 337{ 338 __asm __volatile("cld; rep; outsl" 339 : "+S" (addr), "+c" (cnt) 340 : "d" (port)); 341} 342 343static __inline void 344outw(u_int port, u_short data) 345{ 346 __asm __volatile("outw %0,%%dx" : : "a" (data), "d" (port)); 347} 348 349static __inline void 350ia32_pause(void) 351{ 352 __asm __volatile("pause"); 353} 354 355static __inline u_long 356read_rflags(void) 357{ 358 u_long rf; 359 360 __asm __volatile("pushfq; popq %0" : "=r" (rf)); 361 return (rf); 362} 363 364static __inline u_int64_t 365rdmsr(u_int msr) 366{ 367 u_int32_t low, high; 368 369 __asm __volatile("rdmsr" : "=a" (low), "=d" (high) : "c" (msr)); 370 return (low | ((u_int64_t)high << 32)); 371} 372 373static __inline u_int64_t 374rdpmc(u_int pmc) 375{ 376 u_int32_t low, high; 377 378 __asm __volatile("rdpmc" : "=a" (low), "=d" (high) : "c" (pmc)); 379 return (low | ((u_int64_t)high << 32)); 380} 381 382static __inline u_int64_t 383rdtsc(void) 384{ 385 u_int32_t low, high; 386 387 __asm __volatile("rdtsc" : "=a" (low), "=d" (high)); 388 return (low | ((u_int64_t)high << 32)); 389} 390 391static __inline void 392wbinvd(void) 393{ 394 __asm __volatile("wbinvd"); 395} 396 397static __inline void 398write_rflags(u_long rf) 399{ 400 __asm __volatile("pushq %0; popfq" : : "r" (rf)); 401} 402 403static __inline void 404wrmsr(u_int msr, u_int64_t newval) 405{ 406 u_int32_t low, high; 407 408 low = newval; 409 high = newval >> 32; 410 __asm __volatile("wrmsr" : : "a" (low), "d" (high), "c" (msr)); 411} 412 413static __inline void 414load_cr0(u_long data) 415{ 416 417 __asm __volatile("movq %0,%%cr0" : : "r" (data)); 418} 419 420static __inline u_long 421rcr0(void) 422{ 423 u_long data; 424 425 __asm __volatile("movq %%cr0,%0" : "=r" (data)); 426 return (data); 427} 428 429static __inline u_long 430rcr2(void) 431{ 432 u_long data; 433 434 __asm __volatile("movq %%cr2,%0" : "=r" (data)); 435 return (data); 436} 437 438static __inline void 439load_cr3(u_long data) 440{ 441 442 __asm __volatile("movq %0,%%cr3" : : "r" (data) : "memory"); 443} 444 445static __inline u_long 446rcr3(void) 447{ 448 u_long data; 449 450 __asm __volatile("movq %%cr3,%0" : "=r" (data)); 451 return (data); 452} 453 454static __inline void 455load_cr4(u_long data) 456{ 457 __asm __volatile("movq %0,%%cr4" : : "r" (data)); 458} 459 460static __inline u_long 461rcr4(void) 462{ 463 u_long data; 464 465 __asm __volatile("movq %%cr4,%0" : "=r" (data)); 466 return (data); 467} 468 469/* 470 * Global TLB flush (except for thise for pages marked PG_G) 471 */ 472static __inline void 473invltlb(void) 474{ 475 476 load_cr3(rcr3()); 477} 478 479/* 480 * TLB flush for an individual page (even if it has PG_G). 481 * Only works on 486+ CPUs (i386 does not have PG_G). 482 */ 483static __inline void 484invlpg(u_long addr) 485{ 486 487 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 488} 489 490static __inline u_int 491rfs(void) 492{ 493 u_int sel; 494 __asm __volatile("movl %%fs,%0" : "=rm" (sel)); 495 return (sel); 496} 497 498static __inline u_int 499rgs(void) 500{ 501 u_int sel; 502 __asm __volatile("movl %%gs,%0" : "=rm" (sel)); 503 return (sel); 504} 505 506static __inline void 507load_ds(u_int sel) 508{ 509 __asm __volatile("movl %0,%%ds" : : "rm" (sel)); 510} 511 512static __inline void 513load_es(u_int sel) 514{ 515 __asm __volatile("movl %0,%%es" : : "rm" (sel)); 516} 517 518#ifdef _KERNEL 519/* This is defined in <machine/specialreg.h> but is too painful to get to */ 520#ifndef MSR_FSBASE 521#define MSR_FSBASE 0xc0000100 522#endif 523static __inline void 524load_fs(u_int sel) 525{ 526 register u_int32_t fsbase __asm("ecx"); 527 528 /* Preserve the fsbase value across the selector load */ 529 fsbase = MSR_FSBASE; 530 __asm __volatile("rdmsr; movl %0,%%fs; wrmsr" 531 : : "rm" (sel), "c" (fsbase) : "eax", "edx"); 532} 533 534#ifndef MSR_GSBASE 535#define MSR_GSBASE 0xc0000101 536#endif 537static __inline void 538load_gs(u_int sel) 539{ 540 register u_int32_t gsbase __asm("ecx"); 541 542 /* 543 * Preserve the gsbase value across the selector load. 544 * Note that we have to disable interrupts because the gsbase 545 * being trashed happens to be the kernel gsbase at the time. 546 */ 547 gsbase = MSR_GSBASE; 548 __asm __volatile("pushfq; cli; rdmsr; movl %0,%%gs; wrmsr; popfq" 549 : : "rm" (sel), "c" (gsbase) : "eax", "edx"); 550} 551#else 552/* Usable by userland */ 553static __inline void 554load_fs(u_int sel) 555{ 556 __asm __volatile("movl %0,%%fs" : : "rm" (sel)); 557} 558 559static __inline void 560load_gs(u_int sel) 561{ 562 __asm __volatile("movl %0,%%gs" : : "rm" (sel)); 563} 564#endif 565 566/* void lidt(struct region_descriptor *addr); */ 567static __inline void 568lidt(struct region_descriptor *addr) 569{ 570 __asm __volatile("lidt (%0)" : : "r" (addr)); 571} 572 573/* void lldt(u_short sel); */ 574static __inline void 575lldt(u_short sel) 576{ 577 __asm __volatile("lldt %0" : : "r" (sel)); 578} 579 580/* void ltr(u_short sel); */ 581static __inline void 582ltr(u_short sel) 583{ 584 __asm __volatile("ltr %0" : : "r" (sel)); 585} 586 587static __inline register_t 588intr_disable(void) 589{ 590 register_t rflags; 591 592 rflags = read_rflags(); 593 disable_intr(); 594 return (rflags); 595} 596 597static __inline void 598intr_restore(register_t rflags) 599{ 600 write_rflags(rflags); 601} 602 603#else /* !__GNUC__ */ 604 605int breakpoint(void); 606u_int bsfl(u_int mask); 607u_int bsrl(u_int mask); 608void cpu_invlpg(u_long addr); 609void cpu_invlpg_range(u_long start, u_long end); 610void disable_intr(void); 611void do_cpuid(u_int ax, u_int *p); 612void enable_intr(void); 613void halt(void); 614u_char inb(u_int port); 615u_int inl(u_int port); 616void insb(u_int port, void *addr, size_t cnt); 617void insl(u_int port, void *addr, size_t cnt); 618void insw(u_int port, void *addr, size_t cnt); 619void invd(void); 620void invlpg(u_int addr); 621void invlpg_range(u_int start, u_int end); 622void invltlb(void); 623u_short inw(u_int port); 624void load_cr0(u_int cr0); 625void load_cr3(u_int cr3); 626void load_cr4(u_int cr4); 627void load_fs(u_int sel); 628void load_gs(u_int sel); 629struct region_descriptor; 630void lidt(struct region_descriptor *addr); 631void lldt(u_short sel); 632void ltr(u_short sel); 633void outb(u_int port, u_char data); 634void outl(u_int port, u_int data); 635void outsb(u_int port, void *addr, size_t cnt); 636void outsl(u_int port, void *addr, size_t cnt); 637void outsw(u_int port, void *addr, size_t cnt); 638void outw(u_int port, u_short data); 639void ia32_pause(void); 640u_int rcr0(void); 641u_int rcr2(void); 642u_int rcr3(void); 643u_int rcr4(void); 644u_int rfs(void); 645u_int rgs(void); 646u_int64_t rdmsr(u_int msr); 647u_int64_t rdpmc(u_int pmc); 648u_int64_t rdtsc(void); 649u_int read_rflags(void); 650void wbinvd(void); 651void write_rflags(u_int rf); 652void wrmsr(u_int msr, u_int64_t newval); 653void load_dr7(u_int dr7); 654register_t intr_disable(void); 655void intr_restore(register_t rf); 656 657#endif /* __GNUC__ */ 658 659void reset_dbregs(void); 660 661__END_DECLS 662 663#endif /* !_MACHINE_CPUFUNC_H_ */ 664