cpufunc.h revision 276084
1/*- 2 * Copyright (c) 1993 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 4. Neither the name of the University nor the names of its contributors 14 * may be used to endorse or promote products derived from this software 15 * without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD: stable/10/sys/i386/include/cpufunc.h 276084 2014-12-22 21:32:39Z jhb $ 30 */ 31 32/* 33 * Functions to provide access to special i386 instructions. 34 * This in included in sys/systm.h, and that file should be 35 * used in preference to this. 36 */ 37 38#ifndef _MACHINE_CPUFUNC_H_ 39#define _MACHINE_CPUFUNC_H_ 40 41#ifndef _SYS_CDEFS_H_ 42#error this file needs sys/cdefs.h as a prerequisite 43#endif 44 45#ifdef XEN 46extern void xen_cli(void); 47extern void xen_sti(void); 48extern u_int xen_rcr2(void); 49extern void xen_load_cr3(u_int data); 50extern void xen_tlb_flush(void); 51extern void xen_invlpg(u_int addr); 52extern void write_eflags(u_int eflags); 53extern u_int read_eflags(void); 54#endif 55 56struct region_descriptor; 57 58#define readb(va) (*(volatile uint8_t *) (va)) 59#define readw(va) (*(volatile uint16_t *) (va)) 60#define readl(va) (*(volatile uint32_t *) (va)) 61 62#define writeb(va, d) (*(volatile uint8_t *) (va) = (d)) 63#define writew(va, d) (*(volatile uint16_t *) (va) = (d)) 64#define writel(va, d) (*(volatile uint32_t *) (va) = (d)) 65 66#if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE) 67 68static __inline void 69breakpoint(void) 70{ 71 __asm __volatile("int $3"); 72} 73 74static __inline u_int 75bsfl(u_int mask) 76{ 77 u_int result; 78 79 __asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 80 return (result); 81} 82 83static __inline u_int 84bsrl(u_int mask) 85{ 86 u_int result; 87 88 __asm("bsrl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 89 return (result); 90} 91 92static __inline void 93clflush(u_long addr) 94{ 95 96 __asm __volatile("clflush %0" : : "m" (*(char *)addr)); 97} 98 99static __inline void 100clts(void) 101{ 102 103 __asm __volatile("clts"); 104} 105 106static __inline void 107disable_intr(void) 108{ 109#ifdef XEN 110 xen_cli(); 111#else 112 __asm __volatile("cli" : : : "memory"); 113#endif 114} 115 116static __inline void 117do_cpuid(u_int ax, u_int *p) 118{ 119 __asm __volatile("cpuid" 120 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 121 : "0" (ax)); 122} 123 124static __inline void 125cpuid_count(u_int ax, u_int cx, u_int *p) 126{ 127 __asm __volatile("cpuid" 128 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 129 : "0" (ax), "c" (cx)); 130} 131 132static __inline void 133enable_intr(void) 134{ 135#ifdef XEN 136 xen_sti(); 137#else 138 __asm __volatile("sti"); 139#endif 140} 141 142static __inline void 143cpu_monitor(const void *addr, u_long extensions, u_int hints) 144{ 145 146 __asm __volatile("monitor" 147 : : "a" (addr), "c" (extensions), "d" (hints)); 148} 149 150static __inline void 151cpu_mwait(u_long extensions, u_int hints) 152{ 153 154 __asm __volatile("mwait" : : "a" (hints), "c" (extensions)); 155} 156 157static __inline void 158lfence(void) 159{ 160 161 __asm __volatile("lfence" : : : "memory"); 162} 163 164static __inline void 165mfence(void) 166{ 167 168 __asm __volatile("mfence" : : : "memory"); 169} 170 171#ifdef _KERNEL 172 173#define HAVE_INLINE_FFS 174 175static __inline int 176ffs(int mask) 177{ 178 /* 179 * Note that gcc-2's builtin ffs would be used if we didn't declare 180 * this inline or turn off the builtin. The builtin is faster but 181 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later 182 * versions. 183 */ 184 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1); 185} 186 187#define HAVE_INLINE_FLS 188 189static __inline int 190fls(int mask) 191{ 192 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1); 193} 194 195#endif /* _KERNEL */ 196 197static __inline void 198halt(void) 199{ 200 __asm __volatile("hlt"); 201} 202 203static __inline u_char 204inb(u_int port) 205{ 206 u_char data; 207 208 __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port)); 209 return (data); 210} 211 212static __inline u_int 213inl(u_int port) 214{ 215 u_int data; 216 217 __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port)); 218 return (data); 219} 220 221static __inline void 222insb(u_int port, void *addr, size_t count) 223{ 224 __asm __volatile("cld; rep; insb" 225 : "+D" (addr), "+c" (count) 226 : "d" (port) 227 : "memory"); 228} 229 230static __inline void 231insw(u_int port, void *addr, size_t count) 232{ 233 __asm __volatile("cld; rep; insw" 234 : "+D" (addr), "+c" (count) 235 : "d" (port) 236 : "memory"); 237} 238 239static __inline void 240insl(u_int port, void *addr, size_t count) 241{ 242 __asm __volatile("cld; rep; insl" 243 : "+D" (addr), "+c" (count) 244 : "d" (port) 245 : "memory"); 246} 247 248static __inline void 249invd(void) 250{ 251 __asm __volatile("invd"); 252} 253 254static __inline u_short 255inw(u_int port) 256{ 257 u_short data; 258 259 __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port)); 260 return (data); 261} 262 263static __inline void 264outb(u_int port, u_char data) 265{ 266 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port)); 267} 268 269static __inline void 270outl(u_int port, u_int data) 271{ 272 __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port)); 273} 274 275static __inline void 276outsb(u_int port, const void *addr, size_t count) 277{ 278 __asm __volatile("cld; rep; outsb" 279 : "+S" (addr), "+c" (count) 280 : "d" (port)); 281} 282 283static __inline void 284outsw(u_int port, const void *addr, size_t count) 285{ 286 __asm __volatile("cld; rep; outsw" 287 : "+S" (addr), "+c" (count) 288 : "d" (port)); 289} 290 291static __inline void 292outsl(u_int port, const void *addr, size_t count) 293{ 294 __asm __volatile("cld; rep; outsl" 295 : "+S" (addr), "+c" (count) 296 : "d" (port)); 297} 298 299static __inline void 300outw(u_int port, u_short data) 301{ 302 __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port)); 303} 304 305static __inline void 306ia32_pause(void) 307{ 308 __asm __volatile("pause"); 309} 310 311static __inline u_int 312#ifdef XEN 313_read_eflags(void) 314#else 315read_eflags(void) 316#endif 317{ 318 u_int ef; 319 320 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 321 return (ef); 322} 323 324static __inline uint64_t 325rdmsr(u_int msr) 326{ 327 uint64_t rv; 328 329 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); 330 return (rv); 331} 332 333static __inline uint64_t 334rdpmc(u_int pmc) 335{ 336 uint64_t rv; 337 338 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); 339 return (rv); 340} 341 342static __inline uint64_t 343rdtsc(void) 344{ 345 uint64_t rv; 346 347 __asm __volatile("rdtsc" : "=A" (rv)); 348 return (rv); 349} 350 351static __inline uint32_t 352rdtsc32(void) 353{ 354 uint32_t rv; 355 356 __asm __volatile("rdtsc" : "=a" (rv) : : "edx"); 357 return (rv); 358} 359 360static __inline void 361wbinvd(void) 362{ 363 __asm __volatile("wbinvd"); 364} 365 366static __inline void 367#ifdef XEN 368_write_eflags(u_int ef) 369#else 370write_eflags(u_int ef) 371#endif 372{ 373 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 374} 375 376static __inline void 377wrmsr(u_int msr, uint64_t newval) 378{ 379 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); 380} 381 382static __inline void 383load_cr0(u_int data) 384{ 385 386 __asm __volatile("movl %0,%%cr0" : : "r" (data)); 387} 388 389static __inline u_int 390rcr0(void) 391{ 392 u_int data; 393 394 __asm __volatile("movl %%cr0,%0" : "=r" (data)); 395 return (data); 396} 397 398static __inline u_int 399rcr2(void) 400{ 401 u_int data; 402 403#ifdef XEN 404 return (xen_rcr2()); 405#endif 406 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 407 return (data); 408} 409 410static __inline void 411load_cr3(u_int data) 412{ 413#ifdef XEN 414 xen_load_cr3(data); 415#else 416 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory"); 417#endif 418} 419 420static __inline u_int 421rcr3(void) 422{ 423 u_int data; 424 425 __asm __volatile("movl %%cr3,%0" : "=r" (data)); 426 return (data); 427} 428 429static __inline void 430load_cr4(u_int data) 431{ 432 __asm __volatile("movl %0,%%cr4" : : "r" (data)); 433} 434 435static __inline u_int 436rcr4(void) 437{ 438 u_int data; 439 440 __asm __volatile("movl %%cr4,%0" : "=r" (data)); 441 return (data); 442} 443 444static __inline uint64_t 445rxcr(u_int reg) 446{ 447 u_int low, high; 448 449 __asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg)); 450 return (low | ((uint64_t)high << 32)); 451} 452 453static __inline void 454load_xcr(u_int reg, uint64_t val) 455{ 456 u_int low, high; 457 458 low = val; 459 high = val >> 32; 460 __asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high)); 461} 462 463/* 464 * Global TLB flush (except for thise for pages marked PG_G) 465 */ 466static __inline void 467invltlb(void) 468{ 469#ifdef XEN 470 xen_tlb_flush(); 471#else 472 load_cr3(rcr3()); 473#endif 474} 475 476/* 477 * TLB flush for an individual page (even if it has PG_G). 478 * Only works on 486+ CPUs (i386 does not have PG_G). 479 */ 480static __inline void 481invlpg(u_int addr) 482{ 483 484#ifdef XEN 485 xen_invlpg(addr); 486#else 487 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 488#endif 489} 490 491static __inline u_short 492rfs(void) 493{ 494 u_short sel; 495 __asm __volatile("movw %%fs,%0" : "=rm" (sel)); 496 return (sel); 497} 498 499static __inline uint64_t 500rgdt(void) 501{ 502 uint64_t gdtr; 503 __asm __volatile("sgdt %0" : "=m" (gdtr)); 504 return (gdtr); 505} 506 507static __inline u_short 508rgs(void) 509{ 510 u_short sel; 511 __asm __volatile("movw %%gs,%0" : "=rm" (sel)); 512 return (sel); 513} 514 515static __inline uint64_t 516ridt(void) 517{ 518 uint64_t idtr; 519 __asm __volatile("sidt %0" : "=m" (idtr)); 520 return (idtr); 521} 522 523static __inline u_short 524rldt(void) 525{ 526 u_short ldtr; 527 __asm __volatile("sldt %0" : "=g" (ldtr)); 528 return (ldtr); 529} 530 531static __inline u_short 532rss(void) 533{ 534 u_short sel; 535 __asm __volatile("movw %%ss,%0" : "=rm" (sel)); 536 return (sel); 537} 538 539static __inline u_short 540rtr(void) 541{ 542 u_short tr; 543 __asm __volatile("str %0" : "=g" (tr)); 544 return (tr); 545} 546 547static __inline void 548load_fs(u_short sel) 549{ 550 __asm __volatile("movw %0,%%fs" : : "rm" (sel)); 551} 552 553static __inline void 554load_gs(u_short sel) 555{ 556 __asm __volatile("movw %0,%%gs" : : "rm" (sel)); 557} 558 559static __inline void 560lidt(struct region_descriptor *addr) 561{ 562 __asm __volatile("lidt (%0)" : : "r" (addr)); 563} 564 565static __inline void 566lldt(u_short sel) 567{ 568 __asm __volatile("lldt %0" : : "r" (sel)); 569} 570 571static __inline void 572ltr(u_short sel) 573{ 574 __asm __volatile("ltr %0" : : "r" (sel)); 575} 576 577static __inline u_int 578rdr0(void) 579{ 580 u_int data; 581 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 582 return (data); 583} 584 585static __inline void 586load_dr0(u_int dr0) 587{ 588 __asm __volatile("movl %0,%%dr0" : : "r" (dr0)); 589} 590 591static __inline u_int 592rdr1(void) 593{ 594 u_int data; 595 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 596 return (data); 597} 598 599static __inline void 600load_dr1(u_int dr1) 601{ 602 __asm __volatile("movl %0,%%dr1" : : "r" (dr1)); 603} 604 605static __inline u_int 606rdr2(void) 607{ 608 u_int data; 609 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 610 return (data); 611} 612 613static __inline void 614load_dr2(u_int dr2) 615{ 616 __asm __volatile("movl %0,%%dr2" : : "r" (dr2)); 617} 618 619static __inline u_int 620rdr3(void) 621{ 622 u_int data; 623 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 624 return (data); 625} 626 627static __inline void 628load_dr3(u_int dr3) 629{ 630 __asm __volatile("movl %0,%%dr3" : : "r" (dr3)); 631} 632 633static __inline u_int 634rdr4(void) 635{ 636 u_int data; 637 __asm __volatile("movl %%dr4,%0" : "=r" (data)); 638 return (data); 639} 640 641static __inline void 642load_dr4(u_int dr4) 643{ 644 __asm __volatile("movl %0,%%dr4" : : "r" (dr4)); 645} 646 647static __inline u_int 648rdr5(void) 649{ 650 u_int data; 651 __asm __volatile("movl %%dr5,%0" : "=r" (data)); 652 return (data); 653} 654 655static __inline void 656load_dr5(u_int dr5) 657{ 658 __asm __volatile("movl %0,%%dr5" : : "r" (dr5)); 659} 660 661static __inline u_int 662rdr6(void) 663{ 664 u_int data; 665 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 666 return (data); 667} 668 669static __inline void 670load_dr6(u_int dr6) 671{ 672 __asm __volatile("movl %0,%%dr6" : : "r" (dr6)); 673} 674 675static __inline u_int 676rdr7(void) 677{ 678 u_int data; 679 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 680 return (data); 681} 682 683static __inline void 684load_dr7(u_int dr7) 685{ 686 __asm __volatile("movl %0,%%dr7" : : "r" (dr7)); 687} 688 689static __inline u_char 690read_cyrix_reg(u_char reg) 691{ 692 outb(0x22, reg); 693 return inb(0x23); 694} 695 696static __inline void 697write_cyrix_reg(u_char reg, u_char data) 698{ 699 outb(0x22, reg); 700 outb(0x23, data); 701} 702 703static __inline register_t 704intr_disable(void) 705{ 706 register_t eflags; 707 708 eflags = read_eflags(); 709 disable_intr(); 710 return (eflags); 711} 712 713static __inline void 714intr_restore(register_t eflags) 715{ 716 write_eflags(eflags); 717} 718 719#else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */ 720 721int breakpoint(void); 722u_int bsfl(u_int mask); 723u_int bsrl(u_int mask); 724void clflush(u_long addr); 725void clts(void); 726void cpuid_count(u_int ax, u_int cx, u_int *p); 727void disable_intr(void); 728void do_cpuid(u_int ax, u_int *p); 729void enable_intr(void); 730void halt(void); 731void ia32_pause(void); 732u_char inb(u_int port); 733u_int inl(u_int port); 734void insb(u_int port, void *addr, size_t count); 735void insl(u_int port, void *addr, size_t count); 736void insw(u_int port, void *addr, size_t count); 737register_t intr_disable(void); 738void intr_restore(register_t ef); 739void invd(void); 740void invlpg(u_int addr); 741void invltlb(void); 742u_short inw(u_int port); 743void lidt(struct region_descriptor *addr); 744void lldt(u_short sel); 745void load_cr0(u_int cr0); 746void load_cr3(u_int cr3); 747void load_cr4(u_int cr4); 748void load_dr0(u_int dr0); 749void load_dr1(u_int dr1); 750void load_dr2(u_int dr2); 751void load_dr3(u_int dr3); 752void load_dr4(u_int dr4); 753void load_dr5(u_int dr5); 754void load_dr6(u_int dr6); 755void load_dr7(u_int dr7); 756void load_fs(u_short sel); 757void load_gs(u_short sel); 758void ltr(u_short sel); 759void outb(u_int port, u_char data); 760void outl(u_int port, u_int data); 761void outsb(u_int port, const void *addr, size_t count); 762void outsl(u_int port, const void *addr, size_t count); 763void outsw(u_int port, const void *addr, size_t count); 764void outw(u_int port, u_short data); 765u_int rcr0(void); 766u_int rcr2(void); 767u_int rcr3(void); 768u_int rcr4(void); 769uint64_t rdmsr(u_int msr); 770uint64_t rdpmc(u_int pmc); 771u_int rdr0(void); 772u_int rdr1(void); 773u_int rdr2(void); 774u_int rdr3(void); 775u_int rdr4(void); 776u_int rdr5(void); 777u_int rdr6(void); 778u_int rdr7(void); 779uint64_t rdtsc(void); 780u_char read_cyrix_reg(u_char reg); 781u_int read_eflags(void); 782u_int rfs(void); 783uint64_t rgdt(void); 784u_int rgs(void); 785uint64_t ridt(void); 786u_short rldt(void); 787u_short rtr(void); 788void wbinvd(void); 789void write_cyrix_reg(u_char reg, u_char data); 790void write_eflags(u_int ef); 791void wrmsr(u_int msr, uint64_t newval); 792 793#endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */ 794 795void reset_dbregs(void); 796 797#ifdef _KERNEL 798int rdmsr_safe(u_int msr, uint64_t *val); 799int wrmsr_safe(u_int msr, uint64_t newval); 800#endif 801 802#endif /* !_MACHINE_CPUFUNC_H_ */ 803