1221828Sgrehan/*- 2221828Sgrehan * Copyright (c) 2011 NetApp, Inc. 3221828Sgrehan * All rights reserved. 4221828Sgrehan * 5221828Sgrehan * Redistribution and use in source and binary forms, with or without 6221828Sgrehan * modification, are permitted provided that the following conditions 7221828Sgrehan * are met: 8221828Sgrehan * 1. Redistributions of source code must retain the above copyright 9221828Sgrehan * notice, this list of conditions and the following disclaimer. 10221828Sgrehan * 2. Redistributions in binary form must reproduce the above copyright 11221828Sgrehan * notice, this list of conditions and the following disclaimer in the 12221828Sgrehan * documentation and/or other materials provided with the distribution. 13221828Sgrehan * 14221828Sgrehan * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 15221828Sgrehan * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16221828Sgrehan * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17221828Sgrehan * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 18221828Sgrehan * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19221828Sgrehan * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20221828Sgrehan * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21221828Sgrehan * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22221828Sgrehan * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23221828Sgrehan * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24221828Sgrehan * SUCH DAMAGE. 25221828Sgrehan * 26221828Sgrehan * $FreeBSD$ 27221828Sgrehan */ 28221828Sgrehan 29221828Sgrehan#include <sys/cdefs.h> 30221828Sgrehan__FBSDID("$FreeBSD$"); 31221828Sgrehan 32256072Sneel#include <sys/param.h> 33256072Sneel#include <sys/kernel.h> 34221828Sgrehan#include <sys/types.h> 35221828Sgrehan#include <sys/systm.h> 36221828Sgrehan#include <sys/smp.h> 37256072Sneel#include <sys/sysctl.h> 38221828Sgrehan 39221828Sgrehan#include <vm/vm.h> 40221828Sgrehan#include <vm/pmap.h> 41256072Sneel#include <vm/vm_extern.h> 42221828Sgrehan 43256072Sneel#include <machine/vmm.h> 44221828Sgrehan 45221828Sgrehan#include "vmx_cpufunc.h" 46221828Sgrehan#include "vmx_msr.h" 47221828Sgrehan#include "ept.h" 48221828Sgrehan 49256072Sneel#define EPT_SUPPORTS_EXEC_ONLY(cap) ((cap) & (1UL << 0)) 50221828Sgrehan#define EPT_PWL4(cap) ((cap) & (1UL << 6)) 51221828Sgrehan#define EPT_MEMORY_TYPE_WB(cap) ((cap) & (1UL << 14)) 52221828Sgrehan#define EPT_PDE_SUPERPAGE(cap) ((cap) & (1UL << 16)) /* 2MB pages */ 53221828Sgrehan#define EPT_PDPTE_SUPERPAGE(cap) ((cap) & (1UL << 17)) /* 1GB pages */ 54256072Sneel#define INVEPT_SUPPORTED(cap) ((cap) & (1UL << 20)) 55256072Sneel#define AD_BITS_SUPPORTED(cap) ((cap) & (1UL << 21)) 56221828Sgrehan#define INVVPID_SUPPORTED(cap) ((cap) & (1UL << 32)) 57221828Sgrehan 58221828Sgrehan#define INVVPID_ALL_TYPES_MASK 0xF0000000000UL 59221828Sgrehan#define INVVPID_ALL_TYPES_SUPPORTED(cap) \ 60221828Sgrehan (((cap) & INVVPID_ALL_TYPES_MASK) == INVVPID_ALL_TYPES_MASK) 61221828Sgrehan 62221828Sgrehan#define INVEPT_ALL_TYPES_MASK 0x6000000UL 63221828Sgrehan#define INVEPT_ALL_TYPES_SUPPORTED(cap) \ 64221828Sgrehan (((cap) & INVEPT_ALL_TYPES_MASK) == INVEPT_ALL_TYPES_MASK) 65221828Sgrehan 66256072Sneel#define EPT_PWLEVELS 4 /* page walk levels */ 67256072Sneel#define EPT_ENABLE_AD_BITS (1 << 6) 68221828Sgrehan 69256072SneelSYSCTL_DECL(_hw_vmm); 70256072SneelSYSCTL_NODE(_hw_vmm, OID_AUTO, ept, CTLFLAG_RW, NULL, NULL); 71221828Sgrehan 72256072Sneelstatic int ept_enable_ad_bits; 73221828Sgrehan 74256072Sneelstatic int ept_pmap_flags; 75256072SneelSYSCTL_INT(_hw_vmm_ept, OID_AUTO, pmap_flags, CTLFLAG_RD, 76256072Sneel &ept_pmap_flags, 0, NULL); 77221828Sgrehan 78221828Sgrehanint 79221828Sgrehanept_init(void) 80221828Sgrehan{ 81256072Sneel int use_hw_ad_bits, use_superpages, use_exec_only; 82221828Sgrehan uint64_t cap; 83221828Sgrehan 84221828Sgrehan cap = rdmsr(MSR_VMX_EPT_VPID_CAP); 85221828Sgrehan 86221828Sgrehan /* 87221828Sgrehan * Verify that: 88221828Sgrehan * - page walk length is 4 steps 89221828Sgrehan * - extended page tables can be laid out in write-back memory 90221828Sgrehan * - invvpid instruction with all possible types is supported 91221828Sgrehan * - invept instruction with all possible types is supported 92221828Sgrehan */ 93221828Sgrehan if (!EPT_PWL4(cap) || 94221828Sgrehan !EPT_MEMORY_TYPE_WB(cap) || 95221828Sgrehan !INVVPID_SUPPORTED(cap) || 96221828Sgrehan !INVVPID_ALL_TYPES_SUPPORTED(cap) || 97221828Sgrehan !INVEPT_SUPPORTED(cap) || 98221828Sgrehan !INVEPT_ALL_TYPES_SUPPORTED(cap)) 99221828Sgrehan return (EINVAL); 100221828Sgrehan 101256072Sneel use_superpages = 1; 102256072Sneel TUNABLE_INT_FETCH("hw.vmm.ept.use_superpages", &use_superpages); 103256072Sneel if (use_superpages && EPT_PDE_SUPERPAGE(cap)) 104256072Sneel ept_pmap_flags |= PMAP_PDE_SUPERPAGE; /* 2MB superpage */ 105221828Sgrehan 106256072Sneel use_hw_ad_bits = 1; 107256072Sneel TUNABLE_INT_FETCH("hw.vmm.ept.use_hw_ad_bits", &use_hw_ad_bits); 108256072Sneel if (use_hw_ad_bits && AD_BITS_SUPPORTED(cap)) 109256072Sneel ept_enable_ad_bits = 1; 110256072Sneel else 111256072Sneel ept_pmap_flags |= PMAP_EMULATE_AD_BITS; 112221828Sgrehan 113256072Sneel use_exec_only = 1; 114256072Sneel TUNABLE_INT_FETCH("hw.vmm.ept.use_exec_only", &use_exec_only); 115256072Sneel if (use_exec_only && EPT_SUPPORTS_EXEC_ONLY(cap)) 116256072Sneel ept_pmap_flags |= PMAP_SUPPORTS_EXEC_ONLY; 117221828Sgrehan 118221828Sgrehan return (0); 119221828Sgrehan} 120221828Sgrehan 121241147Sneel#if 0 122241147Sneelstatic void 123241147Sneelept_dump(uint64_t *ptp, int nlevels) 124241147Sneel{ 125241147Sneel int i, t, tabs; 126241147Sneel uint64_t *ptpnext, ptpval; 127241147Sneel 128241147Sneel if (--nlevels < 0) 129241147Sneel return; 130241147Sneel 131241147Sneel tabs = 3 - nlevels; 132241147Sneel for (t = 0; t < tabs; t++) 133241147Sneel printf("\t"); 134241147Sneel printf("PTP = %p\n", ptp); 135241147Sneel 136241147Sneel for (i = 0; i < 512; i++) { 137241147Sneel ptpval = ptp[i]; 138241147Sneel 139241147Sneel if (ptpval == 0) 140241147Sneel continue; 141241147Sneel 142241147Sneel for (t = 0; t < tabs; t++) 143241147Sneel printf("\t"); 144241147Sneel printf("%3d 0x%016lx\n", i, ptpval); 145241147Sneel 146241147Sneel if (nlevels != 0 && (ptpval & EPT_PG_SUPERPAGE) == 0) { 147241147Sneel ptpnext = (uint64_t *) 148241147Sneel PHYS_TO_DMAP(ptpval & EPT_ADDR_MASK); 149241147Sneel ept_dump(ptpnext, nlevels); 150241147Sneel } 151241147Sneel } 152241147Sneel} 153241147Sneel#endif 154241147Sneel 155221828Sgrehanstatic void 156256072Sneelinvept_single_context(void *arg) 157221828Sgrehan{ 158256072Sneel struct invept_desc desc = *(struct invept_desc *)arg; 159221828Sgrehan 160256072Sneel invept(INVEPT_TYPE_SINGLE_CONTEXT, desc); 161221828Sgrehan} 162221828Sgrehan 163256072Sneelvoid 164256072Sneelept_invalidate_mappings(u_long eptp) 165221828Sgrehan{ 166256072Sneel struct invept_desc invept_desc = { 0 }; 167221828Sgrehan 168256072Sneel invept_desc.eptp = eptp; 169221828Sgrehan 170256072Sneel smp_rendezvous(NULL, invept_single_context, NULL, &invept_desc); 171221828Sgrehan} 172221828Sgrehan 173256072Sneelstatic int 174256072Sneelept_pinit(pmap_t pmap) 175221828Sgrehan{ 176221828Sgrehan 177256072Sneel return (pmap_pinit_type(pmap, PT_EPT, ept_pmap_flags)); 178221828Sgrehan} 179221828Sgrehan 180256072Sneelstruct vmspace * 181256072Sneelept_vmspace_alloc(vm_offset_t min, vm_offset_t max) 182221828Sgrehan{ 183221828Sgrehan 184256072Sneel return (vmspace_alloc(min, max, ept_pinit)); 185221828Sgrehan} 186221828Sgrehan 187221828Sgrehanvoid 188256072Sneelept_vmspace_free(struct vmspace *vmspace) 189221828Sgrehan{ 190221828Sgrehan 191256072Sneel vmspace_free(vmspace); 192221828Sgrehan} 193221828Sgrehan 194256072Sneeluint64_t 195256072Sneeleptp(uint64_t pml4) 196221828Sgrehan{ 197256072Sneel uint64_t eptp_val; 198221828Sgrehan 199256072Sneel eptp_val = pml4 | (EPT_PWLEVELS - 1) << 3 | PAT_WRITE_BACK; 200256072Sneel if (ept_enable_ad_bits) 201256072Sneel eptp_val |= EPT_ENABLE_AD_BITS; 202221828Sgrehan 203256072Sneel return (eptp_val); 204221828Sgrehan} 205