182498Sroberto/*- 282498Sroberto * SPDX-License-Identifier: BSD-2-Clause 382498Sroberto * 482498Sroberto * Copyright (c) 2011 NetApp, Inc. 582498Sroberto * All rights reserved. 682498Sroberto * 782498Sroberto * Redistribution and use in source and binary forms, with or without 8285612Sdelphij * modification, are permitted provided that the following conditions 982498Sroberto * are met: 10285612Sdelphij * 1. Redistributions of source code must retain the above copyright 1182498Sroberto * notice, this list of conditions and the following disclaimer. 12132451Sroberto * 2. Redistributions in binary form must reproduce the above copyright 1382498Sroberto * notice, this list of conditions and the following disclaimer in the 1482498Sroberto * documentation and/or other materials provided with the distribution. 1582498Sroberto * 1682498Sroberto * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 1782498Sroberto * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18132451Sroberto * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1982498Sroberto * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 20285612Sdelphij * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21285612Sdelphij * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22285612Sdelphij * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23285612Sdelphij * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2482498Sroberto * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25316069Sdelphij * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26132451Sroberto * SUCH DAMAGE. 27316069Sdelphij */ 28132451Sroberto 29132451Sroberto#include <sys/param.h> 30316069Sdelphij#ifndef WITHOUT_CAPSICUM 31132451Sroberto#include <sys/capsicum.h> 32132451Sroberto#endif 33316069Sdelphij#include <sys/types.h> 34132451Sroberto#include <sys/mman.h> 35309008Sdelphij#include <sys/pciio.h> 36132451Sroberto#include <sys/ioctl.h> 3782498Sroberto#include <sys/stat.h> 3882498Sroberto 3982498Sroberto#include <dev/io/iodev.h> 4082498Sroberto#include <dev/pci/pcireg.h> 4182498Sroberto 42285612Sdelphij#include <vm/vm.h> 43285612Sdelphij 44285612Sdelphij#include <machine/iodev.h> 45285612Sdelphij#include <machine/vm.h> 46285612Sdelphij 47285612Sdelphij#ifndef WITHOUT_CAPSICUM 48285612Sdelphij#include <capsicum_helpers.h> 49285612Sdelphij#endif 50285612Sdelphij#include <ctype.h> 51285612Sdelphij#include <stdio.h> 52285612Sdelphij#include <stdlib.h> 53285612Sdelphij#include <string.h> 54285612Sdelphij#include <err.h> 55285612Sdelphij#include <errno.h> 56285612Sdelphij#include <fcntl.h> 57285612Sdelphij#include <sysexits.h> 58285612Sdelphij#include <unistd.h> 59285612Sdelphij 60285612Sdelphij#include <machine/vmm.h> 61285612Sdelphij 62285612Sdelphij#include "debug.h" 63285612Sdelphij#include "mem.h" 64285612Sdelphij#include "pci_passthru.h" 65285612Sdelphij 66285612Sdelphij#ifndef _PATH_DEVPCI 67285612Sdelphij#define _PATH_DEVPCI "/dev/pci" 68285612Sdelphij#endif 69132451Sroberto 7082498Sroberto#define LEGACY_SUPPORT 1 71132451Sroberto 72132451Sroberto#define MSIX_TABLE_COUNT(ctrl) (((ctrl) & PCIM_MSIXCTRL_TABLE_SIZE) + 1) 73132451Sroberto#define MSIX_CAPLEN 12 7482498Sroberto 75132451Sroberto#define PASSTHRU_MMIO_MAX 2 76132451Sroberto 77132451Srobertostatic int pcifd = -1; 78285612Sdelphij 79132451SrobertoSET_DECLARE(passthru_dev_set, struct passthru_dev); 80132451Sroberto 81132451Srobertostruct passthru_softc { 82132451Sroberto struct pci_devinst *psc_pi; 83132451Sroberto /* ROM is handled like a BAR */ 84132451Sroberto struct pcibar psc_bar[PCI_BARMAX_WITH_ROM + 1]; 85132451Sroberto struct { 86132451Sroberto int capoff; 87132451Sroberto int msgctrl; 88132451Sroberto int emulated; 89132451Sroberto } psc_msi; 90132451Sroberto struct { 91132451Sroberto int capoff; 92132451Sroberto } psc_msix; 93132451Sroberto struct pcisel psc_sel; 94132451Sroberto 95132451Sroberto struct passthru_mmio_mapping psc_mmio_map[PASSTHRU_MMIO_MAX]; 9682498Sroberto cfgread_handler psc_pcir_rhandler[PCI_REGMAX + 1]; 97132451Sroberto cfgwrite_handler psc_pcir_whandler[PCI_REGMAX + 1]; 98132451Sroberto}; 99132451Sroberto 100132451Srobertostatic int 101132451Srobertomsi_caplen(int msgctrl) 102132451Sroberto{ 103132451Sroberto int len; 104132451Sroberto 105132451Sroberto len = 10; /* minimum length of msi capability */ 106132451Sroberto 107132451Sroberto if (msgctrl & PCIM_MSICTRL_64BIT) 108132451Sroberto len += 4; 109132451Sroberto 110132451Sroberto#if 0 111132451Sroberto /* 112132451Sroberto * Ignore the 'mask' and 'pending' bits in the MSI capability. 113132451Sroberto * We'll let the guest manipulate them directly. 114132451Sroberto */ 115132451Sroberto if (msgctrl & PCIM_MSICTRL_VECTOR) 116132451Sroberto len += 10; 117285612Sdelphij#endif 118285612Sdelphij 119285612Sdelphij return (len); 120285612Sdelphij} 121285612Sdelphij 122285612Sdelphijstatic int 123285612Sdelphijpcifd_init(void) 124285612Sdelphij{ 125285612Sdelphij pcifd = open(_PATH_DEVPCI, O_RDWR, 0); 126285612Sdelphij if (pcifd < 0) { 127285612Sdelphij warn("failed to open %s", _PATH_DEVPCI); 128285612Sdelphij return (1); 129285612Sdelphij } 130285612Sdelphij 131285612Sdelphij#ifndef WITHOUT_CAPSICUM 132285612Sdelphij cap_rights_t pcifd_rights; 133285612Sdelphij cap_rights_init(&pcifd_rights, CAP_IOCTL, CAP_READ, CAP_WRITE); 134285612Sdelphij if (caph_rights_limit(pcifd, &pcifd_rights) == -1) 135285612Sdelphij errx(EX_OSERR, "Unable to apply rights for sandbox"); 136285612Sdelphij 137285612Sdelphij const cap_ioctl_t pcifd_ioctls[] = { PCIOCREAD, PCIOCWRITE, PCIOCGETBAR, 138285612Sdelphij PCIOCBARIO, PCIOCBARMMAP, PCIOCGETCONF }; 13982498Sroberto if (caph_ioctls_limit(pcifd, pcifd_ioctls, nitems(pcifd_ioctls)) == -1) 14082498Sroberto errx(EX_OSERR, "Unable to apply rights for sandbox"); 141132451Sroberto#endif 14282498Sroberto 143132451Sroberto return (0); 144132451Sroberto} 145132451Sroberto 146281230Sdelphijuint32_t 147132451Srobertopci_host_read_config(const struct pcisel *sel, long reg, int width) 14882498Sroberto{ 14982498Sroberto struct pci_io pi; 150132451Sroberto 15182498Sroberto if (pcifd < 0 && pcifd_init()) { 152132451Sroberto return (0); 153285612Sdelphij } 154285612Sdelphij 155285612Sdelphij bzero(&pi, sizeof(pi)); 156285612Sdelphij pi.pi_sel = *sel; 157285612Sdelphij pi.pi_reg = reg; 15882498Sroberto pi.pi_width = width; 15982498Sroberto 160132451Sroberto if (ioctl(pcifd, PCIOCREAD, &pi) < 0) 16182498Sroberto return (0); /* XXX */ 162285612Sdelphij else 163285612Sdelphij return (pi.pi_data); 164285612Sdelphij} 165132451Sroberto 166132451Srobertovoid 167285612Sdelphijpci_host_write_config(const struct pcisel *sel, long reg, int width, 168285612Sdelphij uint32_t data) 169285612Sdelphij{ 170285612Sdelphij struct pci_io pi; 17182498Sroberto 17282498Sroberto if (pcifd < 0 && pcifd_init()) { 173132451Sroberto return; 17482498Sroberto } 175132451Sroberto 176132451Sroberto bzero(&pi, sizeof(pi)); 177132451Sroberto pi.pi_sel = *sel; 178132451Sroberto pi.pi_reg = reg; 179132451Sroberto pi.pi_width = width; 180132451Sroberto pi.pi_data = data; 18182498Sroberto 18282498Sroberto (void)ioctl(pcifd, PCIOCWRITE, &pi); /* XXX */ 18382498Sroberto} 18482498Sroberto 185285612Sdelphij#ifdef LEGACY_SUPPORT 186285612Sdelphijstatic int 187285612Sdelphijpassthru_add_msicap(struct pci_devinst *pi, int msgnum, int nextptr) 188285612Sdelphij{ 189285612Sdelphij int capoff; 190285612Sdelphij struct msicap msicap; 191285612Sdelphij u_char *capdata; 192285612Sdelphij 193285612Sdelphij pci_populate_msicap(&msicap, msgnum, nextptr); 194285612Sdelphij 195285612Sdelphij /* 196285612Sdelphij * XXX 197285612Sdelphij * Copy the msi capability structure in the last 16 bytes of the 198285612Sdelphij * config space. This is wrong because it could shadow something 199285612Sdelphij * useful to the device. 200316069Sdelphij */ 201285612Sdelphij capoff = 256 - roundup(sizeof(msicap), 4); 202285612Sdelphij capdata = (u_char *)&msicap; 203285612Sdelphij for (size_t i = 0; i < sizeof(msicap); i++) 204285612Sdelphij pci_set_cfgdata8(pi, capoff + i, capdata[i]); 205285612Sdelphij 206285612Sdelphij return (capoff); 207285612Sdelphij} 208285612Sdelphij#endif /* LEGACY_SUPPORT */ 209289997Sglebius 21082498Srobertostatic int 211132451Srobertocfginitmsi(struct passthru_softc *sc) 212132451Sroberto{ 213132451Sroberto int i, ptr, capptr, cap, sts, caplen, table_size; 214132451Sroberto uint32_t u32; 215132451Sroberto struct pcisel sel; 216132451Sroberto struct pci_devinst *pi; 21782498Sroberto struct msixcap msixcap; 21882498Sroberto char *msixcap_ptr; 21982498Sroberto 22082498Sroberto pi = sc->psc_pi; 22182498Sroberto sel = sc->psc_sel; 22282498Sroberto 22382498Sroberto /* 22482498Sroberto * Parse the capabilities and cache the location of the MSI 225132451Sroberto * and MSI-X capabilities. 226285612Sdelphij */ 22782498Sroberto sts = pci_host_read_config(&sel, PCIR_STATUS, 2); 228132451Sroberto if (sts & PCIM_STATUS_CAPPRESENT) { 22982498Sroberto ptr = pci_host_read_config(&sel, PCIR_CAP_PTR, 1); 230285612Sdelphij while (ptr != 0 && ptr != 0xff) { 231285612Sdelphij cap = pci_host_read_config(&sel, ptr + PCICAP_ID, 1); 232132451Sroberto if (cap == PCIY_MSI) { 233132451Sroberto /* 234132451Sroberto * Copy the MSI capability into the config 23582498Sroberto * space of the emulated pci device 23682498Sroberto */ 237309008Sdelphij sc->psc_msi.capoff = ptr; 238132451Sroberto sc->psc_msi.msgctrl = pci_host_read_config(&sel, 239132451Sroberto ptr + 2, 2); 240132451Sroberto sc->psc_msi.emulated = 0; 241132451Sroberto caplen = msi_caplen(sc->psc_msi.msgctrl); 24282498Sroberto capptr = ptr; 243182007Sroberto while (caplen > 0) { 244182007Sroberto u32 = pci_host_read_config(&sel, capptr, 245182007Sroberto 4); 24682498Sroberto pci_set_cfgdata32(pi, capptr, u32); 24782498Sroberto caplen -= 4; 24882498Sroberto capptr += 4; 24982498Sroberto } 250132451Sroberto } else if (cap == PCIY_MSIX) { 251285612Sdelphij /* 252132451Sroberto * Copy the MSI-X capability 253285612Sdelphij */ 254285612Sdelphij sc->psc_msix.capoff = ptr; 255132451Sroberto caplen = 12; 256132451Sroberto msixcap_ptr = (char *)&msixcap; 257132451Sroberto capptr = ptr; 258132451Sroberto while (caplen > 0) { 259182007Sroberto u32 = pci_host_read_config(&sel, capptr, 260132451Sroberto 4); 261285612Sdelphij memcpy(msixcap_ptr, &u32, 4); 262132451Sroberto pci_set_cfgdata32(pi, capptr, u32); 263285612Sdelphij caplen -= 4; 264132451Sroberto capptr += 4; 265132451Sroberto msixcap_ptr += 4; 266132451Sroberto } 267132451Sroberto } 268132451Sroberto ptr = pci_host_read_config(&sel, ptr + PCICAP_NEXTPTR, 269132451Sroberto 1); 270309008Sdelphij } 271330141Sdelphij } 272330141Sdelphij 273330141Sdelphij if (sc->psc_msix.capoff != 0) { 274330141Sdelphij pi->pi_msix.pba_bar = 275330141Sdelphij msixcap.pba_info & PCIM_MSIX_BIR_MASK; 276309008Sdelphij pi->pi_msix.pba_offset = 277330141Sdelphij msixcap.pba_info & ~PCIM_MSIX_BIR_MASK; 278309008Sdelphij pi->pi_msix.table_bar = 279309008Sdelphij msixcap.table_info & PCIM_MSIX_BIR_MASK; 280309008Sdelphij pi->pi_msix.table_offset = 281132451Sroberto msixcap.table_info & ~PCIM_MSIX_BIR_MASK; 28282498Sroberto pi->pi_msix.table_count = MSIX_TABLE_COUNT(msixcap.msgctrl); 28382498Sroberto pi->pi_msix.pba_size = PBA_SIZE(pi->pi_msix.table_count); 284294569Sdelphij 28582498Sroberto /* Allocate the emulated MSI-X table array */ 28682498Sroberto table_size = pi->pi_msix.table_count * MSIX_TABLE_ENTRY_SIZE; 287285612Sdelphij pi->pi_msix.table = calloc(1, table_size); 288132451Sroberto 289285612Sdelphij /* Mask all table entries */ 290285612Sdelphij for (i = 0; i < pi->pi_msix.table_count; i++) { 29182498Sroberto pi->pi_msix.table[i].vector_control |= 29282498Sroberto PCIM_MSIX_VCTRL_MASK; 29382498Sroberto } 29482498Sroberto } 29582498Sroberto 29682498Sroberto#ifdef LEGACY_SUPPORT 29782498Sroberto /* 298182007Sroberto * If the passthrough device does not support MSI then craft a 299182007Sroberto * MSI capability for it. We link the new MSI capability at the 300285612Sdelphij * head of the list of capabilities. 301182007Sroberto */ 30282498Sroberto if ((sts & PCIM_STATUS_CAPPRESENT) != 0 && sc->psc_msi.capoff == 0) { 30382498Sroberto int origptr, msiptr; 30482498Sroberto origptr = pci_host_read_config(&sel, PCIR_CAP_PTR, 1); 30582498Sroberto msiptr = passthru_add_msicap(pi, 1, origptr); 30682498Sroberto sc->psc_msi.capoff = msiptr; 30782498Sroberto sc->psc_msi.msgctrl = pci_get_cfgdata16(pi, msiptr + 2); 308182007Sroberto sc->psc_msi.emulated = 1; 30982498Sroberto pci_set_cfgdata8(pi, PCIR_CAP_PTR, msiptr); 31082498Sroberto } 31182498Sroberto#endif 31282498Sroberto 31382498Sroberto /* Make sure one of the capabilities is present */ 314309008Sdelphij if (sc->psc_msi.capoff == 0 && sc->psc_msix.capoff == 0) 315132451Sroberto return (-1); 31682498Sroberto else 317132451Sroberto return (0); 318132451Sroberto} 319132451Sroberto 320285612Sdelphijstatic uint64_t 321182007Srobertomsix_table_read(struct passthru_softc *sc, uint64_t offset, int size) 322132451Sroberto{ 32382498Sroberto struct pci_devinst *pi; 324182007Sroberto struct msix_table_entry *entry; 325285612Sdelphij uint8_t *src8; 326182007Sroberto uint16_t *src16; 32782498Sroberto uint32_t *src32; 32882498Sroberto uint64_t *src64; 32982498Sroberto uint64_t data; 330132451Sroberto size_t entry_offset; 33182498Sroberto uint32_t table_offset; 332285612Sdelphij int index, table_count; 333285612Sdelphij 33482498Sroberto pi = sc->psc_pi; 33582498Sroberto 33682498Sroberto table_offset = pi->pi_msix.table_offset; 33782498Sroberto table_count = pi->pi_msix.table_count; 33882498Sroberto if (offset < table_offset || 33982498Sroberto offset >= table_offset + table_count * MSIX_TABLE_ENTRY_SIZE) { 340285612Sdelphij switch (size) { 341285612Sdelphij case 1: 342285612Sdelphij src8 = (uint8_t *)(pi->pi_msix.mapped_addr + offset); 343285612Sdelphij data = *src8; 34482498Sroberto break; 34582498Sroberto case 2: 34682498Sroberto src16 = (uint16_t *)(pi->pi_msix.mapped_addr + offset); 34782498Sroberto data = *src16; 34882498Sroberto break; 34982498Sroberto case 4: 35082498Sroberto src32 = (uint32_t *)(pi->pi_msix.mapped_addr + offset); 35182498Sroberto data = *src32; 35282498Sroberto break; 35382498Sroberto case 8: 35482498Sroberto src64 = (uint64_t *)(pi->pi_msix.mapped_addr + offset); 35582498Sroberto data = *src64; 356182007Sroberto break; 357285612Sdelphij default: 35882498Sroberto return (-1); 35982498Sroberto } 36082498Sroberto return (data); 361132451Sroberto } 36282498Sroberto 36382498Sroberto offset -= table_offset; 36482498Sroberto index = offset / MSIX_TABLE_ENTRY_SIZE; 36582498Sroberto assert(index < table_count); 366285612Sdelphij 367182007Sroberto entry = &pi->pi_msix.table[index]; 36882498Sroberto entry_offset = offset % MSIX_TABLE_ENTRY_SIZE; 369285612Sdelphij 37082498Sroberto switch (size) { 37182498Sroberto case 1: 37282498Sroberto src8 = (uint8_t *)((uint8_t *)entry + entry_offset); 37382498Sroberto data = *src8; 37482498Sroberto break; 37582498Sroberto case 2: 376132451Sroberto src16 = (uint16_t *)((uint8_t *)entry + entry_offset); 377132451Sroberto data = *src16; 37882498Sroberto break; 379132451Sroberto case 4: 380132451Sroberto src32 = (uint32_t *)((uint8_t *)entry + entry_offset); 381132451Sroberto data = *src32; 382132451Sroberto break; 38382498Sroberto case 8: 38482498Sroberto src64 = (uint64_t *)((uint8_t *)entry + entry_offset); 385132451Sroberto data = *src64; 386132451Sroberto break; 387132451Sroberto default: 388132451Sroberto return (-1); 389182007Sroberto } 390132451Sroberto 391132451Sroberto return (data); 392309008Sdelphij} 393309008Sdelphij 394309008Sdelphijstatic void 395309008Sdelphijmsix_table_write(struct passthru_softc *sc, uint64_t offset, int size, 396309008Sdelphij uint64_t data) 397289997Sglebius{ 398132451Sroberto struct pci_devinst *pi; 399285612Sdelphij struct msix_table_entry *entry; 400285612Sdelphij uint8_t *dest8; 401309008Sdelphij uint16_t *dest16; 402132451Sroberto uint32_t *dest32; 403285612Sdelphij uint64_t *dest64; 404285612Sdelphij size_t entry_offset; 405285612Sdelphij uint32_t table_offset, vector_control; 406182007Sroberto int index, table_count; 40782498Sroberto 40882498Sroberto pi = sc->psc_pi; 40982498Sroberto 41082498Sroberto table_offset = pi->pi_msix.table_offset; 41182498Sroberto table_count = pi->pi_msix.table_count; 41282498Sroberto if (offset < table_offset || 41382498Sroberto offset >= table_offset + table_count * MSIX_TABLE_ENTRY_SIZE) { 41482498Sroberto switch (size) { 41582498Sroberto case 1: 41682498Sroberto dest8 = (uint8_t *)(pi->pi_msix.mapped_addr + offset); 41782498Sroberto *dest8 = data; 418132451Sroberto break; 41982498Sroberto case 2: 420285612Sdelphij dest16 = (uint16_t *)(pi->pi_msix.mapped_addr + offset); 421285612Sdelphij *dest16 = data; 422285612Sdelphij break; 423285612Sdelphij case 4: 424285612Sdelphij dest32 = (uint32_t *)(pi->pi_msix.mapped_addr + offset); 42582498Sroberto *dest32 = data; 426132451Sroberto break; 42782498Sroberto case 8: 42882498Sroberto dest64 = (uint64_t *)(pi->pi_msix.mapped_addr + offset); 42982498Sroberto *dest64 = data; 43082498Sroberto break; 43182498Sroberto } 432132451Sroberto return; 433132451Sroberto } 434132451Sroberto 435132451Sroberto offset -= table_offset; 436285612Sdelphij index = offset / MSIX_TABLE_ENTRY_SIZE; 437289997Sglebius assert(index < table_count); 438132451Sroberto 439132451Sroberto entry = &pi->pi_msix.table[index]; 440132451Sroberto entry_offset = offset % MSIX_TABLE_ENTRY_SIZE; 441132451Sroberto 442132451Sroberto /* Only 4 byte naturally-aligned writes are supported */ 443132451Sroberto assert(size == 4); 444132451Sroberto assert(entry_offset % 4 == 0); 445132451Sroberto 446132451Sroberto vector_control = entry->vector_control; 447182007Sroberto dest32 = (uint32_t *)((uint8_t *)entry + entry_offset); 448132451Sroberto *dest32 = data; 449285612Sdelphij /* If MSI-X hasn't been enabled, do nothing */ 450132451Sroberto if (pi->pi_msix.enabled) { 45182498Sroberto /* If the entry is masked, don't set it up */ 45282498Sroberto if ((entry->vector_control & PCIM_MSIX_VCTRL_MASK) == 0 || 45382498Sroberto (vector_control & PCIM_MSIX_VCTRL_MASK) == 0) { 454132451Sroberto (void)vm_setup_pptdev_msix(sc->psc_pi->pi_vmctx, 455132451Sroberto sc->psc_sel.pc_bus, sc->psc_sel.pc_dev, 456132451Sroberto sc->psc_sel.pc_func, index, entry->addr, 457132451Sroberto entry->msg_data, entry->vector_control); 458132451Sroberto } 459132451Sroberto } 460132451Sroberto} 46182498Sroberto 46282498Srobertostatic int 463182007Srobertoinit_msix_table(struct passthru_softc *sc) 464289997Sglebius{ 465289997Sglebius struct pci_devinst *pi = sc->psc_pi; 466289997Sglebius struct pci_bar_mmap pbm; 467289997Sglebius int b, s, f; 468289997Sglebius uint32_t table_size, table_offset; 469132451Sroberto 470132451Sroberto assert(pci_msix_table_bar(pi) >= 0 && pci_msix_pba_bar(pi) >= 0); 471132451Sroberto 472132451Sroberto b = sc->psc_sel.pc_bus; 473285612Sdelphij s = sc->psc_sel.pc_dev; 474285612Sdelphij f = sc->psc_sel.pc_func; 475132451Sroberto 476285612Sdelphij /* 477132451Sroberto * Map the region of the BAR containing the MSI-X table. This is 478285612Sdelphij * necessary for two reasons: 479132451Sroberto * 1. The PBA may reside in the first or last page containing the MSI-X 480132451Sroberto * table. 481132451Sroberto * 2. While PCI devices are not supposed to use the page(s) containing 482132451Sroberto * the MSI-X table for other purposes, some do in practice. 483132451Sroberto */ 484182007Sroberto memset(&pbm, 0, sizeof(pbm)); 485285612Sdelphij pbm.pbm_sel = sc->psc_sel; 486132451Sroberto pbm.pbm_flags = PCIIO_BAR_MMAP_RW; 487132451Sroberto pbm.pbm_reg = PCIR_BAR(pi->pi_msix.table_bar); 488132451Sroberto pbm.pbm_memattr = VM_MEMATTR_DEVICE; 489289997Sglebius 490293650Sglebius if (ioctl(pcifd, PCIOCBARMMAP, &pbm) != 0) { 491289997Sglebius warn("Failed to map MSI-X table BAR on %d/%d/%d", b, s, f); 492293650Sglebius return (-1); 493289997Sglebius } 494289997Sglebius assert(pbm.pbm_bar_off == 0); 495289997Sglebius pi->pi_msix.mapped_addr = (uint8_t *)(uintptr_t)pbm.pbm_map_base; 496289997Sglebius pi->pi_msix.mapped_size = pbm.pbm_map_length; 497289997Sglebius 498289997Sglebius table_offset = rounddown2(pi->pi_msix.table_offset, 4096); 499289997Sglebius 500289997Sglebius table_size = pi->pi_msix.table_offset - table_offset; 501132451Sroberto table_size += pi->pi_msix.table_count * MSIX_TABLE_ENTRY_SIZE; 502132451Sroberto table_size = roundup2(table_size, 4096); 503132451Sroberto 504281230Sdelphij /* 505281230Sdelphij * Unmap any pages not containing the table, we do not need to emulate 506281230Sdelphij * accesses to them. Avoid releasing address space to help ensure that 507281230Sdelphij * a buggy out-of-bounds access causes a crash. 508281230Sdelphij */ 509281230Sdelphij if (table_offset != 0) 510132451Sroberto if (mprotect(pi->pi_msix.mapped_addr, table_offset, 51182498Sroberto PROT_NONE) != 0) 51282498Sroberto warn("Failed to unmap MSI-X table BAR region"); 51382498Sroberto if (table_offset + table_size != pi->pi_msix.mapped_size) 514132451Sroberto if (mprotect( 515132451Sroberto pi->pi_msix.mapped_addr + table_offset + table_size, 516132451Sroberto pi->pi_msix.mapped_size - (table_offset + table_size), 517132451Sroberto PROT_NONE) != 0) 518132451Sroberto warn("Failed to unmap MSI-X table BAR region"); 519132451Sroberto 520182007Sroberto return (0); 521132451Sroberto} 52282498Sroberto 523132451Srobertostatic int 524132451Srobertocfginitbar(struct passthru_softc *sc) 525132451Sroberto{ 526285612Sdelphij int i, error; 527285612Sdelphij struct pci_devinst *pi; 528285612Sdelphij struct pci_bar_io bar; 529285612Sdelphij enum pcibar_type bartype; 530285612Sdelphij uint64_t base, size; 531285612Sdelphij 532132451Sroberto pi = sc->psc_pi; 533285612Sdelphij 534182007Sroberto /* 535182007Sroberto * Initialize BAR registers 536182007Sroberto */ 537285612Sdelphij for (i = 0; i <= PCI_BARMAX; i++) { 538285612Sdelphij bzero(&bar, sizeof(bar)); 539285612Sdelphij bar.pbi_sel = sc->psc_sel; 540285612Sdelphij bar.pbi_reg = PCIR_BAR(i); 541285612Sdelphij 542289997Sglebius if (ioctl(pcifd, PCIOCGETBAR, &bar) < 0) 543285612Sdelphij continue; 544132451Sroberto 545132451Sroberto if (PCI_BAR_IO(bar.pbi_base)) { 546285612Sdelphij bartype = PCIBAR_IO; 547132451Sroberto base = bar.pbi_base & PCIM_BAR_IO_BASE; 548132451Sroberto } else { 549132451Sroberto switch (bar.pbi_base & PCIM_BAR_MEM_TYPE) { 55082498Sroberto case PCIM_BAR_MEM_64: 551132451Sroberto bartype = PCIBAR_MEM64; 552132451Sroberto break; 553132451Sroberto default: 554182007Sroberto bartype = PCIBAR_MEM32; 555132451Sroberto break; 556285612Sdelphij } 557285612Sdelphij base = bar.pbi_base & PCIM_BAR_MEM_BASE; 558285612Sdelphij } 559132451Sroberto size = bar.pbi_length; 560285612Sdelphij 561285612Sdelphij if (bartype != PCIBAR_IO) { 562182007Sroberto if (((base | size) & PAGE_MASK) != 0) { 563281230Sdelphij warnx("passthru device %d/%d/%d BAR %d: " 564132451Sroberto "base %#lx or size %#lx not page aligned\n", 565132451Sroberto sc->psc_sel.pc_bus, sc->psc_sel.pc_dev, 56682498Sroberto sc->psc_sel.pc_func, i, base, size); 567285612Sdelphij return (-1); 568285612Sdelphij } 569285612Sdelphij } 570285612Sdelphij 571132451Sroberto /* Cache information about the "real" BAR */ 572132451Sroberto sc->psc_bar[i].type = bartype; 573285612Sdelphij sc->psc_bar[i].size = size; 574285612Sdelphij sc->psc_bar[i].addr = base; 575285612Sdelphij sc->psc_bar[i].lobits = 0; 576285612Sdelphij 577285612Sdelphij /* Allocate the BAR in the guest I/O or MMIO space */ 578285612Sdelphij error = pci_emul_alloc_bar(pi, i, bartype, size); 579132451Sroberto if (error) 580132451Sroberto return (-1); 581182007Sroberto 582132451Sroberto /* Use same lobits as physical bar */ 583182007Sroberto uint8_t lobits = pci_host_read_config(&sc->psc_sel, PCIR_BAR(i), 584285612Sdelphij 0x01); 585285612Sdelphij if (bartype == PCIBAR_MEM32 || bartype == PCIBAR_MEM64) { 586285612Sdelphij lobits &= ~PCIM_BAR_MEM_BASE; 587182007Sroberto } else { 588182007Sroberto lobits &= ~PCIM_BAR_IO_BASE; 589285612Sdelphij } 590285612Sdelphij sc->psc_bar[i].lobits = lobits; 591182007Sroberto pi->pi_bar[i].lobits = lobits; 592285612Sdelphij 593285612Sdelphij /* 594285612Sdelphij * 64-bit BAR takes up two slots so skip the next one. 595285612Sdelphij */ 596285612Sdelphij if (bartype == PCIBAR_MEM64) { 597285612Sdelphij i++; 598285612Sdelphij assert(i <= PCI_BARMAX); 599285612Sdelphij sc->psc_bar[i].type = PCIBAR_MEMHI64; 600285612Sdelphij } 601132451Sroberto } 602132451Sroberto return (0); 603132451Sroberto} 604182007Sroberto 605182007Srobertostatic int 606132451Srobertocfginit(struct pci_devinst *pi, int bus, int slot, int func) 607132451Sroberto{ 608132451Sroberto int error; 609132451Sroberto struct passthru_softc *sc; 610182007Sroberto uint8_t intline, intpin; 611132451Sroberto 612132451Sroberto error = 1; 613182007Sroberto sc = pi->pi_arg; 614132451Sroberto 615132451Sroberto bzero(&sc->psc_sel, sizeof(struct pcisel)); 616132451Sroberto sc->psc_sel.pc_bus = bus; 617285612Sdelphij sc->psc_sel.pc_dev = slot; 618285612Sdelphij sc->psc_sel.pc_func = func; 619285612Sdelphij 620132451Sroberto /* 621285612Sdelphij * Copy physical PCI header to virtual config space. INTLINE and INTPIN 622285612Sdelphij * shouldn't be aligned with their physical value and they are already set by 623285612Sdelphij * pci_emul_init(). 624285612Sdelphij */ 625285612Sdelphij intline = pci_get_cfgdata8(pi, PCIR_INTLINE); 626285612Sdelphij intpin = pci_get_cfgdata8(pi, PCIR_INTPIN); 627132451Sroberto for (int i = 0; i <= PCIR_MAXLAT; i += 4) { 628132451Sroberto pci_set_cfgdata32(pi, i, 629285612Sdelphij pci_host_read_config(&sc->psc_sel, i, 4)); 630285612Sdelphij } 631132451Sroberto pci_set_cfgdata8(pi, PCIR_INTLINE, intline); 632132451Sroberto pci_set_cfgdata8(pi, PCIR_INTPIN, intpin); 633132451Sroberto 634285612Sdelphij if (cfginitmsi(sc) != 0) { 635285612Sdelphij warnx("failed to initialize MSI for PCI %d/%d/%d", 636285612Sdelphij bus, slot, func); 637285612Sdelphij goto done; 638285612Sdelphij } 639285612Sdelphij 640285612Sdelphij if (cfginitbar(sc) != 0) { 641132451Sroberto warnx("failed to initialize BARs for PCI %d/%d/%d", 642285612Sdelphij bus, slot, func); 64382498Sroberto goto done; 64482498Sroberto } 64582498Sroberto 646132451Sroberto pci_host_write_config(&sc->psc_sel, PCIR_COMMAND, 2, 647132451Sroberto pci_get_cfgdata16(pi, PCIR_COMMAND)); 648132451Sroberto 649132451Sroberto /* 650132451Sroberto * We need to do this after PCIR_COMMAND got possibly updated, e.g., 65182498Sroberto * a BAR was enabled, as otherwise the PCIOCBARMMAP might fail on us. 652132451Sroberto */ 653132451Sroberto if (pci_msix_table_bar(pi) >= 0) { 654132451Sroberto error = init_msix_table(sc); 655285612Sdelphij if (error != 0) { 656132451Sroberto warnx( 657285612Sdelphij "failed to initialize MSI-X table for PCI %d/%d/%d: %d", 658285612Sdelphij bus, slot, func, error); 659285612Sdelphij goto done; 660132451Sroberto } 661132451Sroberto } 662132451Sroberto 663132451Sroberto error = 0; /* success */ 664132451Srobertodone: 665132451Sroberto return (error); 666132451Sroberto} 667285612Sdelphij 668285612Sdelphijstruct passthru_mmio_mapping * 669285612Sdelphijpassthru_get_mmio(struct passthru_softc *sc, int num) 670132451Sroberto{ 671285612Sdelphij assert(sc != NULL); 672285612Sdelphij assert(num < PASSTHRU_MMIO_MAX); 673132451Sroberto 674285612Sdelphij return (&sc->psc_mmio_map[num]); 675285612Sdelphij} 676285612Sdelphij 677132451Srobertostruct pcisel * 678132451Srobertopassthru_get_sel(struct passthru_softc *sc) 679182007Sroberto{ 680132451Sroberto assert(sc != NULL); 681132451Sroberto 682132451Sroberto return (&sc->psc_sel); 683132451Sroberto} 684132451Sroberto 685132451Srobertoint 686285612Sdelphijset_pcir_handler(struct passthru_softc *sc, int reg, int len, 687285612Sdelphij cfgread_handler rhandler, cfgwrite_handler whandler) 688285612Sdelphij{ 689132451Sroberto if (reg > PCI_REGMAX || reg + len > PCI_REGMAX + 1) 690132451Sroberto return (-1); 69182498Sroberto 692182007Sroberto for (int i = reg; i < reg + len; ++i) { 693285612Sdelphij assert(sc->psc_pcir_rhandler[i] == NULL || rhandler == NULL); 694285612Sdelphij assert(sc->psc_pcir_whandler[i] == NULL || whandler == NULL); 695285612Sdelphij sc->psc_pcir_rhandler[i] = rhandler; 696285612Sdelphij sc->psc_pcir_whandler[i] = whandler; 697132451Sroberto } 698132451Sroberto 699132451Sroberto return (0); 700285612Sdelphij} 70182498Sroberto 70282498Srobertostatic int 70382498Srobertopassthru_legacy_config(nvlist_t *nvl, const char *opts) 704285612Sdelphij{ 705285612Sdelphij const char *cp; 706285612Sdelphij char *tofree; 707285612Sdelphij char value[16]; 708285612Sdelphij int bus, slot, func; 709285612Sdelphij 710132451Sroberto if (opts == NULL) 71182498Sroberto return (0); 712132451Sroberto 71382498Sroberto cp = strchr(opts, ','); 714132451Sroberto 715285612Sdelphij if (strncmp(opts, "ppt", strlen("ppt")) == 0) { 716132451Sroberto tofree = strndup(opts, cp - opts); 717132451Sroberto set_config_value_node(nvl, "pptdev", tofree); 718132451Sroberto free(tofree); 719132451Sroberto } else if (sscanf(opts, "pci0:%d:%d:%d", &bus, &slot, &func) == 3 || 720132451Sroberto sscanf(opts, "pci%d:%d:%d", &bus, &slot, &func) == 3 || 72182498Sroberto sscanf(opts, "%d/%d/%d", &bus, &slot, &func) == 3) { 722285612Sdelphij snprintf(value, sizeof(value), "%d", bus); 723285612Sdelphij set_config_value_node(nvl, "bus", value); 724132451Sroberto snprintf(value, sizeof(value), "%d", slot); 725132451Sroberto set_config_value_node(nvl, "slot", value); 726285612Sdelphij snprintf(value, sizeof(value), "%d", func); 727285612Sdelphij set_config_value_node(nvl, "func", value); 72882498Sroberto } else { 729132451Sroberto EPRINTLN("passthru: invalid options \"%s\"", opts); 730132451Sroberto return (-1); 731132451Sroberto } 732285612Sdelphij 733182007Sroberto if (cp == NULL) { 734285612Sdelphij return (0); 735285612Sdelphij } 736132451Sroberto 737285612Sdelphij return (pci_parse_legacy_config(nvl, cp + 1)); 73882498Sroberto} 73982498Sroberto 74082498Srobertostatic int 741132451Srobertopassthru_init_rom(struct passthru_softc *const sc, const char *const romfile) 742132451Sroberto{ 743132451Sroberto if (romfile == NULL) { 744132451Sroberto return (0); 745132451Sroberto } 746132451Sroberto 747132451Sroberto const int fd = open(romfile, O_RDONLY); 748132451Sroberto if (fd < 0) { 74982498Sroberto warnx("%s: can't open romfile \"%s\"", __func__, romfile); 750132451Sroberto return (-1); 751132451Sroberto } 752132451Sroberto 753285612Sdelphij struct stat sbuf; 754132451Sroberto if (fstat(fd, &sbuf) < 0) { 755132451Sroberto warnx("%s: can't fstat romfile \"%s\"", __func__, romfile); 756132451Sroberto close(fd); 757132451Sroberto return (-1); 758132451Sroberto } 759132451Sroberto const uint64_t rom_size = sbuf.st_size; 760285612Sdelphij 761285612Sdelphij void *const rom_data = mmap(NULL, rom_size, PROT_READ, MAP_SHARED, fd, 762132451Sroberto 0); 763132451Sroberto if (rom_data == MAP_FAILED) { 764285612Sdelphij warnx("%s: unable to mmap romfile \"%s\" (%d)", __func__, 765285612Sdelphij romfile, errno); 766132451Sroberto close(fd); 767132451Sroberto return (-1); 768132451Sroberto } 769132451Sroberto 770285612Sdelphij void *rom_addr; 771182007Sroberto int error = pci_emul_alloc_rom(sc->psc_pi, rom_size, &rom_addr); 772285612Sdelphij if (error) { 773285612Sdelphij warnx("%s: failed to alloc rom segment", __func__); 774132451Sroberto munmap(rom_data, rom_size); 775285612Sdelphij close(fd); 77682498Sroberto return (error); 77782498Sroberto } 77882498Sroberto memcpy(rom_addr, rom_data, rom_size); 779285612Sdelphij 780285612Sdelphij sc->psc_bar[PCI_ROM_IDX].type = PCIBAR_ROM; 781285612Sdelphij sc->psc_bar[PCI_ROM_IDX].addr = (uint64_t)rom_addr; 782285612Sdelphij sc->psc_bar[PCI_ROM_IDX].size = rom_size; 783285612Sdelphij 784285612Sdelphij munmap(rom_data, rom_size); 785285612Sdelphij close(fd); 78682498Sroberto 787132451Sroberto return (0); 788132451Sroberto} 789132451Sroberto 790285612Sdelphijstatic bool 791132451Srobertopassthru_lookup_pptdev(const char *name, int *bus, int *slot, int *func) 792132451Sroberto{ 793132451Sroberto struct pci_conf_io pc; 794132451Sroberto struct pci_conf conf[1]; 795132451Sroberto struct pci_match_conf patterns[1]; 796132451Sroberto char *cp; 797285612Sdelphij 798285612Sdelphij bzero(&pc, sizeof(struct pci_conf_io)); 799132451Sroberto pc.match_buf_len = sizeof(conf); 800132451Sroberto pc.matches = conf; 801285612Sdelphij 802285612Sdelphij bzero(&patterns, sizeof(patterns)); 803132451Sroberto 804132451Sroberto /* 805132451Sroberto * The pattern structure requires the unit to be split out from 806132451Sroberto * the driver name. Walk backwards from the end of the name to 807285612Sdelphij * find the start of the unit. 808182007Sroberto */ 809285612Sdelphij cp = strchr(name, '\0'); 810285612Sdelphij assert(cp != NULL); 811132451Sroberto while (cp != name && isdigit(cp[-1])) 812285612Sdelphij cp--; 813132451Sroberto if (cp == name || !isdigit(*cp)) { 81482498Sroberto EPRINTLN("Invalid passthru device name %s", name); 815132451Sroberto return (false); 816132451Sroberto } 817132451Sroberto if ((size_t)(cp - name) + 1 > sizeof(patterns[0].pd_name)) { 818132451Sroberto EPRINTLN("Passthru device name %s is too long", name); 819132451Sroberto return (false); 820132451Sroberto } 821132451Sroberto memcpy(patterns[0].pd_name, name, cp - name); 822132451Sroberto patterns[0].pd_unit = strtol(cp, &cp, 10); 82382498Sroberto if (*cp != '\0') { 824285612Sdelphij EPRINTLN("Invalid passthru device name %s", name); 825285612Sdelphij return (false); 826285612Sdelphij } 82782498Sroberto patterns[0].flags = PCI_GETCONF_MATCH_NAME | PCI_GETCONF_MATCH_UNIT; 828132451Sroberto pc.num_patterns = 1; 829132451Sroberto pc.pat_buf_len = sizeof(patterns); 830132451Sroberto pc.patterns = patterns; 831132451Sroberto 83282498Sroberto if (ioctl(pcifd, PCIOCGETCONF, &pc) == -1) { 833132451Sroberto EPRINTLN("ioctl(PCIOCGETCONF): %s", strerror(errno)); 834132451Sroberto return (false); 83582498Sroberto } 836285612Sdelphij if (pc.status != PCI_GETCONF_LAST_DEVICE && 837309008Sdelphij pc.status != PCI_GETCONF_MORE_DEVS) { 838309008Sdelphij EPRINTLN("error returned from PCIOCGETCONF ioctl"); 839285612Sdelphij return (false); 840276072Sdelphij } 841276072Sdelphij if (pc.num_matches == 0) { 842276072Sdelphij EPRINTLN("Passthru device %s not found", name); 843285612Sdelphij return (false); 844276072Sdelphij } 845132451Sroberto 846276072Sdelphij if (conf[0].pc_sel.pc_domain != 0) { 847309008Sdelphij EPRINTLN("Passthru device %s on unsupported domain", name); 848276072Sdelphij return (false); 849276072Sdelphij } 850276072Sdelphij *bus = conf[0].pc_sel.pc_bus; 851276072Sdelphij *slot = conf[0].pc_sel.pc_dev; 852276072Sdelphij *func = conf[0].pc_sel.pc_func; 853276072Sdelphij return (true); 854276072Sdelphij} 855276072Sdelphij 856132451Srobertostatic int 857132451Srobertopassthru_init(struct pci_devinst *pi, nvlist_t *nvl) 858132451Sroberto{ 859132451Sroberto int bus, slot, func, error, memflags; 86082498Sroberto struct passthru_softc *sc; 86182498Sroberto struct passthru_dev **devpp; 862132451Sroberto struct passthru_dev *devp, *dev = NULL; 863132451Sroberto const char *value; 864132451Sroberto 86582498Sroberto sc = NULL; 866132451Sroberto error = 1; 867285612Sdelphij 868285612Sdelphij memflags = vm_get_memflags(pi->pi_vmctx); 869285612Sdelphij if (!(memflags & VM_MEM_F_WIRED)) { 870132451Sroberto warnx("passthru requires guest memory to be wired"); 871132451Sroberto return (error); 872285612Sdelphij } 873182007Sroberto 874285612Sdelphij if (pcifd < 0 && pcifd_init()) { 875285612Sdelphij return (error); 876285612Sdelphij } 877132451Sroberto 878285612Sdelphij#define GET_INT_CONFIG(var, name) do { \ 87982498Sroberto value = get_config_value_node(nvl, name); \ 88082498Sroberto if (value == NULL) { \ 88182498Sroberto EPRINTLN("passthru: missing required %s setting", name); \ 882132451Sroberto return (error); \ 883132451Sroberto } \ 884132451Sroberto var = atoi(value); \ 885132451Sroberto} while (0) 886132451Sroberto 887132451Sroberto value = get_config_value_node(nvl, "pptdev"); 888132451Sroberto if (value != NULL) { 889132451Sroberto if (!passthru_lookup_pptdev(value, &bus, &slot, &func)) 89082498Sroberto return (error); 891132451Sroberto } else { 89282498Sroberto GET_INT_CONFIG(bus, "bus"); 893132451Sroberto GET_INT_CONFIG(slot, "slot"); 894285612Sdelphij GET_INT_CONFIG(func, "func"); 895285612Sdelphij } 896285612Sdelphij 897132451Sroberto if (vm_assign_pptdev(pi->pi_vmctx, bus, slot, func) != 0) { 898132451Sroberto warnx("PCI device at %d/%d/%d is not using the ppt(4) driver", 899285612Sdelphij bus, slot, func); 900132451Sroberto goto done; 901132451Sroberto } 90282498Sroberto 903285612Sdelphij sc = calloc(1, sizeof(struct passthru_softc)); 904285612Sdelphij 905285612Sdelphij pi->pi_arg = sc; 906285612Sdelphij sc->psc_pi = pi; 907285612Sdelphij 908285612Sdelphij /* initialize config space */ 909285612Sdelphij if ((error = cfginit(pi, bus, slot, func)) != 0) 910285612Sdelphij goto done; 911285612Sdelphij 912285612Sdelphij /* initialize ROM */ 913285612Sdelphij if ((error = passthru_init_rom(sc, 914132451Sroberto get_config_value_node(nvl, "rom"))) != 0) 915132451Sroberto goto done; 91682498Sroberto 917285612Sdelphij /* Emulate most PCI header register. */ 918285612Sdelphij if ((error = set_pcir_handler(sc, 0, PCIR_MAXLAT + 1, 919285612Sdelphij passthru_cfgread_emulate, passthru_cfgwrite_emulate)) != 0) 920132451Sroberto goto done; 921132451Sroberto 922132451Sroberto /* Allow access to the physical command and status register. */ 923132451Sroberto if ((error = set_pcir_handler(sc, PCIR_COMMAND, 0x04, NULL, NULL)) != 0) 924132451Sroberto goto done; 925132451Sroberto 926132451Sroberto SET_FOREACH(devpp, passthru_dev_set) { 927132451Sroberto devp = *devpp; 928132451Sroberto assert(devp->probe != NULL); 929132451Sroberto if (devp->probe(pi) == 0) { 930132451Sroberto dev = devp; 931182007Sroberto break; 932285612Sdelphij } 933132451Sroberto } 934132451Sroberto 935132451Sroberto if (dev != NULL) { 936132451Sroberto error = dev->init(pi, nvl); 937285612Sdelphij if (error != 0) 93882498Sroberto goto done; 939182007Sroberto } 940182007Sroberto 941182007Sroberto error = 0; /* success */ 942182007Srobertodone: 943182007Sroberto if (error) { 944182007Sroberto if (dev != NULL) 945182007Sroberto dev->deinit(pi); 946182007Sroberto free(sc); 947182007Sroberto vm_unassign_pptdev(pi->pi_vmctx, bus, slot, func); 94882498Sroberto } 949182007Sroberto return (error); 950285612Sdelphij} 951182007Sroberto 952182007Srobertostatic int 953182007Srobertomsicap_access(struct passthru_softc *sc, int coff) 954182007Sroberto{ 955182007Sroberto int caplen; 956182007Sroberto 957182007Sroberto if (sc->psc_msi.capoff == 0) 958182007Sroberto return (0); 959182007Sroberto 960182007Sroberto caplen = msi_caplen(sc->psc_msi.msgctrl); 961285612Sdelphij 962285612Sdelphij if (coff >= sc->psc_msi.capoff && coff < sc->psc_msi.capoff + caplen) 963182007Sroberto return (1); 964285612Sdelphij else 965182007Sroberto return (0); 966182007Sroberto} 967285612Sdelphij 968285612Sdelphijstatic int 969285612Sdelphijmsixcap_access(struct passthru_softc *sc, int coff) 970285612Sdelphij{ 971182007Sroberto if (sc->psc_msix.capoff == 0) 972182007Sroberto return (0); 973182007Sroberto 974285612Sdelphij return (coff >= sc->psc_msix.capoff && 975182007Sroberto coff < sc->psc_msix.capoff + MSIX_CAPLEN); 976182007Sroberto} 97782498Sroberto 978285612Sdelphijstatic int 979285612Sdelphijpassthru_cfgread_default(struct passthru_softc *sc, 980285612Sdelphij struct pci_devinst *pi __unused, int coff, int bytes, uint32_t *rv) 981285612Sdelphij{ 98282498Sroberto /* 983285612Sdelphij * MSI capability is emulated. 984132451Sroberto */ 985285612Sdelphij if (msicap_access(sc, coff) || msixcap_access(sc, coff)) 986285612Sdelphij return (-1); 987285612Sdelphij 988132451Sroberto /* 989285612Sdelphij * Emulate the command register. If a single read reads both the 990285612Sdelphij * command and status registers, read the status register from the 991285612Sdelphij * device's config space. 992132451Sroberto */ 993132451Sroberto if (coff == PCIR_COMMAND) { 994285612Sdelphij if (bytes <= 2) 995285612Sdelphij return (-1); 996285612Sdelphij *rv = pci_host_read_config(&sc->psc_sel, PCIR_STATUS, 2) << 16 | 997285612Sdelphij pci_get_cfgdata16(pi, PCIR_COMMAND); 998132451Sroberto return (0); 999285612Sdelphij } 1000285612Sdelphij 1001285612Sdelphij /* Everything else just read from the device's config space */ 1002285612Sdelphij *rv = pci_host_read_config(&sc->psc_sel, coff, bytes); 1003285612Sdelphij 1004285612Sdelphij return (0); 1005285612Sdelphij} 1006285612Sdelphij 1007132451Srobertoint 1008182007Srobertopassthru_cfgread_emulate(struct passthru_softc *sc __unused, 1009285612Sdelphij struct pci_devinst *pi __unused, int coff __unused, int bytes __unused, 1010285612Sdelphij uint32_t *rv __unused) 1011285612Sdelphij{ 1012285612Sdelphij return (-1); 1013132451Sroberto} 1014285612Sdelphij 101582498Srobertostatic int 101682498Srobertopassthru_cfgread(struct pci_devinst *pi, int coff, int bytes, uint32_t *rv) 101782498Sroberto{ 1018132451Sroberto struct passthru_softc *sc; 1019132451Sroberto 1020132451Sroberto sc = pi->pi_arg; 1021285612Sdelphij 1022285612Sdelphij if (sc->psc_pcir_rhandler[coff] != NULL) 102382498Sroberto return (sc->psc_pcir_rhandler[coff](sc, pi, coff, bytes, rv)); 1024182007Sroberto 1025132451Sroberto return (passthru_cfgread_default(sc, pi, coff, bytes, rv)); 1026132451Sroberto} 1027132451Sroberto 1028285612Sdelphijstatic int 1029132451Srobertopassthru_cfgwrite_default(struct passthru_softc *sc, struct pci_devinst *pi, 1030132451Sroberto int coff, int bytes, uint32_t val) 1031132451Sroberto{ 103282498Sroberto int error, msix_table_entries, i; 1033132451Sroberto uint16_t cmd_old; 1034132451Sroberto 1035132451Sroberto /* 1036132451Sroberto * MSI capability is emulated 1037285612Sdelphij */ 1038285612Sdelphij if (msicap_access(sc, coff)) { 1039285612Sdelphij pci_emul_capwrite(pi, coff, bytes, val, sc->psc_msi.capoff, 1040285612Sdelphij PCIY_MSI); 1041285612Sdelphij error = vm_setup_pptdev_msi(pi->pi_vmctx, sc->psc_sel.pc_bus, 1042285612Sdelphij sc->psc_sel.pc_dev, sc->psc_sel.pc_func, 1043132451Sroberto pi->pi_msi.addr, pi->pi_msi.msg_data, 1044132451Sroberto pi->pi_msi.maxmsgnum); 1045132451Sroberto if (error != 0) 1046182007Sroberto err(1, "vm_setup_pptdev_msi"); 1047285612Sdelphij return (0); 1048132451Sroberto } 1049132451Sroberto 1050132451Sroberto if (msixcap_access(sc, coff)) { 1051132451Sroberto pci_emul_capwrite(pi, coff, bytes, val, sc->psc_msix.capoff, 105282498Sroberto PCIY_MSIX); 1053132451Sroberto if (pi->pi_msix.enabled) { 1054132451Sroberto msix_table_entries = pi->pi_msix.table_count; 1055132451Sroberto for (i = 0; i < msix_table_entries; i++) { 1056285612Sdelphij error = vm_setup_pptdev_msix(pi->pi_vmctx, 1057132451Sroberto sc->psc_sel.pc_bus, sc->psc_sel.pc_dev, 1058285612Sdelphij sc->psc_sel.pc_func, i, 1059285612Sdelphij pi->pi_msix.table[i].addr, 1060285612Sdelphij pi->pi_msix.table[i].msg_data, 1061285612Sdelphij pi->pi_msix.table[i].vector_control); 1062182007Sroberto 1063285612Sdelphij if (error) 1064285612Sdelphij err(1, "vm_setup_pptdev_msix"); 1065132451Sroberto } 1066285612Sdelphij } else { 106782498Sroberto error = vm_disable_pptdev_msix(pi->pi_vmctx, 1068132451Sroberto sc->psc_sel.pc_bus, sc->psc_sel.pc_dev, 106982498Sroberto sc->psc_sel.pc_func); 107082498Sroberto if (error) 107182498Sroberto err(1, "vm_disable_pptdev_msix"); 107282498Sroberto } 107382498Sroberto return (0); 107482498Sroberto } 107582498Sroberto 1076132451Sroberto#ifdef LEGACY_SUPPORT 1077132451Sroberto /* 107882498Sroberto * If this device does not support MSI natively then we cannot let 1079132451Sroberto * the guest disable legacy interrupts from the device. It is the 1080285612Sdelphij * legacy interrupt that is triggering the virtual MSI to the guest. 1081285612Sdelphij */ 1082285612Sdelphij if (sc->psc_msi.emulated && pci_msi_enabled(pi)) { 1083285612Sdelphij if (coff == PCIR_COMMAND && bytes == 2) 1084285612Sdelphij val &= ~PCIM_CMD_INTxDIS; 1085285612Sdelphij } 1086285612Sdelphij#endif 1087285612Sdelphij 108882498Sroberto pci_host_write_config(&sc->psc_sel, coff, bytes, val); 1089132451Sroberto if (coff == PCIR_COMMAND) { 109082498Sroberto cmd_old = pci_get_cfgdata16(pi, PCIR_COMMAND); 1091285612Sdelphij if (bytes == 1) 1092132451Sroberto pci_set_cfgdata8(pi, PCIR_COMMAND, val); 1093285612Sdelphij else if (bytes == 2) 1094132451Sroberto pci_set_cfgdata16(pi, PCIR_COMMAND, val); 1095132451Sroberto pci_emul_cmd_changed(pi, cmd_old); 1096132451Sroberto } 109782498Sroberto 109882498Sroberto return (0); 1099285612Sdelphij} 1100285612Sdelphij 1101285612Sdelphijint 1102132451Srobertopassthru_cfgwrite_emulate(struct passthru_softc *sc __unused, 1103132451Sroberto struct pci_devinst *pi __unused, int coff __unused, int bytes __unused, 1104132451Sroberto uint32_t val __unused) 1105132451Sroberto{ 1106182007Sroberto return (-1); 1107285612Sdelphij} 1108132451Sroberto 1109132451Srobertostatic int 1110132451Srobertopassthru_cfgwrite(struct pci_devinst *pi, int coff, int bytes, uint32_t val) 1111132451Sroberto{ 1112285612Sdelphij struct passthru_softc *sc; 1113132451Sroberto 111482498Sroberto sc = pi->pi_arg; 111582498Sroberto 111682498Sroberto if (sc->psc_pcir_whandler[coff] != NULL) 1117132451Sroberto return (sc->psc_pcir_whandler[coff](sc, pi, coff, bytes, val)); 1118132451Sroberto 111982498Sroberto return (passthru_cfgwrite_default(sc, pi, coff, bytes, val)); 1120132451Sroberto} 1121132451Sroberto 1122132451Srobertostatic void 1123285612Sdelphijpassthru_write(struct pci_devinst *pi, int baridx, uint64_t offset, int size, 1124285612Sdelphij uint64_t value) 1125285612Sdelphij{ 1126285612Sdelphij struct passthru_softc *sc; 1127285612Sdelphij struct pci_bar_ioreq pio; 1128285612Sdelphij 1129285612Sdelphij sc = pi->pi_arg; 1130132451Sroberto 113182498Sroberto if (baridx == pci_msix_table_bar(pi)) { 1132285612Sdelphij msix_table_write(sc, offset, size, value); 1133285612Sdelphij } else { 1134132451Sroberto assert(pi->pi_bar[baridx].type == PCIBAR_IO); 1135182007Sroberto assert(size == 1 || size == 2 || size == 4); 1136132451Sroberto assert(offset <= UINT32_MAX && offset + size <= UINT32_MAX); 113782498Sroberto 113882498Sroberto bzero(&pio, sizeof(pio)); 1139132451Sroberto pio.pbi_sel = sc->psc_sel; 1140132451Sroberto pio.pbi_op = PCIBARIO_WRITE; 1141182007Sroberto pio.pbi_bar = baridx; 114282498Sroberto pio.pbi_offset = (uint32_t)offset; 1143285612Sdelphij pio.pbi_width = size; 114482498Sroberto pio.pbi_value = (uint32_t)value; 1145285612Sdelphij 1146182007Sroberto (void)ioctl(pcifd, PCIOCBARIO, &pio); 1147182007Sroberto } 1148182007Sroberto} 114982498Sroberto 1150132451Srobertostatic uint64_t 1151132451Srobertopassthru_read(struct pci_devinst *pi, int baridx, uint64_t offset, int size) 115282498Sroberto{ 1153132451Sroberto struct passthru_softc *sc; 1154132451Sroberto struct pci_bar_ioreq pio; 1155132451Sroberto uint64_t val; 1156132451Sroberto 1157132451Sroberto sc = pi->pi_arg; 1158182007Sroberto 1159285612Sdelphij if (baridx == pci_msix_table_bar(pi)) { 1160132451Sroberto val = msix_table_read(sc, offset, size); 1161132451Sroberto } else { 1162132451Sroberto assert(pi->pi_bar[baridx].type == PCIBAR_IO); 1163285612Sdelphij assert(size == 1 || size == 2 || size == 4); 1164285612Sdelphij assert(offset <= UINT32_MAX && offset + size <= UINT32_MAX); 1165132451Sroberto 1166132451Sroberto bzero(&pio, sizeof(pio)); 1167285612Sdelphij pio.pbi_sel = sc->psc_sel; 1168285612Sdelphij pio.pbi_op = PCIBARIO_READ; 1169285612Sdelphij pio.pbi_bar = baridx; 1170285612Sdelphij pio.pbi_offset = (uint32_t)offset; 1171285612Sdelphij pio.pbi_width = size; 1172285612Sdelphij 1173285612Sdelphij (void)ioctl(pcifd, PCIOCBARIO, &pio); 1174285612Sdelphij 1175285612Sdelphij val = pio.pbi_value; 1176285612Sdelphij } 1177285612Sdelphij 1178285612Sdelphij return (val); 1179285612Sdelphij} 1180285612Sdelphij 1181285612Sdelphijstatic void 1182285612Sdelphijpassthru_msix_addr(struct pci_devinst *pi, int baridx, int enabled, 1183285612Sdelphij uint64_t address) 1184285612Sdelphij{ 1185285612Sdelphij struct passthru_softc *sc; 1186285612Sdelphij size_t remaining; 1187132451Sroberto uint32_t table_size, table_offset; 1188289997Sglebius 1189289997Sglebius sc = pi->pi_arg; 1190182007Sroberto table_offset = rounddown2(pi->pi_msix.table_offset, 4096); 119182498Sroberto if (table_offset > 0) { 119282498Sroberto if (!enabled) { 1193182007Sroberto if (vm_unmap_pptdev_mmio(pi->pi_vmctx, 1194182007Sroberto sc->psc_sel.pc_bus, 1195285612Sdelphij sc->psc_sel.pc_dev, 1196285612Sdelphij sc->psc_sel.pc_func, address, 1197285612Sdelphij table_offset) != 0) 1198285612Sdelphij warnx("pci_passthru: unmap_pptdev_mmio failed"); 1199182007Sroberto } else { 1200285612Sdelphij if (vm_map_pptdev_mmio(pi->pi_vmctx, sc->psc_sel.pc_bus, 1201285612Sdelphij sc->psc_sel.pc_dev, 1202285612Sdelphij sc->psc_sel.pc_func, address, 1203132451Sroberto table_offset, 1204285612Sdelphij sc->psc_bar[baridx].addr) != 0) 1205132451Sroberto warnx("pci_passthru: map_pptdev_mmio failed"); 1206182007Sroberto } 1207285612Sdelphij } 1208285612Sdelphij table_size = pi->pi_msix.table_offset - table_offset; 1209285612Sdelphij table_size += pi->pi_msix.table_count * MSIX_TABLE_ENTRY_SIZE; 1210285612Sdelphij table_size = roundup2(table_size, 4096); 1211285612Sdelphij remaining = pi->pi_bar[baridx].size - table_offset - table_size; 1212285612Sdelphij if (remaining > 0) { 1213285612Sdelphij address += table_offset + table_size; 1214285612Sdelphij if (!enabled) { 1215132451Sroberto if (vm_unmap_pptdev_mmio(pi->pi_vmctx, 1216182007Sroberto sc->psc_sel.pc_bus, 1217182007Sroberto sc->psc_sel.pc_dev, 1218285612Sdelphij sc->psc_sel.pc_func, address, 1219285612Sdelphij remaining) != 0) 1220285612Sdelphij warnx("pci_passthru: unmap_pptdev_mmio failed"); 1221182007Sroberto } else { 1222285612Sdelphij if (vm_map_pptdev_mmio(pi->pi_vmctx, sc->psc_sel.pc_bus, 1223182007Sroberto sc->psc_sel.pc_dev, 1224182007Sroberto sc->psc_sel.pc_func, address, 1225182007Sroberto remaining, 1226182007Sroberto sc->psc_bar[baridx].addr + 1227182007Sroberto table_offset + table_size) != 0) 1228285612Sdelphij warnx("pci_passthru: map_pptdev_mmio failed"); 1229182007Sroberto } 1230182007Sroberto } 1231285612Sdelphij} 1232285612Sdelphij 1233285612Sdelphijstatic void 1234285612Sdelphijpassthru_mmio_addr(struct pci_devinst *pi, int baridx, int enabled, 1235285612Sdelphij uint64_t address) 1236182007Sroberto{ 1237182007Sroberto struct passthru_softc *sc; 1238182007Sroberto 1239285612Sdelphij sc = pi->pi_arg; 124082498Sroberto if (!enabled) { 124182498Sroberto if (vm_unmap_pptdev_mmio(pi->pi_vmctx, sc->psc_sel.pc_bus, 124282498Sroberto sc->psc_sel.pc_dev, 1243132451Sroberto sc->psc_sel.pc_func, address, 124482498Sroberto sc->psc_bar[baridx].size) != 0) 1245132451Sroberto warnx("pci_passthru: unmap_pptdev_mmio failed"); 1246285612Sdelphij } else { 1247285612Sdelphij if (vm_map_pptdev_mmio(pi->pi_vmctx, sc->psc_sel.pc_bus, 1248285612Sdelphij sc->psc_sel.pc_dev, 1249182007Sroberto sc->psc_sel.pc_func, address, 1250285612Sdelphij sc->psc_bar[baridx].size, 1251182007Sroberto sc->psc_bar[baridx].addr) != 0) 1252182007Sroberto warnx("pci_passthru: map_pptdev_mmio failed"); 125382498Sroberto } 125482498Sroberto} 125582498Sroberto 1256132451Srobertostatic void 125782498Srobertopassthru_addr_rom(struct pci_devinst *const pi, const int idx, 1258132451Sroberto const int enabled) 1259182007Sroberto{ 1260285612Sdelphij const uint64_t addr = pi->pi_bar[idx].addr; 1261182007Sroberto const uint64_t size = pi->pi_bar[idx].size; 1262182007Sroberto 1263132451Sroberto if (!enabled) { 1264132451Sroberto if (vm_munmap_memseg(pi->pi_vmctx, addr, size) != 0) { 126582498Sroberto errx(4, "%s: munmap_memseg @ [%016lx - %016lx] failed", 1266132451Sroberto __func__, addr, addr + size); 126782498Sroberto } 1268132451Sroberto 1269285612Sdelphij } else { 1270285612Sdelphij if (vm_mmap_memseg(pi->pi_vmctx, addr, VM_PCIROM, 1271285612Sdelphij pi->pi_romoffset, size, PROT_READ | PROT_EXEC) != 0) { 1272182007Sroberto errx(4, "%s: mmap_memseg @ [%016lx - %016lx] failed", 1273285612Sdelphij __func__, addr, addr + size); 1274182007Sroberto } 1275182007Sroberto } 127682498Sroberto} 127782498Sroberto 127882498Srobertostatic void 1279132451Srobertopassthru_addr(struct pci_devinst *pi, int baridx, int enabled, uint64_t address) 128082498Sroberto{ 1281132451Sroberto switch (pi->pi_bar[baridx].type) { 1282182007Sroberto case PCIBAR_IO: 1283285612Sdelphij /* IO BARs are emulated */ 1284182007Sroberto break; 1285182007Sroberto case PCIBAR_ROM: 1286132451Sroberto passthru_addr_rom(pi, baridx, enabled); 1287132451Sroberto break; 1288132451Sroberto case PCIBAR_MEM32: 1289132451Sroberto case PCIBAR_MEM64: 1290132451Sroberto if (baridx == pci_msix_table_bar(pi)) 1291132451Sroberto passthru_msix_addr(pi, baridx, enabled, address); 1292285612Sdelphij else 1293285612Sdelphij passthru_mmio_addr(pi, baridx, enabled, address); 1294285612Sdelphij break; 1295182007Sroberto default: 1296285612Sdelphij errx(4, "%s: invalid BAR type %d", __func__, 1297182007Sroberto pi->pi_bar[baridx].type); 1298182007Sroberto } 129982498Sroberto} 130082498Sroberto 130182498Srobertostatic const struct pci_devemu passthru = { 1302132451Sroberto .pe_emu = "passthru", 130382498Sroberto .pe_init = passthru_init, 1304132451Sroberto .pe_legacy_config = passthru_legacy_config, 1305182007Sroberto .pe_cfgwrite = passthru_cfgwrite, 1306285612Sdelphij .pe_cfgread = passthru_cfgread, 1307182007Sroberto .pe_barwrite = passthru_write, 1308182007Sroberto .pe_barread = passthru_read, 1309132451Sroberto .pe_baraddr = passthru_addr, 1310132451Sroberto}; 1311132451SrobertoPCI_EMUL_SET(passthru); 1312132451Sroberto